<commit_before>create_sql = """
    create table sample_work_item (
        uuid char(36) unique not null,
        organisation_id int not null,
        test_id int not null,
        activity_id varchar(100) not null,
        package_id varchar(100) not null,
        xml_data text not null,
        xml_parent_data text,
        test_kind varchar(20) not null
    );
"""
create_sql2 = """
    create table sample_result (
        uuid char(36) unique not null,
        response int not null,
        unsure int not null
    );
"""
create_sql_offered = """
    create table sample_offer (
        uuid char(36) unique not null,
        offered bool not null
    );
"""

create_sql3 = """
    create view sample_full as
        select * from sample_work_item
            left join sample_result using (uuid)
            left join sample_offer using (uuid);
"""

from sqlite3 import dbapi2 as sqlite
import sqlite3
import os
import config

keys = ["uuid", "organisation_id", "test_id", "activity_id", "package_id",
        "xml_data", "xml_parent_data", "test_kind"]

keys_response = ["uuid", "organisation_id", "test_id", "activity_id",
    "package_id", "xml_data", "xml_parent_data", "test_kind", "response",
    "unsure"]

def default_filename():
    return config.DB_FILENAME

def make_db(filename, work_items):
    if os.path.exists(filename):
        os.unlink(filename)

    database = sqlite.connect(filename)
    c = database.cursor()

    stmts = [create_sql, create_sql2, create_sql_offered, create_sql3]

    [ c.execute(stmt) for stmt in stmts ]

    for wi in work_items:
        wi_info = tuple(map(lambda k: wi[k], keys))

        c.execute("""insert into sample_work_item
                        ("uuid", "organisation_id", "test_id", "activity_id",
                         "package_id", "xml_data",
                         "xml_parent_data", "test_kind")
                        values (?,?,?,?,?,?,?,?);
                  """, wi_info)

    database.commit()

def read_db(filename):
    database = sqlite.connect(filename)
    c = database.cursor()

    c.execute("""select "uuid", "organisation_id", "test_id", "activity_id",
                         "package_id", "xml_data", "xml_parent_data",
                         "test_kind"
                 from sample_work_item;""")

    for wi in c.fetchall():
        data = dict([ (keys[i], wi[i]) for i in range(0, 8) ])

        yield data

def read_db_response():
    filename = default_filename()

    database = sqlite.connect(filename)
    c = database.cursor()

    c.execute("""select sample_work_item.uuid as uuid,
                sample_work_item.organisation_id as organisation_id,
                sample_work_item.test_id as test_id,
                sample_work_item.activity_id as activity_id,
                sample_work_item.package_id as package_id,
                sample_work_item.xml_data as xml_data,
                sample_work_item.xml_parent_data as xml_parent_data,
                sample_work_item.test_kind as test_kind,
                sample_result.response as response,
                sample_result.unsure as unsure
                from sample_work_item
                left join sample_result on
                sample_work_item.uuid=sample_result.uuid;""")

    for wi in c.fetchall():
        data = dict([ (keys_response[i], wi[i]) for i in range(0, 10) ])
        yield data

def work_item_generator(f):
    filename = default_filename()

    for wi in read_db(filename):
        yield f(wi)

def save_response(work_item_uuid, response, unsure=False):
    filename = default_filename()
    database = sqlite.connect(filename)
    c = database.cursor()

    try:
        c.execute('''insert into sample_result ("uuid", "response", "unsure")
                       values (?, ?, ?);''', (work_item_uuid, response, unsure))
    except sqlite3.IntegrityError:
        database.rollback()
        raise

    database.commit()
<commit_msg>try to suppress already offered sample items<commit_after>create_sql = """
    create table sample_work_item (
        uuid char(36) unique not null,
        organisation_id int not null,
        test_id int not null,
        activity_id varchar(100) not null,
        package_id varchar(100) not null,
        xml_data text not null,
        xml_parent_data text,
        test_kind varchar(20) not null
    );
"""
create_sql2 = """
    create table sample_result (
        uuid char(36) unique not null,
        response int not null,
        unsure int not null
    );
"""
create_sql_offered = """
    create table sample_offer (
        uuid char(36) unique not null,
        offered bool not null
    );
"""

create_sql3 = """
    create view sample_full as
        select * from sample_work_item
            left join sample_result using (uuid)
            left join sample_offer using (uuid);
"""

from sqlite3 import dbapi2 as sqlite
import sqlite3
import os
import config

keys = ["uuid", "organisation_id", "test_id", "activity_id", "package_id",
        "xml_data", "xml_parent_data", "test_kind"]

keys_response = ["uuid", "organisation_id", "test_id", "activity_id",
    "package_id", "xml_data", "xml_parent_data", "test_kind", "response",
    "unsure"]

def default_filename():
    return config.DB_FILENAME

def make_db(filename, work_items):
    if os.path.exists(filename):
        os.unlink(filename)

    database = sqlite.connect(filename)
    c = database.cursor()

    stmts = [create_sql, create_sql2, create_sql_offered, create_sql3]

    [ c.execute(stmt) for stmt in stmts ]

    for wi in work_items:
        wi_info = tuple(map(lambda k: wi[k], keys))

        c.execute("""insert into sample_work_item
                        ("uuid", "organisation_id", "test_id", "activity_id",
                         "package_id", "xml_data",
                         "xml_parent_data", "test_kind")
                        values (?,?,?,?,?,?,?,?);
                  """, wi_info)

    database.commit()

def read_db(filename):
    database = sqlite.connect(filename)
    c = database.cursor()

    c.execute("""select "uuid", "organisation_id", "test_id", "activity_id",
                         "package_id", "xml_data", "xml_parent_data",
                         "test_kind"
                 from sample_full where offered is null limit 1;""")

    for wi in c.fetchall():
        data = dict([ (keys[i], wi[i]) for i in range(0, 8) ])

        uuid = wi[0] # hack
        c2 = database.cursor()
        c2.execute("""insert into sample_offer ("uuid", "offered")
                        values (?, ?)""", (uuid, True))
        db.commit()

        yield data

def read_db_response():
    filename = default_filename()

    database = sqlite.connect(filename)
    c = database.cursor()

    c.execute("""select sample_work_item.uuid as uuid,
                sample_work_item.organisation_id as organisation_id,
                sample_work_item.test_id as test_id,
                sample_work_item.activity_id as activity_id,
                sample_work_item.package_id as package_id,
                sample_work_item.xml_data as xml_data,
                sample_work_item.xml_parent_data as xml_parent_data,
                sample_work_item.test_kind as test_kind,
                sample_result.response as response,
                sample_result.unsure as unsure
                from sample_work_item
                left join sample_result on
                sample_work_item.uuid=sample_result.uuid;""")

    for wi in c.fetchall():
        data = dict([ (keys_response[i], wi[i]) for i in range(0, 10) ])
        yield data

def work_item_generator(f):
    filename = default_filename()

    for wi in read_db(filename):
        yield f(wi)

def save_response(work_item_uuid, response, unsure=False):
    filename = default_filename()
    database = sqlite.connect(filename)
    c = database.cursor()

    try:
        c.execute('''insert into sample_result ("uuid", "response", "unsure")
                       values (?, ?, ?);''', (work_item_uuid, response, unsure))
    except sqlite3.IntegrityError:
        database.rollback()
        raise

    database.commit()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import numpy as np
import random
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.externals import joblib
import random
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from math import floor
import argparse
import MAP_eval as MAP
import data_parser as reader

FLAG = None
color_list = ['firebrick','red','darkorange','orange','forestgreen','lime',
    'aqua', 'dodgerblue','orchid', 'darkmagenta']

def average_over_words(feat_dic):
    '''Assume feat_dic[word_ID] is a list of word utterance
      args:
        feat_dic: a dictionary with
                  feat_dic[word_ID] = shape(num_occur, feat_dim)
      returns:
        dic: a dict with dic[word_ID] = average over the word utterance.

    '''
    dic= {}
    for i in feat_dic:
        feat_dim = len(feat_dic[i][0])
        dic[i] = [0. for k in range(feat_dim)]
        for feat in feat_dic[i]:
            for k in range(len(feat)):
                dic[i][k] += feat[k]
        for k in range(feat_dim):
            dic[i][k] /= len(feat_dic[i])

    return dic

def PCA_transform(feats):
    pca = PCA(n_components=FLAG.pca_dim)
    pca.fit(feats)
    feat_trans = pca.transform(feats)
    return feat_trans, pca

def plot_with_anno(f_2d, list_dic, rev_dic, ax):
    x = []
    y = []
    for i, f in enumerate(f_2d):
        x.append(f[0])
        y.append(f[1])

    ax.scatter(x, y, color='k')
    for i in range(len(list_dic)):
        ax.annotate(rev_dic[list_dic[i]], (x[i],y[i]))

    return ax

def plot_with_anno_3d(f_3d, list_dic, rev_dic, ax):
    x = []
    y = []
    z = []
    for i, f in enumerate(f_3d):
        x.append(f[0])
        y.append(f[1])
        z.append(f[2])
    ax.scatter(x, y, z, color='k', label="average_words")
    for i in range(len(list_dic)):
        ax.text(x[i],y[i],z[i],rev_dic[list_dic[i]])
    return ax

def plot_all_color(f_2d, delta_lab, rev_dic, ax, word_color_dict):
    x = []
    y = []
    for i, f in enumerate(f_2d):
        x.append(f[0])
        y.append(f[1])

    start = 0
    for i in range(len(delta_lab)):
        delta, label = delta_lab[i]
        word_label = rev_dic[label]
        ax.scatter(x[start:start+delta],y[start:start+delta],
            color=word_color_dict[label], label=word_label )
        start += delta
    return  ax

def plot_all_color_3d(f_3d, delta_lab, rev_dic, ax, word_color_dict):
    x = []
    y = []
    z = []
    for i, f in enumerate(f_3d):
        x.append(f[0])
        y.append(f[1])
        z.append(f[2])
    start = 0
    for i in range(len(delta_lab)):
        delta, label = delta_lab[i]
        word_label = rev_dic[label]
        ax.scatter(x[start:start+delta], y[start:start+delta],
            z[start:start+delta],
            color=word_color_dict[label],
            label=word_label)
        start += delta
    return ax

def sampling(f,delta_lab):
    ''' sample through the same label
    args:
      f: the feats of a word utterance, shape = [num_occur, feat_dim]
      delta_lab: It's a list that combines the label and the occurance time

    returns:
      new_f, new_delta_lab
    '''
    new_f = []
    new_delta_lab = []
    sample_num = FLAG.sample_num

    start = 0
    for i in delta_lab:
        delta = i[0]
        lab = i[1]
        if delta <= sample_num :
            new_f.extend(f[start:start+delta])
            new_delta_lab.append((delta,lab))
        else:
            tmp_f = random.sample(range(delta), sample_num)
            for i in tmp_f:
                new_f.append(f[start+i])
            new_delta_lab.append((sample_num, lab))
        start += delta

    return new_f, new_delta_lab

def extract_targets(all_feats, all_labs, targets):
    target_feat_dic = {}
    for ID in targets:
        target_feat_dic[ID] = []
    for i, ID in enumerate(all_labs):
        if ID in targets:
            target_feat_dic[ID].append(all_feats[i])

    return target_feat_dic

def target_dic2list(feat_dic):
    feat_list = []
    delta_lab_list = []
    for i in feat_dic:
        num_occur = len(feat_dic[i])
        for j in feat_dic[i]:
            feat_list.append(j)
        delta_lab_list.append((num_occur,i))

    return feat_list, delta_lab_list

def average_over_words_num(feat_dic, target_list):
    num = FLAG.ave_num
    feat_lists = []
    for i in target_list:
        iter_num = int(floor(float(len(feat_dic[i]))/num))
        feat_dim = len(feat_dic[i][0])
        for j in range(min(30,iter_num)):
            l = [0. for tmp in range(feat_dim)]
            for k in range(feat_dim):
                l[k] += feat_dic[i][j][k]/num
            feat_lists.append(l)
    return feat_lists

def plot_additional_words(word_list, ):

    return

def main():
    ### preprocessing ###
    feats, labs = reader.read_csv_file(FLAG.train_file)
    dic, rev_dic = reader.build_dic(FLAG.word_dic)
    targets = reader.build_targets(FLAG.target_words, dic)
    test_feat_dic = extract_targets(feats, labs, targets)
    word_color_dict = reader.build_label_color_list(targets, color_list)

    ### PCA through the average target words ###
    ave_test_feat_dic = average_over_words(test_feat_dic)
    ave_test_feat_list = [ ave_test_feat_dic[i] for i in ave_test_feat_dic]

    ave_num_feat_lists = average_over_words_num(test_feat_dic, targets)
    ave_num_trans, model = PCA_transform(ave_num_feat_lists)

    anno_list = [ i for i in ave_test_feat_dic ]
    ave_test_trans = model.transform(ave_test_feat_list)

    ### use the PCA model to transform only testing data ###
    all_feats, delta_lab_list = target_dic2list(test_feat_dic)
    all_feats_trans = model.transform(all_feats)
    ### samples number of word occurances  ###

    sampled_feats, sampled_delta_lab = sampling(all_feats_trans, delta_lab_list)

    if FLAG.save_model :
        s = joblib.dumps(model,FLAG.model_fn)

    fig = plt.figure()
    if FLAG.pca_dim == 2:
        ### start plotting the average results ###
        ax = fig.add_subplot(111)
        ax = plot_with_anno(ave_test_trans, anno_list, rev_dic, ax)
        ### plotting all word utterance ###
        ax = plot_all_color(sampled_feats, sampled_delta_lab, rev_dic, ax,
            word_color_dict)

    elif FLAG.pca_dim ==3 :
        #### start plotting the 3D projections ####
        ax = fig.add_subplot(111, projection='3d')
        ax = plot_with_anno_3d(ave_test_trans, anno_list, rev_dic, ax)
        ax = plot_all_color_3d(sampled_feats, sampled_delta_lab, rev_dic, ax,
            word_color_dict)

    else:
        print ("no plotting but testing through MAP")

        all_list = []
        feat_trans = model.transform(feats)
        print (len(feat_trans[0]))
        for i in range(len(feats)):
            all_list.append((feat_trans[i],labs[i]))

        train_list, test_list = MAP.split_train_test(all_list)
        print (MAP.MAP(test_list[:100], train_list, feat_dim=FLAG.pca_dim))

        return

    ax.legend(loc='upper right')
    plt.show()
    return

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='do the PCA transform and plot the output')
    parser.add_argument('train_file',
        help='the training file with feats and labels')
    parser.add_argument('word_dic',
        help='the dictionary of the label')
    parser.add_argument('target_words',
        help='main PCA works on only the specific words')
    parser.add_argument('--sample_num',type=int, default=10,
        help='the number for sampling while plotting, '
             'default=10')
    parser.add_argument('--ave_num',type=int, default=10,
        help='the number for averaging over the word occurance and '
             'get the average feat vectors')
    parser.add_argument('--pca_dim',type=int,default=2,
        help='the dimension to project the transformed PCA features. '
             'default=2')
    parser.add_argument('--save_model', type=bool, default=False,
        help='save the PCA model or not. if True ,save if as'
             'pca_model')
    parser.add_argument('--model_fn',type=str, default='pca.mdl',
        help='the model name to save/load')

    FLAG = parser.parse_args()

    main()

<commit_msg>update PCA :<commit_after>#!/usr/bin/env python3

import numpy as np
import random
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.externals import joblib
import random
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from math import floor
import argparse
import MAP_eval as MAP
import data_parser as reader

FLAG = None
color_list = ['firebrick','red','darkorange','orange','forestgreen','lime',
    'aqua', 'dodgerblue','orchid', 'darkmagenta']

def average_over_words(feat_dic):
    '''Assume feat_dic[word_ID] is a list of word utterance
      args:
        feat_dic: a dictionary with
                  feat_dic[word_ID] = shape(num_occur, feat_dim)
      returns:
        dic: a dict with dic[word_ID] = average over the word utterance.
    '''

    dic= {}
    for i in feat_dic:
        feat_dim = len(feat_dic[i][0])
        dic[i] = [0. for k in range(feat_dim)]
        for feat in feat_dic[i]:
            for k in range(len(feat)):
                dic[i][k] += feat[k]
        for k in range(feat_dim):
            dic[i][k] /= len(feat_dic[i])

    return dic

def PCA_transform(feats):
    ''' Process PCA to transform feat
    args:
      feat: feature list, shape = (feat_num, feat_dim)

    returns:
      feat_trans: the transformed feature list,
        shape = (feat_num, pca_project_dim)
      pca: the trained pca model
    '''

    pca = PCA(n_components=FLAG.pca_dim)
    pca.fit(feats)
    feat_trans = pca.transform(feats)
    return feat_trans, pca

def plot_with_anno(f_2d, list_dic, rev_dic, ax):
    ''' Plot 2d figurewith annotation
    args:
      f_2d: feature 2d list, shape = (feat_num, 2)
      list_dic: the list of annotations
      rev_dic: the reverse dictionary with rev_dic[ID]=word
      ax: the subplot of plt
    returns:
      ax: the updated subplot
    '''
    x = []
    y = []
    for i, f in enumerate(f_2d):
        x.append(f[0])
        y.append(f[1])

    ax.scatter(x, y, color='k')
    for i in range(len(list_dic)):
        ax.annotate(rev_dic[list_dic[i]], (x[i],y[i]))

    return ax

def plot_with_anno_3d(f_3d, list_dic, rev_dic, ax):
    ''' Plot 3d figure with annotation
    args:
      f_3d: feature 2d list, shape = (feat_num, 2)
      list_dic: the list of annotations
      rev_dic: the reverse dictionary with rev_dic[ID]=word
      ax: the subplot of plt
    returns:
      ax: the updated subplot
    '''
    x = []
    y = []
    z = []
    for i, f in enumerate(f_3d):
        x.append(f[0])
        y.append(f[1])
        z.append(f[2])
    ax.scatter(x, y, z, color='k', label="average_words")
    for i in range(len(list_dic)):
        ax.text(x[i],y[i],z[i],rev_dic[list_dic[i]])
    return ax

def plot_all_color(f_2d, delta_lab, rev_dic, ax, word_color_dict):
    ''' 2d plotting
    args:
      f_2d: the 3d features
      delta_lab: the delta range and the label
      rev_dic: reverse dictionary, dic[ID] = word
      ax: the subplot of the main plot
      word_color_dict: a dictionary with
           key = word, value = color
    returns:
      ax: the updated subplot
    '''

    x = []
    y = []
    for i, f in enumerate(f_2d):
        x.append(f[0])
        y.append(f[1])

    start = 0
    for i in range(len(delta_lab)):
        delta, label = delta_lab[i]
        word_label = rev_dic[label]
        ax.scatter(x[start:start+delta],y[start:start+delta],
            color=word_color_dict[label], label=word_label )
        start += delta
    return  ax

def plot_all_color_3d(f_3d, delta_lab, rev_dic, ax, word_color_dict):
    ''' 3d plotting
    args:
      f_3d: the 3d features
      delta_lab: the delta range and the label
      rev_dic: reverse dictionary, dic[ID] = word
      ax: the subplot of the main plot
      word_color_dict: a dictionary with
           key = word, value = color
    returns:
      ax: the updated subplot
    '''

    x = []
    y = []
    z = []
    for i, f in enumerate(f_3d):
        x.append(f[0])
        y.append(f[1])
        z.append(f[2])
    start = 0
    for i in range(len(delta_lab)):
        delta, label = delta_lab[i]
        word_label = rev_dic[label]
        ax.scatter(x[start:start+delta], y[start:start+delta],
            z[start:start+delta],
            color=word_color_dict[label],
            label=word_label)
        start += delta
    return ax

def sampling(f,delta_lab):
    ''' sample through the same label
    args:
      f: the feats of a word utterance, shape = [num_occur, feat_dim]
      delta_lab: It's a list that combines the label and the occurance time

    returns:
      new_f, new_delta_lab
    '''
    new_f = []
    new_delta_lab = []
    sample_num = FLAG.sample_num

    start = 0
    for i in delta_lab:
        delta = i[0]
        lab = i[1]
        if delta <= sample_num :
            new_f.extend(f[start:start+delta])
            new_delta_lab.append((delta,lab))
        else:
            tmp_f = random.sample(range(delta), sample_num)
            for i in tmp_f:
                new_f.append(f[start+i])
            new_delta_lab.append((sample_num, lab))
        start += delta

    return new_f, new_delta_lab

def extract_targets(all_feats, all_labs, targets):
    ''' extract the target feats in all feats

    '''
    target_feat_dic = {}
    for ID in targets:
        target_feat_dic[ID] = []
    for i, ID in enumerate(all_labs):
        if ID in targets:
            target_feat_dic[ID].append(all_feats[i])

    return target_feat_dic

def target_dic2list(feat_dic):
    ''' Transform the feat_dic to target
    args:
      feat_dic: feat_dic[word_ID] = feat lists, shape = (num_occur, feat_dim)

    returns:
      feat_list: feat_list with shape [ total_feat_num, feat_dim ]
      delta_lab_list: the range of delta and the label list
    '''
    feat_list = []
    delta_lab_list = []
    for i in feat_dic:
        num_occur = len(feat_dic[i])
        for j in feat_dic[i]:
            feat_list.append(j)
        delta_lab_list.append((num_occur,i))

    return feat_list, delta_lab_list

def average_over_words_num(feat_dic, target_list):
    ''' given the FLAG.ave_num, average over every FLAG.ave_num occurance
        of words and returns the feat list
    args:
      feat_dic: feat_dic[word_ID] = feat lists, shape = (num_occur, feat_dim)
      target_list: the target word lists

    retruns:
      feat_lists: averaged 2-dimension matrix, [feat_num, feat_dim]
    '''
    num = FLAG.ave_num
    feat_lists = []
    for i in target_list:
        iter_num = int(floor(float(len(feat_dic[i]))/num))
        feat_dim = len(feat_dic[i][0])
        for j in range(min(30,iter_num)):
            l = [0. for tmp in range(feat_dim)]
            for k in range(feat_dim):
                l[k] += feat_dic[i][j][k]/num
            feat_lists.append(l)
    return feat_lists

def extract_additional_words(feat_dic,word_list, word_color_dict ):
    '''given the FLAG.ave_num, average over every FLAG.ave_num occurance
       and each word contains only one avearage vector
    args:
      feat_dic: feat_dic[word_ID] = lists of feats, shape=(num_occur, feat_dim)
      word_list: the plotting word list
      word_color_dict: the plotting word color dictionary

    returns:
      ave_feat_lb_color:
    '''
    ave_feat_lb_color = []
    for i in word_list:
        num_occur = min(FLAG.ave_num, len(feat_dic[i]))
        feat_dim = len(feat_dic[i][0])
        feat_list = [ 0. for i in range(feat_dim)]
        for j in range(num_occur):
            for k in range(feat_dim):
                feat_list[k] += feat_dic[i][j][k]

        for k in range(feat_dim):
            feat_list[k] /= num_occur

        ave_feat_lb_color.append((feat_list, i, word_color_dict[i]))

    return ave_feat_lb_color

def plot_additional_words(ave_ftrans_lb_color,ax):
    '''plot the additional words, i.e. not used for transforming words
    args:
      ave_ftrans_lb_color: the average feature list,
        each object in the list contains:
          ([ feat_dim ], label(word_id), color of the word)
      ax: the subplot of the figure

    returns:
      ax: the updated subplot
    '''

    return

def main():
    ### preprocessing ###
    feats, labs = reader.read_csv_file(FLAG.train_file)
    dic, rev_dic = reader.build_dic(FLAG.word_dic)
    targets = reader.build_targets(FLAG.target_words, dic)
    test_feat_dic = extract_targets(feats, labs, targets)
    word_color_dict = reader.build_label_color_list(targets, color_list)

    ### PCA through the average target words ###
    ave_test_feat_dic = average_over_words(test_feat_dic)
    ave_test_feat_list = [ ave_test_feat_dic[i] for i in ave_test_feat_dic]

    ave_num_feat_lists = average_over_words_num(test_feat_dic, targets)
    ave_num_trans, model = PCA_transform(ave_num_feat_lists)

    anno_list = [ i for i in ave_test_feat_dic ]
    ave_test_trans = model.transform(ave_test_feat_list)

    ### use the PCA model to transform only testing data ###
    all_feats, delta_lab_list = target_dic2list(test_feat_dic)
    all_feats_trans = model.transform(all_feats)
    ### samples number of word occurances  ###

    sampled_feats, sampled_delta_lab = sampling(all_feats_trans, delta_lab_list)

    if FLAG.save_model :
        s = joblib.dumps(model,FLAG.model_fn)

    fig = plt.figure()
    if FLAG.pca_dim == 2:
        ### start plotting the average results ###
        ax = fig.add_subplot(111)
        ax = plot_with_anno(ave_test_trans, anno_list, rev_dic, ax)
        ### plotting all word utterance ###
        ax = plot_all_color(sampled_feats, sampled_delta_lab, rev_dic, ax,
            word_color_dict)

    elif FLAG.pca_dim ==3 :
        #### start plotting the 3D projections ####
        ax = fig.add_subplot(111, projection='3d')
        ax = plot_with_anno_3d(ave_test_trans, anno_list, rev_dic, ax)
        ax = plot_all_color_3d(sampled_feats, sampled_delta_lab, rev_dic, ax,
            word_color_dict)

    else:
        print ("no plotting but testing through MAP")

        all_list = []
        feat_trans = model.transform(feats)
        print (len(feat_trans[0]))
        for i in range(len(feats)):
            all_list.append((feat_trans[i],labs[i]))

        train_list, test_list = MAP.split_train_test(all_list)
        print (MAP.MAP(test_list[:100], train_list, feat_dim=FLAG.pca_dim))

        return

    ax.legend(loc='upper right')
    plt.show()
    return

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='do the PCA transform and plot the output')
    parser.add_argument('train_file',
        help='the training file with feats and labels')
    parser.add_argument('word_dic',
        help='the dictionary of the label')
    parser.add_argument('target_words',
        help='main PCA works on only the specific words')
    parser.add_argument('--sample_num',type=int, default=10,
        help='the number for sampling while plotting, '
             'default=10')
    parser.add_argument('--ave_num',type=int, default=10,
        help='the number for averaging over the word occurance and '
             'get the average feat vectors')
    parser.add_argument('--pca_dim',type=int,default=2,
        help='the dimension to project the transformed PCA features. '
             'default=2')
    parser.add_argument('--save_model', type=bool, default=False,
        help='save the PCA model or not. if True ,save if as'
             'pca_model')
    parser.add_argument('--model_fn',type=str, default='pca.mdl',
        help='the model name to save/load')

    FLAG = parser.parse_args()

    main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from flask import flash, redirect, render_template, request, g, url_for, jsonify, make_response
from sqlalchemy.exc import IntegrityError
from baseframe import _
from baseframe import forms
from baseframe.forms import render_form
from coaster.views import load_models
from .. import app, lastuser
from ..models import (db, Profile, ProposalSpace, Attendee, ProposalSpaceRedirect, Participant, Event, ContactExchange)
from ..forms import ParticipantForm
from funnel.util import split_name, format_twitter_handle, make_qrcode

def participant_badge_data(participants, space):
    badges = []
    for participant in participants:
        first_name, last_name = split_name(participant.fullname)
        badges.append({
            'first_name': first_name,
            'last_name': last_name,
            'twitter': format_twitter_handle(participant.twitter),
            'company': participant.company,
            'qrcode_content': make_qrcode(u"{puk}{key}".format(puk=participant.puk, key=participant.key))
        })
    return badges

def participant_data(participant, space_id, full=False):
    if full:
        return {
            '_id': participant.id,
            'puk': participant.puk,
            'fullname': participant.fullname,
            'job_title': participant.job_title,
            'company': participant.company,
            'email': participant.email,
            'twitter': participant.twitter,
            'phone': participant.phone,
            'space_id': space_id
        }
    else:
        return {
            '_id': participant.id,
            'puk': participant.puk,
            'fullname': participant.fullname,
            'job_title': participant.job_title,
            'company': participant.company,
            'space_id': space_id
        }

def participant_checkin_data(participant, space, event):
    return {
        'pid': participant.id,
        'fullname': participant.fullname,
        'company': participant.company,
        'email': participant.email,
        'badge_printed': participant.badge_printed,
        'checked_in': participant.checked_in
    }

@app.route('/<space>/participants/json', subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    permission='view')
def participants_json(profile, space):
    return jsonify(participants=[participant_data(participant, space.id) for participant in space.participants])

@app.route('/<space>/participants/new', methods=['GET', 'POST'], subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    permission='new-participant')
def new_participant(profile, space):
    form = ParticipantForm()
    form.events.query = space.events
    if form.validate_on_submit():
        participant = Participant(proposal_space=space)
        form.populate_obj(participant)
        try:
            db.session.add(participant)
            db.session.commit()
        except IntegrityError:
            db.session.rollback()
            flash(_(u"This participant already exists."), 'info')
        return redirect(space.url_for('admin'), code=303)
    return render_form(form=form, title=_(u"New Participant"), submit=_(u"Add Participant"))

@app.route('/<space>/participant/<participant_id>/edit', methods=['GET', 'POST'], subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    (Participant, {'id': 'participant_id'}, 'participant'),
    permission='edit-participant')
def participant_edit(profile, space, participant):
    form = ParticipantForm(obj=participant, model=Participant)
    form.events.query = space.events
    if form.validate_on_submit():
        form.populate_obj(participant)
        db.session.commit()
        flash(_(u"Your changes have been saved"), 'info')
        return redirect(space.url_for('admin'), code=303)
    return render_form(form=form, title=_(u"Edit Participant"), submit=_(u"Save changes"))

@app.route('/<space>/participant', subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    permission='view')
def participant(profile, space):
    participant = Participant.query.filter_by(puk=request.args.get('puk')).first()
    if not participant:
        return jsonify(message=u"Not found", code=404)
    elif participant.key == request.args.get('key'):
        try:
            contact_exchange = ContactExchange(user_id=g.user.id, participant_id=participant.id, proposal_space_id=space.id)
            db.session.add(contact_exchange)
            db.session.commit()
        except IntegrityError:
            app.logger.warning(u"Contact Exchange already present")
            db.session.rollback()
        return jsonify(participant=participant_data(participant, space.id, full=True))
    else:
        return jsonify(message=u"Unauthorized", code=401)

@app.route('/<space>/participant/<participant_id>/badge', subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    (Participant, {'id': 'participant_id'}, 'participant'),
    permission='view-participant')
def participant_badge(profile, space, participant):
    badge_type = request.args.getlist('type')
    badge_template = 'https://images.hasgeek.com/embed/file/ba026de2a77b4fba8e7bb5afcedc215b'
    if badge_type:
        if 'blank' in badge_type:
            badge_template = 'https://images.hasgeek.com/embed/file/24e9bafb1f6445038c8f5d5026b410b9'
            return render_template('blank_badge.html', badge_template=badge_template, badges=participant_badge_data([participant], space))
        else:
            if 'nofrill' in badge_type:
                print 'nofrill'
                badge_template = 'https://images.hasgeek.com/embed/file/949fe38ebdf94d00ac1188017a7795d0'
            if 'premium' in badge_type:
                badge_template = 'https://images.hasgeek.com/embed/file/2193f450441740d188e933a5a6a9497c'
    return render_template('badge.html', badge_template=badge_template, badges=participant_badge_data([participant], space))

@app.route('/<space>/event/<name>/checkin', methods=['POST'], subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    (Event, {'name': 'name', 'proposal_space': 'space'}, 'event'),
    permission='checkin-event')
def event_checkin(profile, space, event):
    form = forms.Form()
    if form.validate_on_submit():
        checked_in = True if request.form.get('checkin') == 't' else False
        participant_ids = request.form.getlist('pid')
        for participant_id in participant_ids:
            participant = Participant.query.filter_by(id=participant_id).first()
            attendee = Attendee.get(event, participant)
            attendee.checked_in = checked_in
        db.session.commit()
        if request.is_xhr:
            return jsonify(status=True, participant_ids=participant_ids, checked_in=checked_in)
    return redirect(url_for('event', profile=space.profile.name, space=space.name, name=event.name), code=303)

@app.route('/<space>/event/<name>/checkin/api', methods=['POST'], subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    (Event, {'name': 'name', 'proposal_space': 'space'}, 'event'),
    permission='checkin-event')
def checkin_puk(profile, space, event):
    checked_in = True if request.form.get('checkin') == 't' else False
    participant_puks = request.form.getlist('puk')
    attendees = []
    for participant_puk in participant_puks:
        participant = Participant.query.filter_by(puk=participant_puk).first()
        attendee = Attendee.get(event, participant)
        if not attendee:
            return make_response(jsonify(error='not_found', error_description="Attendee not found"), 404)
        attendee.checked_in = checked_in
        attendees.append({
            'puk': participant_puk,
            'fullname': participant.fullname,
            'checked_in': attendee.checked_in
            })
    db.session.commit()
    return jsonify(result=attendees)

@app.route('/<space>/event/<name>/participants/json', subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    (Event, {'name': 'name', 'proposal_space': 'space'}, 'event'),
    permission='checkin-event')
def event_participants_json(profile, space, event):
    checkin_count = 0
    participants = []
    for participant in Participant.checkin_list(event):
        participants.append(participant_checkin_data(participant, space, event))
        if participant.checked_in:
            checkin_count += 1

    return jsonify(participants=participants, total_participants=len(participants), total_checkedin=checkin_count)

@app.route('/<space>/event/<name>/badges', subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    (Event, {'name': 'name', 'proposal_space': 'space'}, 'event'),
    permission='view-event')
def event_badges(profile, space, event):
    badge_printed = True if request.args.get('badge_printed') == 't' else False
    participants = Participant.query.join(Attendee).filter(Attendee.event_id == event.id).filter(Participant.badge_printed == badge_printed).all()
    badge_type = request.args.getlist('type')
    badge_template = 'https://images.hasgeek.com/embed/file/ba026de2a77b4fba8e7bb5afcedc215b'
    if badge_type:
        if 'blank' in badge_type:
            badge_template = 'https://images.hasgeek.com/embed/file/24e9bafb1f6445038c8f5d5026b410b9'
            return render_template('blank_badge', badge_template=badge_template, badges=participant_badge_data(participants, space))
        else:
            if 'nofrill' in badge_type:
                badge_template = 'https://images.hasgeek.com/embed/file/949fe38ebdf94d00ac1188017a7795d0'
            if 'premium' in badge_type:
                badge_template = 'https://images.hasgeek.com/embed/file/2193f450441740d188e933a5a6a9497c'
    return render_template('badge.html', badge_template=badge_template, badges=participant_badge_data(participants, space))
<commit_msg>Change badge template.<commit_after># -*- coding: utf-8 -*-
from flask import flash, redirect, render_template, request, g, url_for, jsonify, make_response
from sqlalchemy.exc import IntegrityError
from baseframe import _
from baseframe import forms
from baseframe.forms import render_form
from coaster.views import load_models
from .. import app, lastuser
from ..models import (db, Profile, ProposalSpace, Attendee, ProposalSpaceRedirect, Participant, Event, ContactExchange)
from ..forms import ParticipantForm
from funnel.util import split_name, format_twitter_handle, make_qrcode

def participant_badge_data(participants, space):
    badges = []
    for participant in participants:
        first_name, last_name = split_name(participant.fullname)
        badges.append({
            'first_name': first_name,
            'last_name': last_name,
            'twitter': format_twitter_handle(participant.twitter),
            'company': participant.company,
            'qrcode_content': make_qrcode(u"{puk}{key}".format(puk=participant.puk, key=participant.key))
        })
    return badges

def participant_data(participant, space_id, full=False):
    if full:
        return {
            '_id': participant.id,
            'puk': participant.puk,
            'fullname': participant.fullname,
            'job_title': participant.job_title,
            'company': participant.company,
            'email': participant.email,
            'twitter': participant.twitter,
            'phone': participant.phone,
            'space_id': space_id
        }
    else:
        return {
            '_id': participant.id,
            'puk': participant.puk,
            'fullname': participant.fullname,
            'job_title': participant.job_title,
            'company': participant.company,
            'space_id': space_id
        }

def participant_checkin_data(participant, space, event):
    return {
        'pid': participant.id,
        'fullname': participant.fullname,
        'company': participant.company,
        'email': participant.email,
        'badge_printed': participant.badge_printed,
        'checked_in': participant.checked_in
    }

@app.route('/<space>/participants/json', subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    permission='view')
def participants_json(profile, space):
    return jsonify(participants=[participant_data(participant, space.id) for participant in space.participants])

@app.route('/<space>/participants/new', methods=['GET', 'POST'], subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    permission='new-participant')
def new_participant(profile, space):
    form = ParticipantForm()
    form.events.query = space.events
    if form.validate_on_submit():
        participant = Participant(proposal_space=space)
        form.populate_obj(participant)
        try:
            db.session.add(participant)
            db.session.commit()
        except IntegrityError:
            db.session.rollback()
            flash(_(u"This participant already exists."), 'info')
        return redirect(space.url_for('admin'), code=303)
    return render_form(form=form, title=_(u"New Participant"), submit=_(u"Add Participant"))

@app.route('/<space>/participant/<participant_id>/edit', methods=['GET', 'POST'], subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    (Participant, {'id': 'participant_id'}, 'participant'),
    permission='edit-participant')
def participant_edit(profile, space, participant):
    form = ParticipantForm(obj=participant, model=Participant)
    form.events.query = space.events
    if form.validate_on_submit():
        form.populate_obj(participant)
        db.session.commit()
        flash(_(u"Your changes have been saved"), 'info')
        return redirect(space.url_for('admin'), code=303)
    return render_form(form=form, title=_(u"Edit Participant"), submit=_(u"Save changes"))

@app.route('/<space>/participant', subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    permission='view')
def participant(profile, space):
    participant = Participant.query.filter_by(puk=request.args.get('puk')).first()
    if not participant:
        return jsonify(message=u"Not found", code=404)
    elif participant.key == request.args.get('key'):
        try:
            contact_exchange = ContactExchange(user_id=g.user.id, participant_id=participant.id, proposal_space_id=space.id)
            db.session.add(contact_exchange)
            db.session.commit()
        except IntegrityError:
            app.logger.warning(u"Contact Exchange already present")
            db.session.rollback()
        return jsonify(participant=participant_data(participant, space.id, full=True))
    else:
        return jsonify(message=u"Unauthorized", code=401)

@app.route('/<space>/participant/<participant_id>/badge', subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    (Participant, {'id': 'participant_id'}, 'participant'),
    permission='view-participant')
def participant_badge(profile, space, participant):
    badge_type = request.args.getlist('type')
    badge_template = 'https://images.hasgeek.com/embed/file/ba026de2a77b4fba8e7bb5afcedc215b'
    if badge_type:
        if 'blank' in badge_type:
            badge_template = 'https://images.hasgeek.com/embed/file/24e9bafb1f6445038c8f5d5026b410b9'
            return render_template('blank_badge.html', badge_template=badge_template, badges=participant_badge_data([participant], space))
        else:
            if 'nofrill' in badge_type:
                print 'nofrill'
                badge_template = 'https://images.hasgeek.com/embed/file/1ef2570d7a234a65b1b5ae4745533eea'
            if 'premium' in badge_type:
                badge_template = 'https://images.hasgeek.com/embed/file/835a0f131adc48bb8578a30c2eb2dad1'
    return render_template('badge.html', badge_template=badge_template, badges=participant_badge_data([participant], space))

@app.route('/<space>/event/<name>/checkin', methods=['POST'], subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    (Event, {'name': 'name', 'proposal_space': 'space'}, 'event'),
    permission='checkin-event')
def event_checkin(profile, space, event):
    form = forms.Form()
    if form.validate_on_submit():
        checked_in = True if request.form.get('checkin') == 't' else False
        participant_ids = request.form.getlist('pid')
        for participant_id in participant_ids:
            participant = Participant.query.filter_by(id=participant_id).first()
            attendee = Attendee.get(event, participant)
            attendee.checked_in = checked_in
        db.session.commit()
        if request.is_xhr:
            return jsonify(status=True, participant_ids=participant_ids, checked_in=checked_in)
    return redirect(url_for('event', profile=space.profile.name, space=space.name, name=event.name), code=303)

@app.route('/<space>/event/<name>/checkin/api', methods=['POST'], subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    (Event, {'name': 'name', 'proposal_space': 'space'}, 'event'),
    permission='checkin-event')
def checkin_puk(profile, space, event):
    checked_in = True if request.form.get('checkin') == 't' else False
    participant_puks = request.form.getlist('puk')
    attendees = []
    for participant_puk in participant_puks:
        participant = Participant.query.filter_by(puk=participant_puk).first()
        attendee = Attendee.get(event, participant)
        if not attendee:
            return make_response(jsonify(error='not_found', error_description="Attendee not found"), 404)
        attendee.checked_in = checked_in
        attendees.append({
            'puk': participant_puk,
            'fullname': participant.fullname,
            'checked_in': attendee.checked_in
            })
    db.session.commit()
    return jsonify(result=attendees)

@app.route('/<space>/event/<name>/participants/json', subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    (Event, {'name': 'name', 'proposal_space': 'space'}, 'event'),
    permission='checkin-event')
def event_participants_json(profile, space, event):
    checkin_count = 0
    participants = []
    for participant in Participant.checkin_list(event):
        participants.append(participant_checkin_data(participant, space, event))
        if participant.checked_in:
            checkin_count += 1

    return jsonify(participants=participants, total_participants=len(participants), total_checkedin=checkin_count)

@app.route('/<space>/event/<name>/badges', subdomain='<profile>')
@lastuser.requires_login
@load_models(
    (Profile, {'name': 'profile'}, 'g.profile'),
    ((ProposalSpace, ProposalSpaceRedirect), {'name': 'space', 'profile': 'profile'}, 'space'),
    (Event, {'name': 'name', 'proposal_space': 'space'}, 'event'),
    permission='view-event')
def event_badges(profile, space, event):
    badge_printed = True if request.args.get('badge_printed') == 't' else False
    participants = Participant.query.join(Attendee).filter(Attendee.event_id == event.id).filter(Participant.badge_printed == badge_printed).all()
    badge_type = request.args.getlist('type')
    badge_template = 'https://images.hasgeek.com/embed/file/ba026de2a77b4fba8e7bb5afcedc215b'
    if badge_type:
        if 'blank' in badge_type:
            badge_template = 'https://images.hasgeek.com/embed/file/24e9bafb1f6445038c8f5d5026b410b9'
            return render_template('blank_badge', badge_template=badge_template, badges=participant_badge_data(participants, space))
        else:
            if 'nofrill' in badge_type:
                badge_template = 'https://images.hasgeek.com/embed/file/1ef2570d7a234a65b1b5ae4745533eea'
            if 'premium' in badge_type:
                badge_template = 'https://images.hasgeek.com/embed/file/835a0f131adc48bb8578a30c2eb2dad1'
    return render_template('badge.html', badge_template=badge_template, badges=participant_badge_data(participants, space))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
"""
Script to run "buckets" classification experiments.
"""

import argparse
import os
import pprint
import time

from htmresearch.frameworks.nlp.bucket_runner import BucketRunner
from htmresearch.frameworks.nlp.bucket_htm_runner import BucketHTMRunner

def checkInputs(args):
  """Function that displays a set of arguments and asks to proceed."""
  pprint.pprint(vars(args))
  userIn = raw_input("Proceed? (y/n): ")

  if userIn == 'y':
    return True

  if userIn == 'n':
    return False

  print "Incorrect input given\n"
  return checkInputs(args)

def run(args):
  start = time.time()

  root = os.path.dirname(os.path.realpath(__file__))
  resultsDir = os.path.join(root, args.resultsDir)

  metricsDicts = []
  # TODO: would it be okay to just loop over the inference steps for multiple
  # trials? Is this even possible?
  for trial in xrange(args.trials):
    if args.modelName == "HTMNetwork":
      runner = BucketHTMRunner(dataPath=args.dataPath,
                               resultsDir=resultsDir,
                               experimentName=args.experimentName,
                               experimentType="buckets",
                               modelName=args.modelName,
                               retinaScaling=args.retinaScaling,
                               retina=args.retina,
                               apiKey=args.apiKey,
                               loadPath=args.loadPath,
                               plots=args.plots,
                               orderedSplit=args.orderedSplit,
                               verbosity=args.verbosity,
                               generateData=args.generateData,
                               classificationFile=args.classificationFile,
                               networkConfigPath=args.networkConfigPath,
                               trainingReps=args.trainingReps,
                               seed=args.seed,
                               concatenationMethod=args.combineMethod,
                               numClasses=0)
      runner.initModel(0)
    else:
      runner = BucketRunner(dataPath=args.dataPath,
                            resultsDir=resultsDir,
                            experimentName=args.experimentName,
                            experimentType="buckets",
                            modelName=args.modelName,
                            retinaScaling=args.retinaScaling,
                            retina=args.retina,
                            apiKey=args.apiKey,
                            loadPath=args.loadPath,
                            plots=args.plots,
                            orderedSplit=args.orderedSplit,
                            verbosity=args.verbosity,
                            concatenationMethod=args.combineMethod,
                            numClasses=1)
      runner.initModel(args.modelName)

    print "Reading in data, preprocessing, and encoding."
    dataTime = time.time()
    runner.setupData(args.textPreprocess)

    runner.encodeSamples()
    print ("Encoding complete; elapsed time is {0:.2f} seconds.\nNow running the "
           "experiment.".format(time.time() - dataTime))

    runner.bucketData()

    runner.partitionIndices(args.seed, args.numInference)

    runner.train()

    metricsDicts.append(runner.runExperiment(args.numInference))

    print "Saving..."
    runner.saveModel()

    args.seed += 1

  resultCalcs = runner.evaluateResults(metricsDicts)

  print "Experiment complete in {0:.2f} seconds.".format(time.time() - start)

if __name__ == "__main__":

  parser = argparse.ArgumentParser()

  parser.add_argument("dataPath",
                      help="Path to data CSV or folder of CSVs.",
                      type=str)
  parser.add_argument("-n", "--experimentName",
                      default="kfolds",
                      type=str,
                      help="Experiment name.")
  parser.add_argument("-m", "--modelName",
                      default="Keywords",
                      type=str,
                      help="Name of model class. Also used for model results "
                           "directory and pickle checkpoint.")
  parser.add_argument("--retinaScaling",
                      default=1.0,
                      type=float,
                      help="Factor by which to scale the Cortical.io retina.")
  parser.add_argument("--retina",
                      default="en_associative",
                      type=str,
                      help="Name of Cortical.io retina.")
  parser.add_argument("--apiKey",
                      default=None,
                      type=str,
                      help="Key for Cortical.io API.")
  parser.add_argument("--resultsDir",
                      default="results",
                      help="This will hold the experiment results.")
  parser.add_argument("--textPreprocess",
                      action="store_true",
                      default=False,
                      help="Whether or not to use text preprocessing.")
  parser.add_argument("--numInference",
                      help="Number of samples to use for inference per bucket. "
                           "A bucket with too few samples will be skipped.",
                      type=int,
                      default=10)
  parser.add_argument("--trials",
                      help="Number of experiment trials to run, where the "
                           "random selection of inference data samples will "
                           "change each trial.",
                      type=int,
                      default=1)
  parser.add_argument("--loadPath",
                      help="Path from which to load the serialized model.",
                      type=str,
                      default=None)
  parser.add_argument("--plots",
                      default=1,
                      type=int,
                      help="0 for no evaluation plots, 1 for classification "
                           "accuracy plots, 2 includes the confusion matrix.")
  parser.add_argument("--orderedSplit",
                      default=False,
                      action="store_true",
                      help="To split the train and test sets, False will split "
                           "the samples randomly, True will allocate the "
                           "first n samples to training with the remainder "
                           "for testing.")
  parser.add_argument("--seed",
                      default=42,
                      type=int,
                      help="Random seed, used in partitioning the data.")
  parser.add_argument("-v", "--verbosity",
                      default=1,
                      type=int,
                      help="verbosity 0 will print out experiment steps, "
                           "verbosity 1 will include results, and verbosity > "
                           "1 will print out preprocessed tokens and kNN "
                           "inference metrics.")
  parser.add_argument("--skipConfirmation",
                      help="If specified will skip the user confirmation step.",
                      default=False,
                      action="store_true")
  parser.add_argument("--networkConfigPath",
                      default="htm_network_config.json",
                      help="Path to JSON specifying the network params.",
                      type=str)
  parser.add_argument("--generateData",
                      default=False,
                      action="store_true",
                      help="Whether or not to generate network data files.")
  parser.add_argument("--votingMethod",
                      default="last",
                      choices=["last", "most"],
                      help="Method to use when picking final classifications.")
  parser.add_argument("--classificationFile",
                      default="",
                      help="JSON file mapping string labels to ids.")
  parser.add_argument("--trainingReps",
                      help="How many times to repeat training an HTM network.",
                      type=int,
                      default=1)
  parser.add_argument("--combineMethod",
                      help="Method for combining KNN distances over multiple "
                           "inference steps. Acceptable values are 'min' and "
                           "'mean'.",
                      type=str,
                      default="min")

  args = parser.parse_args()

  if args.skipConfirmation or checkInputs(args):
    run(args)
<commit_msg>cleanup experiment args<commit_after>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
"""
Script to run "buckets" classification experiments.
"""

import argparse
import os
import pprint
import time

from htmresearch.frameworks.nlp.bucket_runner import BucketRunner
from htmresearch.frameworks.nlp.bucket_htm_runner import BucketHTMRunner

def checkInputs(args):
  """Function that displays a set of arguments and asks to proceed."""
  pprint.pprint(vars(args))
  userIn = raw_input("Proceed? (y/n): ")

  if userIn == 'y':
    return True

  if userIn == 'n':
    return False

  print "Incorrect input given\n"
  return checkInputs(args)

def run(args):
  start = time.time()

  root = os.path.dirname(os.path.realpath(__file__))
  resultsDir = os.path.join(root, args.resultsDir)

  metricsDicts = []
  # TODO: would it be okay to just loop over the inference steps for multiple
  # trials? Is this even possible?
  for trial in xrange(args.trials):
    if args.modelName == "HTMNetwork":
      runner = BucketHTMRunner(dataPath=args.dataPath,
                               resultsDir=resultsDir,
                               experimentName=args.experimentName,
                               experimentType="buckets",
                               modelName=args.modelName,
                               retinaScaling=args.retinaScaling,
                               retina=args.retina,
                               apiKey=args.apiKey,
                               loadPath=args.loadPath,
                               plots=args.plots,
                               orderedSplit=args.orderedSplit,
                               verbosity=args.verbosity,
                               generateData=args.generateData,
                               classificationFile=args.classificationFile,
                               networkConfigPath=args.networkConfigPath,
                               trainingReps=args.trainingReps,
                               seed=args.seed,
                               concatenationMethod=args.combineMethod,
                               numClasses=0)
      runner.initModel(0)
    else:
      runner = BucketRunner(dataPath=args.dataPath,
                            resultsDir=resultsDir,
                            experimentName=args.experimentName,
                            experimentType="buckets",
                            modelName=args.modelName,
                            retinaScaling=args.retinaScaling,
                            retina=args.retina,
                            apiKey=args.apiKey,
                            loadPath=args.loadPath,
                            plots=args.plots,
                            orderedSplit=args.orderedSplit,
                            verbosity=args.verbosity,
                            concatenationMethod=args.combineMethod,
                            numClasses=1)
      runner.initModel(args.modelName)

    print "Reading in data, preprocessing, and encoding."
    dataTime = time.time()
    runner.setupData(args.textPreprocess)

    runner.encodeSamples()
    print ("Encoding complete; elapsed time is {0:.2f} seconds.\nNow running the "
           "experiment.".format(time.time() - dataTime))

    runner.bucketData()

    runner.partitionIndices(args.seed, args.numInference)

    runner.train()

    metricsDicts.append(runner.runExperiment(args.numInference))

    print "Saving..."
    runner.saveModel()

    args.seed += 1

  resultCalcs = runner.evaluateResults(metricsDicts)

  print "Experiment complete in {0:.2f} seconds.".format(time.time() - start)

if __name__ == "__main__":

  parser = argparse.ArgumentParser()

  parser.add_argument("dataPath",
                      help="Path to data CSV or folder of CSVs.",
                      type=str)
  parser.add_argument("-n", "--experimentName",
                      default="buckets",
                      type=str,
                      help="Experiment name.")
  parser.add_argument("-m", "--modelName",
                      default="CioWordFingerprint",
                      type=str,
                      help="Name of model class. Also used for model results "
                           "directory and pickle checkpoint.")
  parser.add_argument("--retinaScaling",
                      default=1.0,
                      type=float,
                      help="Factor by which to scale the Cortical.io retina.")
  parser.add_argument("--retina",
                      default="en_associative",
                      type=str,
                      help="Name of Cortical.io retina.")
  parser.add_argument("--apiKey",
                      default=None,
                      type=str,
                      help="Key for Cortical.io API.")
  parser.add_argument("--resultsDir",
                      default="results",
                      help="This will hold the experiment results.")
  parser.add_argument("--textPreprocess",
                      action="store_true",
                      default=False,
                      help="Whether or not to use text preprocessing.")
  parser.add_argument("--numInference",
                      help="Number of samples to use for inference per bucket. "
                           "A bucket with too few samples will be skipped.",
                      type=int,
                      default=10)
  parser.add_argument("--trials",
                      help="Number of experiment trials to run, where the "
                           "random selection of inference data samples will "
                           "change each trial.",
                      type=int,
                      default=1)
  parser.add_argument("--loadPath",
                      help="Path from which to load the serialized model.",
                      type=str,
                      default=None)
  parser.add_argument("--plots",
                      default=1,
                      type=int,
                      help="0 for no evaluation plots, 1 for classification "
                           "accuracy plots, 2 includes the confusion matrix.")
  parser.add_argument("--orderedSplit",
                      default=False,
                      action="store_true",
                      help="To split the train and test sets, False will split "
                           "the samples randomly, True will allocate the "
                           "first n samples to training with the remainder "
                           "for testing.")
  parser.add_argument("--seed",
                      default=42,
                      type=int,
                      help="Random seed, used in partitioning the data.")
  parser.add_argument("-v", "--verbosity",
                      default=1,
                      type=int,
                      help="verbosity 0 will print out experiment steps, "
                           "verbosity 1 will include results, and verbosity > "
                           "1 will print out preprocessed tokens and kNN "
                           "inference metrics.")
  parser.add_argument("--skipConfirmation",
                      help="If specified will skip the user confirmation step.",
                      default=False,
                      action="store_true")
  parser.add_argument("--networkConfigPath",
                      default="htm_network_config.json",
                      help="Path to JSON specifying the network params.",
                      type=str)
  parser.add_argument("--generateData",
                      default=False,
                      action="store_true",
                      help="Whether or not to generate network data files.")
  parser.add_argument("--classificationFile",
                      default="",
                      help="JSON file mapping string labels to ids.")
  parser.add_argument("--trainingReps",
                      help="How many times to repeat training an HTM network.",
                      type=int,
                      default=1)
  parser.add_argument("--combineMethod",
                      help="Method for combining KNN distances over multiple "
                           "inference steps. Acceptable values are 'min' and "
                           "'mean'.",
                      type=str,
                      default="min")

  args = parser.parse_args()

  if args.skipConfirmation or checkInputs(args):
    run(args)
<|endoftext|>


<commit_before>import tensorflow as tf

from t3f import decompositions
from t3f import riemannian

def _enforce_gauge_conditions(deltas, left):
  """Project deltas that define tangent space vec onto the gauge conditions."""
  proj_deltas = []
  for i in range(left.ndims()):
    if left.is_tt_matrix():
      r1, n, m, r2 = left.tt_cores[i].shape.as_list()
    else:
      r1, n, r2 = left.tt_cores[i].shape.as_list()
    q = tf.reshape(left.tt_cores[i], (-1, r2))
    if i < left.ndims() - 1:
      proj_delta = deltas[i]
      proj_delta = tf.reshape(proj_delta, (-1, r2))
      proj_delta -= tf.matmul(q, tf.matmul(tf.transpose(q), proj_delta))
      proj_delta = tf.reshape(proj_delta, left.tt_cores[i].shape)
    else:
      proj_delta = deltas[i]
    proj_deltas.append(proj_delta)
  return proj_deltas

def _is_invariant_to_input_transforms(f_value_1, f_value_2,
                                      name="check_autodiff_arguments"):
  """Returns an assert op that checks that the f_value_1 == f_value_2.

  Args:
    f_value_1: tf.Tensor, value of the function computed on x_1
    f_value_2: tf.Tensor, value of the function computed on x_2
    name: String, the name of the returned op

  Here we assume that as tensors x_1 == x_2, but their TT-cores are different,
  e.g. x_2 is a cores orthogonalization version of x_1.

  The function prints a warning about introducing overhead and returns an Assert
  op that checks that the two values are reasonably close to each other.

  Returns:
    tf.op, assertion operation.
  """
  print('Warning: debug mode of Riemannian autodiff is turned on which '
        'makes things a bit slower. It is advisable to keep debug=True untill '
        'actuall production usage, since debug mode does help to catch bugs.')
  rel_diff = tf.abs((f_value_1 - f_value_2) / f_value_1)
  err_msg = "The function passed to Riemannian autodiff returns different " \
            "values for two different versions of the same tensor. " \
            "The function values are"
  assert_op = tf.Assert(rel_diff < 1e-5, [err_msg, f_value_1, f_value_2],
                        name=name)
  return assert_op

def gradients(func, x, name='t3f_gradients', debug=True):
  """Riemannian autodiff: returns gradient projected on tangent space of TT.

  Computes projection of the gradient df/dx onto the tangent space of TT tensor
  at point x.

  Warning: this is experimental feature and it may not work for some function,
  e.g. ones that include QR or SVD decomposition (t3f.project, t3f.round) or
  for functions that work with TT-cores directly (in contrast to working with
  TT-object only via t3f functions). In this cases this function can silently
  return wrong results!

  Example:
      # Scalar product with some predefined tensor squared 0.5 * <x, t>**2.
      # It's gradient is <x, t> t and it's Riemannian gradient is
      #     t3f.project(<x, t> * t, x)
      f = lambda x: 0.5 * t3f.flat_inner(x, t)**2
      projected_grad = t3f.gradients(f, x) # t3f.project(t3f.flat_inner(x, t) * t, x)

  Args:
      func: function that takes TensorTrain object as input and outputs a number.
      x: point at which to compute the gradient and on which tangent space to
        project the gradient.
      name: string, name of the Op.
      debug: [True] whether to do a sanity check that the passed function is
        invariant to different TT representations (otherwise the Rieamnnian
        gradient doesn't even exist). It makes things slower, but helps catching
        bugs, so turn it off during production deployment.

  Returns:
      `TensorTrain`, projection of the gradient df/dx onto the tangent space at
      point x.

  See also:
      t3f.hessian_vector_product
  """
  with tf.name_scope(name, values=x.tt_cores):
    left = decompositions.orthogonalize_tt_cores(x)
    right = decompositions.orthogonalize_tt_cores(left, left_to_right=False)
    deltas = [right.tt_cores[0]]
    deltas += [tf.zeros_like(cc) for cc in right.tt_cores[1:]]
    x_projection = riemannian.deltas_to_tangent_space(deltas, x, left, right)
    function_value = func(x_projection)
    if debug:
      assert_op = _is_invariant_to_input_transforms(function_value, func(right))
    else:
      assert_op = None
    with tf.control_dependencies([assert_op]):
      cores_grad = tf.gradients(function_value, deltas)
    deltas = _enforce_gauge_conditions(cores_grad, left)
    return riemannian.deltas_to_tangent_space(deltas, x, left, right)

def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product',
                           debug=True):
  """P_x d^2f/dx^2 P_x vector, i.e. Riemannian hessian by vector product.

    Computes
      P_x d^2f/dx^2 P_x vector
    where P_x is projection onto the tangent space of TT at point x and
    d^2f/dx^2 is the Hessian of the function.

    Note that the true Riemannian hessian also includes the manifold curvature
    term which is ignored here.

    Warning: this is experimental feature and it may not work for some function,
    e.g. ones that include QR or SVD decomposition (t3f.project, t3f.round) or
    for functions that work with TT-cores directly (in contrast to working with
    TT-object only via t3f functions). In this cases this function can silently
    return wrong results!

    Example:
        # Quadratic form with matrix A: <x, A x>.
        # It's gradient is (A + A.T) x, it's Hessian is (A + A.T)
        # It's Riemannian Hessian by vector product is
        #     t3f.project(t3f.matmul(A + t3f.transpose(A), vector), x)
        f = lambda x: t3f.quadratic_form(A, x, x)
        res = t3f.hessian_vector_product(f, x, vector)

    Args:
        func: function that takes TensorTrain object as input and outputs a number.
        x: point at which to compute the Hessian and on which tangent space to
          project the gradient.
      vector: `TensorTrain` object which to multiply be the Hessian.
      name: string, name of the Op.
      debug: [True] whether to do a sanity check that the passed function is
        invariant to different TT representations (otherwise the Rieamnnian
        gradient doesn't even exist). It makes things slower, but helps catching
        bugs, so turn it off during production deployment.

    Returns:
        `TensorTrain`, projection of the gradient df/dx onto the tangent space at
        point x.

    See also:
        t3f.gradients
    """
  all_cores = list(x.tt_cores) + list(vector.tt_cores)
  with tf.name_scope(name, values=all_cores):
    left = decompositions.orthogonalize_tt_cores(x)
    right = decompositions.orthogonalize_tt_cores(left, left_to_right=False)
    deltas = [right.tt_cores[0]]
    deltas += [tf.zeros_like(cc) for cc in right.tt_cores[1:]]
    x_projection = riemannian.deltas_to_tangent_space(deltas, x, left, right)
    function_value = func(x_projection)
    if debug:
      assert_op = _is_invariant_to_input_transforms(function_value, func(right))
    else:
      assert_op = None
    with tf.control_dependencies([assert_op]):
      vector_projected = riemannian.project(vector, x)
    cores_grad = tf.gradients(function_value, deltas)
    vec_deltas = riemannian.tangent_space_to_deltas(vector_projected)
    products = [tf.reduce_sum(a * b) for a, b in zip(cores_grad, vec_deltas)]
    grad_times_vec = tf.add_n(products)
    second_cores_grad = tf.gradients(grad_times_vec, deltas)
    final_deltas = _enforce_gauge_conditions(second_cores_grad, left)
    return riemannian.deltas_to_tangent_space(final_deltas, x, left, right)
<commit_msg>Simplify rank logic a bit<commit_after>import tensorflow as tf

from t3f import shapes
from t3f import decompositions
from t3f import riemannian

def _enforce_gauge_conditions(deltas, left):
  """Project deltas that define tangent space vec onto the gauge conditions."""
  proj_deltas = []
  tt_ranks = shapes.lazy_tt_ranks(left)
  for i in range(left.ndims()):
    right_r = tt_ranks[i + 1]
    q = tf.reshape(left.tt_cores[i], (-1, right_r))
    if i < left.ndims() - 1:
      proj_delta = deltas[i]
      proj_delta = tf.reshape(proj_delta, (-1, right_r))
      proj_delta -= tf.matmul(q, tf.matmul(tf.transpose(q), proj_delta))
      proj_delta = tf.reshape(proj_delta, left.tt_cores[i].shape)
    else:
      proj_delta = deltas[i]
    proj_deltas.append(proj_delta)
  return proj_deltas

def _is_invariant_to_input_transforms(f_value_1, f_value_2,
                                      name="check_autodiff_arguments"):
  """Returns an assert op that checks that the f_value_1 == f_value_2.

  Args:
    f_value_1: tf.Tensor, value of the function computed on x_1
    f_value_2: tf.Tensor, value of the function computed on x_2
    name: String, the name of the returned op

  Here we assume that as tensors x_1 == x_2, but their TT-cores are different,
  e.g. x_2 is a cores orthogonalization version of x_1.

  The function prints a warning about introducing overhead and returns an Assert
  op that checks that the two values are reasonably close to each other.

  Returns:
    tf.op, assertion operation.
  """
  print('Warning: debug mode of Riemannian autodiff is turned on which '
        'makes things a bit slower. It is advisable to keep debug=True untill '
        'actuall production usage, since debug mode does help to catch bugs.')
  rel_diff = tf.abs((f_value_1 - f_value_2) / f_value_1)
  err_msg = "The function passed to Riemannian autodiff returns different " \
            "values for two different versions of the same tensor. " \
            "The function values are"
  assert_op = tf.Assert(rel_diff < 1e-5, [err_msg, f_value_1, f_value_2],
                        name=name)
  return assert_op

def gradients(func, x, name='t3f_gradients', debug=True):
  """Riemannian autodiff: returns gradient projected on tangent space of TT.

  Computes projection of the gradient df/dx onto the tangent space of TT tensor
  at point x.

  Warning: this is experimental feature and it may not work for some function,
  e.g. ones that include QR or SVD decomposition (t3f.project, t3f.round) or
  for functions that work with TT-cores directly (in contrast to working with
  TT-object only via t3f functions). In this cases this function can silently
  return wrong results!

  Example:
      # Scalar product with some predefined tensor squared 0.5 * <x, t>**2.
      # It's gradient is <x, t> t and it's Riemannian gradient is
      #     t3f.project(<x, t> * t, x)
      f = lambda x: 0.5 * t3f.flat_inner(x, t)**2
      projected_grad = t3f.gradients(f, x) # t3f.project(t3f.flat_inner(x, t) * t, x)

  Args:
      func: function that takes TensorTrain object as input and outputs a number.
      x: point at which to compute the gradient and on which tangent space to
        project the gradient.
      name: string, name of the Op.
      debug: [True] whether to do a sanity check that the passed function is
        invariant to different TT representations (otherwise the Rieamnnian
        gradient doesn't even exist). It makes things slower, but helps catching
        bugs, so turn it off during production deployment.

  Returns:
      `TensorTrain`, projection of the gradient df/dx onto the tangent space at
      point x.

  See also:
      t3f.hessian_vector_product
  """
  with tf.name_scope(name, values=x.tt_cores):
    left = decompositions.orthogonalize_tt_cores(x)
    right = decompositions.orthogonalize_tt_cores(left, left_to_right=False)
    deltas = [right.tt_cores[0]]
    deltas += [tf.zeros_like(cc) for cc in right.tt_cores[1:]]
    x_projection = riemannian.deltas_to_tangent_space(deltas, x, left, right)
    function_value = func(x_projection)
    if debug:
      assert_op = _is_invariant_to_input_transforms(function_value, func(right))
    else:
      assert_op = None
    with tf.control_dependencies([assert_op]):
      cores_grad = tf.gradients(function_value, deltas)
    deltas = _enforce_gauge_conditions(cores_grad, left)
    return riemannian.deltas_to_tangent_space(deltas, x, left, right)

def hessian_vector_product(func, x, vector, name='t3f_hessian_vector_product',
                           debug=True):
  """P_x d^2f/dx^2 P_x vector, i.e. Riemannian hessian by vector product.

    Computes
      P_x d^2f/dx^2 P_x vector
    where P_x is projection onto the tangent space of TT at point x and
    d^2f/dx^2 is the Hessian of the function.

    Note that the true Riemannian hessian also includes the manifold curvature
    term which is ignored here.

    Warning: this is experimental feature and it may not work for some function,
    e.g. ones that include QR or SVD decomposition (t3f.project, t3f.round) or
    for functions that work with TT-cores directly (in contrast to working with
    TT-object only via t3f functions). In this cases this function can silently
    return wrong results!

    Example:
        # Quadratic form with matrix A: <x, A x>.
        # It's gradient is (A + A.T) x, it's Hessian is (A + A.T)
        # It's Riemannian Hessian by vector product is
        #     t3f.project(t3f.matmul(A + t3f.transpose(A), vector), x)
        f = lambda x: t3f.quadratic_form(A, x, x)
        res = t3f.hessian_vector_product(f, x, vector)

    Args:
        func: function that takes TensorTrain object as input and outputs a number.
        x: point at which to compute the Hessian and on which tangent space to
          project the gradient.
      vector: `TensorTrain` object which to multiply be the Hessian.
      name: string, name of the Op.
      debug: [True] whether to do a sanity check that the passed function is
        invariant to different TT representations (otherwise the Rieamnnian
        gradient doesn't even exist). It makes things slower, but helps catching
        bugs, so turn it off during production deployment.

    Returns:
        `TensorTrain`, projection of the gradient df/dx onto the tangent space at
        point x.

    See also:
        t3f.gradients
    """
  all_cores = list(x.tt_cores) + list(vector.tt_cores)
  with tf.name_scope(name, values=all_cores):
    left = decompositions.orthogonalize_tt_cores(x)
    right = decompositions.orthogonalize_tt_cores(left, left_to_right=False)
    deltas = [right.tt_cores[0]]
    deltas += [tf.zeros_like(cc) for cc in right.tt_cores[1:]]
    x_projection = riemannian.deltas_to_tangent_space(deltas, x, left, right)
    function_value = func(x_projection)
    if debug:
      assert_op = _is_invariant_to_input_transforms(function_value, func(right))
    else:
      assert_op = None
    with tf.control_dependencies([assert_op]):
      vector_projected = riemannian.project(vector, x)
    cores_grad = tf.gradients(function_value, deltas)
    vec_deltas = riemannian.tangent_space_to_deltas(vector_projected)
    products = [tf.reduce_sum(a * b) for a, b in zip(cores_grad, vec_deltas)]
    grad_times_vec = tf.add_n(products)
    second_cores_grad = tf.gradients(grad_times_vec, deltas)
    final_deltas = _enforce_gauge_conditions(second_cores_grad, left)
    return riemannian.deltas_to_tangent_space(final_deltas, x, left, right)
<|endoftext|>


<commit_before># gaeupload/handler_openid.py
#
#

""" Openid handler. """

## gozerlib imports

from gozerlib.gae.utils.web import loginurl
from gozerlib.utils.generic import getversion
from gozerlib.utils.exception import handle_exception

## google imports

from google.appengine.api import users
import webapp2 as webapp

## basic imports

import os
import logging
import urlparse
import urllib
import warnings
import socket

## classes

class OpenIdLoginHandler(webapp.RequestHandler):

    def create_openid_url(self, continue_url):
        continue_url = urlparse.urljoin(self.request.url, continue_url)
        return "/_ah/login?continue=%s" % urllib.quote(continue_url)

    def get(self):
        try:
            cont = self.request.get('continue')
            logging.info('openid - login form %s' % cont)
            urlstring = self.create_openid_url(cont)
            template_values = {
                'continue': cont,
                'urlstring': urlstring,
                'appname': getversion()
            }
            try: host = socket.gethostname()
            except AttributeError:
                if os.environ.get('HTTP_HOST'): host = os.environ['HTTP_HOST']
                else: host = os.environ['SERVER_NAME']
            inputdict = {'version': getversion(), 'host': host}
            template_values.update(inputdict)
            temp = os.path.join(os.getcwd(), 'templates/login.html')
            from google.appengine.ext.webapp import template
            outstr = template.render(temp, template_values)
            self.response.out.write(outstr)
        except Exception, ex:
            handle_exception()

    def post(self):
        try:
            cont = self.request.get('continue')
            conturl = self.create_openid_url(cont)
            logging.info('openid - %s' % cont)
            openid = self.request.get('openid_identifier')
            if openid:
                login_url = users.create_login_url(cont, None, openid)
                logging.info('openid - redirecting to url %s (%s)' % (login_url, openid))
                self.redirect(login_url)
            else:
                logging.warn("denied access for %s - %s - %s" % (self.request.remote_addr, cont, openid))
                self.send_error(400)
        except Exception, ex:
            handle_exception()

## the application

application = webapp.WSGIApplication([
                               ('/_ah/login_required', OpenIdLoginHandler)],
                               debug=True)

## main

def main():
    global application
    application.run()

if __name__ == "__main__":
    main()
<commit_msg>dont forget openid<commit_after># gaeupload/handler_openid.py
#
#

""" Openid handler. """

## gozerlib imports

from gozerlib.gae.utils.web import loginurl
from gozerlib.version import getversion
from gozerlib.utils.exception import handle_exception

## google imports

from google.appengine.api import users
import webapp2 as webapp

## basic imports

import os
import logging
import urlparse
import urllib
import warnings
import socket

## classes

class OpenIdLoginHandler(webapp.RequestHandler):

    def create_openid_url(self, continue_url):
        continue_url = urlparse.urljoin(self.request.url, continue_url)
        return "/_ah/login?continue=%s" % urllib.quote(continue_url)

    def get(self):
        try:
            cont = self.request.get('continue')
            logging.info('openid - login form %s' % cont)
            urlstring = self.create_openid_url(cont)
            template_values = {
                'continue': cont,
                'urlstring': urlstring,
                'appname': getversion()
            }
            try: host = socket.gethostname()
            except AttributeError:
                if os.environ.get('HTTP_HOST'): host = os.environ['HTTP_HOST']
                else: host = os.environ['SERVER_NAME']
            inputdict = {'version': getversion(), 'host': host}
            template_values.update(inputdict)
            temp = os.path.join(os.getcwd(), 'templates/login.html')
            from google.appengine.ext.webapp import template
            outstr = template.render(temp, template_values)
            self.response.out.write(outstr)
        except Exception, ex:
            handle_exception()

    def post(self):
        try:
            cont = self.request.get('continue')
            conturl = self.create_openid_url(cont)
            logging.info('openid - %s' % cont)
            openid = self.request.get('openid_identifier')
            if openid:
                login_url = users.create_login_url(cont, None, openid)
                logging.info('openid - redirecting to url %s (%s)' % (login_url, openid))
                self.redirect(login_url)
            else:
                logging.warn("denied access for %s - %s - %s" % (self.request.remote_addr, cont, openid))
                self.send_error(400)
        except Exception, ex:
            handle_exception()

## the application

application = webapp.WSGIApplication([
                               ('/_ah/login_required', OpenIdLoginHandler)],
                               debug=True)

## main

def main():
    global application
    application.run()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import sys
import os
import shutil
import subprocess

if len( sys.argv ) == 2:
    PG_DATA_DIR_DEFAULT = sys.argv[1]
else:
    PG_DATA_DIR_DEFAULT = "/var/lib/postgresql/9.3/main"
PG_DATA_DIR_HOST = os.environ.get("PG_DATA_DIR_HOST", "/export/postgresql/9.3/main/")
PG_CONF = '/etc/postgresql/9.3/main/postgresql.conf'

def change_path( src ):
    """
        src will be copied to /export/`src` and a symlink will be placed in src pointing to /export/
    """
    if os.path.exists( src ):
        dest = os.path.join( '/export/', src.strip('/') )
        # if destination is empty move all files into /export/ and symlink back to source
        if not os.path.exists( dest ):
            dest_dir = os.path.dirname(dest)
            if not os.path.exists( dest_dir ):
                os.makedirs(dest_dir)
            shutil.move( src, dest )
            os.symlink( dest, src.rstrip('/') )
            os.chown( src, int(os.environ['GALAXY_UID']), int(os.environ['GALAXY_GID']) )
            subprocess.call( 'chown -R %s:%s %s' % ( os.environ['GALAXY_UID'], os.environ['GALAXY_GID'], dest ), shell=True )
        # if destination exists (e.g. continuing a previous session), remove source and symlink
        else:
            if os.path.isdir( src ):
                shutil.rmtree( src )
            else:
                os.unlink( src )
            os.symlink( dest, src.rstrip('/') )

if __name__ == "__main__":
    """
        If the '/export/' folder exist, meaning docker was started with '-v /home/foo/bar:/export',
        we will link every file that needs to persist to the host system. Addionaly a file (/.galaxy_save) is
        created that indicates all linking is already done.
        If the user re-starts (with docker start) the container the file /.galaxy_save is found and the linking
        is aborted.
    """
    if os.path.exists( '/export/.distribution_config/' ):
        shutil.rmtree( '/export/.distribution_config/' )
    shutil.copytree( '/galaxy-central/config/', '/export/.distribution_config/' )

    # Copy all files starting with "welcome"
    # This enables a flexible start page design.
    for filename in os.listdir('/export/'):
        if filename.startswith('welcome'):
            export_file = os.path.join( '/export/', filename)
            image_file = os.path.join('/etc/galaxy/', filename)
            shutil.copy(export_file, image_file)

    if not os.path.exists( '/export/galaxy-central/' ):
        os.makedirs("/export/galaxy-central/")
        os.chown( "/export/galaxy-central/", int(os.environ['GALAXY_UID']), int(os.environ['GALAXY_GID']) )
    change_path('/galaxy-central/config/')
    change_path('/galaxy-central/integrated_tool_panel.xml')
    change_path('/galaxy-central/display_applications/')
    change_path('/galaxy-central/tool_deps/')
    change_path('/galaxy-central/tool-data/')
    change_path('/shed_tools/')

    if os.path.exists('/export/reports_htpasswd'):
        shutil.copy('/export/reports_htpasswd', '/etc/nginx/htpasswd')

    try:
        change_path('/var/lib/docker/')
    except:
        # In case of unprivileged access this will result in a "Device or resource busy." error.
        pass

    if not os.path.exists( PG_DATA_DIR_HOST ) or 'PG_VERSION' not in os.listdir( PG_DATA_DIR_HOST ):
        dest_dir = os.path.dirname( PG_DATA_DIR_HOST )
        if not os.path.exists( dest_dir ):
            os.makedirs(dest_dir)
        # User given dbpath, usually a directory from the host machine
        # copy the postgresql data folder to the new location
        subprocess.call('cp -R %s/* %s' % (PG_DATA_DIR_DEFAULT, PG_DATA_DIR_HOST), shell=True)
        # copytree needs an non-existing dst dir, how annoying :(
        # shutil.copytree(PG_DATA_DIR_DEFAULT, PG_DATA_DIR_HOST)
        subprocess.call('chown -R postgres:postgres /export/postgresql/', shell=True)
        subprocess.call('chmod -R 0755 /export/', shell=True)
        subprocess.call('chmod -R 0700 %s' % PG_DATA_DIR_HOST, shell=True)

    # change data_directory of PostgreSQL to the new location
    # This is not strictly needed because we are starting postgresql with pointing to the data directory
    new_data_directory = "'%s'" % PG_DATA_DIR_HOST
    cmd = 'sed -i "s|data_directory = .*|data_directory = %s|g" %s' % (new_data_directory, PG_CONF)
    subprocess.call(cmd, shell=True)
<commit_msg>Use modified configs (galaxy.ini/job_conf.xml) in in /export.<commit_after>import sys
import os
import shutil
import subprocess

if len( sys.argv ) == 2:
    PG_DATA_DIR_DEFAULT = sys.argv[1]
else:
    PG_DATA_DIR_DEFAULT = "/var/lib/postgresql/9.3/main"
PG_DATA_DIR_HOST = os.environ.get("PG_DATA_DIR_HOST", "/export/postgresql/9.3/main/")
PG_CONF = '/etc/postgresql/9.3/main/postgresql.conf'

def change_path( src ):
    """
        src will be copied to /export/`src` and a symlink will be placed in src pointing to /export/
    """
    if os.path.exists( src ):
        dest = os.path.join( '/export/', src.strip('/') )
        # if destination is empty move all files into /export/ and symlink back to source
        if not os.path.exists( dest ):
            dest_dir = os.path.dirname(dest)
            if not os.path.exists( dest_dir ):
                os.makedirs(dest_dir)
            shutil.move( src, dest )
            os.symlink( dest, src.rstrip('/') )
            os.chown( src, int(os.environ['GALAXY_UID']), int(os.environ['GALAXY_GID']) )
            subprocess.call( 'chown -R %s:%s %s' % ( os.environ['GALAXY_UID'], os.environ['GALAXY_GID'], dest ), shell=True )
        # if destination exists (e.g. continuing a previous session), remove source and symlink
        else:
            if os.path.isdir( src ):
                shutil.rmtree( src )
            else:
                os.unlink( src )
            os.symlink( dest, src.rstrip('/') )

if __name__ == "__main__":
    """
        If the '/export/' folder exist, meaning docker was started with '-v /home/foo/bar:/export',
        we will link every file that needs to persist to the host system. Addionaly a file (/.galaxy_save) is
        created that indicates all linking is already done.
        If the user re-starts (with docker start) the container the file /.galaxy_save is found and the linking
        is aborted.
    """
    if os.path.exists( '/export/.distribution_config/' ):
        shutil.rmtree( '/export/.distribution_config/' )
    shutil.copytree( '/galaxy-central/config/', '/export/.distribution_config/' )

    # Copy all files starting with "welcome"
    # This enables a flexible start page design.
    for filename in os.listdir('/export/'):
        if filename.startswith('welcome'):
            export_file = os.path.join( '/export/', filename)
            image_file = os.path.join('/etc/galaxy/', filename)
            shutil.copy(export_file, image_file)

    for config in [ 'galaxy.ini', 'job_conf.xml' ]:
        export_config = os.path.join( '/export/galaxy-central/config', config )
        if os.path.exists(export_config):
            image_config = os.path.join('/etc/galaxy/', config)
            shutil.copy(export_config, image_config)

    if not os.path.exists( '/export/galaxy-central/' ):
        os.makedirs("/export/galaxy-central/")
        os.chown( "/export/galaxy-central/", int(os.environ['GALAXY_UID']), int(os.environ['GALAXY_GID']) )
    change_path('/galaxy-central/config/')
    change_path('/galaxy-central/integrated_tool_panel.xml')
    change_path('/galaxy-central/display_applications/')
    change_path('/galaxy-central/tool_deps/')
    change_path('/galaxy-central/tool-data/')
    change_path('/shed_tools/')

    if os.path.exists('/export/reports_htpasswd'):
        shutil.copy('/export/reports_htpasswd', '/etc/nginx/htpasswd')

    try:
        change_path('/var/lib/docker/')
    except:
        # In case of unprivileged access this will result in a "Device or resource busy." error.
        pass

    if not os.path.exists( PG_DATA_DIR_HOST ) or 'PG_VERSION' not in os.listdir( PG_DATA_DIR_HOST ):
        dest_dir = os.path.dirname( PG_DATA_DIR_HOST )
        if not os.path.exists( dest_dir ):
            os.makedirs(dest_dir)
        # User given dbpath, usually a directory from the host machine
        # copy the postgresql data folder to the new location
        subprocess.call('cp -R %s/* %s' % (PG_DATA_DIR_DEFAULT, PG_DATA_DIR_HOST), shell=True)
        # copytree needs an non-existing dst dir, how annoying :(
        # shutil.copytree(PG_DATA_DIR_DEFAULT, PG_DATA_DIR_HOST)
        subprocess.call('chown -R postgres:postgres /export/postgresql/', shell=True)
        subprocess.call('chmod -R 0755 /export/', shell=True)
        subprocess.call('chmod -R 0700 %s' % PG_DATA_DIR_HOST, shell=True)

    # change data_directory of PostgreSQL to the new location
    # This is not strictly needed because we are starting postgresql with pointing to the data directory
    new_data_directory = "'%s'" % PG_DATA_DIR_HOST
    cmd = 'sed -i "s|data_directory = .*|data_directory = %s|g" %s' % (new_data_directory, PG_CONF)
    subprocess.call(cmd, shell=True)
<|endoftext|>