<commit_before># ----------------------------------------------------------------------------
# Copyright (c) 2016-2019, QIIME 2 development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file LICENSE, distributed with this software.
# ----------------------------------------------------------------------------

from itertools import product, chain
import networkx as nx
import copy
import qiime2

def get_next_arguments(action, type="input"):
    """
    Get a tuple of required/nonrequired inputs or outputs for each method

    Parameters
    ----------
    action : Qiime2.action

    type : {"input", "param", "output"}
        Delineates if getting the action input, param, or output types

    Returns
    -------
    List of tuples containing name and required semantic types
    List of tuples containing name and optional semantic types
    """

    req = []
    non_req = []

    if type == "input":
        for k, v in action.signature.inputs.items():
            if not v.has_default():
                req.append([k, v.qiime_type])
            else:
                non_req.append(["."+k, v.qiime_type])
    elif type == "param":
        for k, v in action.signature.parameters.items():
            if not v.has_default():
                req.append([k, v.qiime_type])
            else:
                non_req.append(["."+k, v.qiime_type])
    else:
        for k, v in action.signature.outputs.items():
            if not v.has_default():
                req.append([k, v.qiime_type])
            else:
                non_req.append(["."+k, v.qiime_type])

    return req, non_req

def unravel(l):
    """
    Unravel Union node to get all permutations of types for each action

    Parameters
    ----------
    list : list of Qiime2.types

    Returns
    -------
    list of lists - list of permuations of types for each action

    """

    result = [l]
    for i, x in enumerate(l):
        if len(list(x[1])) > 1:
            members = list(x[1])
            temp = copy.deepcopy(result)

            # update result with first element of types in member
            for each_list in result:
                each_list[i][1] = members[0]

            # add in other permutations of types in member
            for n in range(1, len(members)):
                copy_result = copy.deepcopy(temp)
                for each_list in copy_result:
                    each_list[i][1] = members[n]
                result += copy_result
    return result

def generate_nodes_by_action(action, opt=False):
    """
    Given a method, generates all combinations of inputs and
    outputs for that particular method and and stores the combinations
    as dictionaries in a resulting list.

    Parameters
    ----------
    method : Qiime2.action
    opt : {True, False}
        Delineates if optional types should be included

    Returns
    -------
    list of dictionaries - each dictionary is a combination inputs and
        outputs for particular node

    """

    input, input_nr = get_next_arguments(action, "input")
    param, param_nr = get_next_arguments(action, "param")
    output, output_nr = get_next_arguments(action, "output")

    input = unravel(input)
    param = unravel(param)

    opt_in_list = []

    if opt:
        opt_in_list += input_nr
        opt_in_list += param_nr
        opt_in_list = unravel(opt_in_list)
        ins = [dict(x) for x in
               [list(chain.from_iterable(i)) for i in
                list(product(input, param, opt_in_list))]]
        outs = dict(output + output_nr)
        results = [{'inputs': i, 'outputs': outs} for i in ins]
        return results

    ins = [dict(x) for x in
           [list(chain.from_iterable(i)) for i in list(product(input, param))]]
    outs = dict(output)
    results = [{'inputs': i, 'outputs': outs} for i in ins]
    return results

def build_graph(action_list=[], opt=False):
    """
    Constructs a networkx graph with different semantic types
    and actions as nodes

    Parameters
    ----------
    action_list : list of Qiime2.action
        If list is empty, will pull from all methods in the Qiime2 plugin
    opt : {True, False}
        Delineates if optional types should be included in the graph

    Returns
    -------
    nx.DiGraph - networkx graph connected based on all or specified methods

    """

    G = nx.DiGraph()
    G.edges(data=True)

    # get all actions or specifc actions if specified in sigs
    pm = qiime2.sdk.PluginManager()
    if not action_list:
        for _, pg in pm.plugins.items():
            action_list += list(pg.actions.values())

    for action in action_list:
        results = generate_nodes_by_action(action, opt)
        for dict_ in results:
            for k, v in dict_.items():
                if not v:
                    continue

                # renaming dictionary to remove '.'
                action_node = {}
                for x, y in v.items():
                    if x[0] == '.':
                        action_node[x[1:]] = y
                    else:
                        action_node[x] = y
                dict_[k] = action_node

                if not G.has_node(str(dict_)):
                    G.add_node(str(dict_), value=action, node='action')

                if k == 'inputs':
                    for in_k, in_v in v.items():
                        if not in_v:
                            continue
                        if in_k[0] == '.':
                            name = "opt_"+str(in_v)
                            G.add_edge(name, str(dict_))
                            G[name][str(dict_)]['name'] = in_k[1:]
                            G.node[name]['type'] = in_v
                            G.node[name]['optional'] = True
                            G.node[name]['node'] = 'type'
                        else:
                            G.add_edge(in_v, str(dict_))
                            G[in_v][str(dict_)]['name'] = in_k
                            G.node[in_v]['type'] = in_v
                            G.node[in_v]['optional'] = False
                            G.node[in_v]['node'] = 'type'
                else:
                    for out_k, out_v in v.items():
                        if not out_v:
                            continue
                        if out_k[0] == '.':
                            name = "opt_"+str(out_v)
                            G.add_edge("opt_"+str(out_v), str(dict_))
                            G[str(dict_)][name]['name'] = out_k[1:]
                            G.node[name]['type'] = in_v
                            G.node[name]['optional'] = True
                            G.node[name]['node'] = 'type'
                        else:
                            G.add_edge(str(dict_), out_v)
                            G[str(dict_)][out_v]['name'] = out_k
                            G.node[out_v]['type'] = out_v
                            G.node[out_v]['optional'] = False
                            G.node[out_v]['node'] = 'type'
    return G
<commit_msg>DEP: networkx 2.4 updates (#502)<commit_after># ----------------------------------------------------------------------------
# Copyright (c) 2016-2019, QIIME 2 development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file LICENSE, distributed with this software.
# ----------------------------------------------------------------------------

from itertools import product, chain
import networkx as nx
import copy
import qiime2

def get_next_arguments(action, type="input"):
    """
    Get a tuple of required/nonrequired inputs or outputs for each method

    Parameters
    ----------
    action : Qiime2.action

    type : {"input", "param", "output"}
        Delineates if getting the action input, param, or output types

    Returns
    -------
    List of tuples containing name and required semantic types
    List of tuples containing name and optional semantic types
    """

    req = []
    non_req = []

    if type == "input":
        for k, v in action.signature.inputs.items():
            if not v.has_default():
                req.append([k, v.qiime_type])
            else:
                non_req.append(["."+k, v.qiime_type])
    elif type == "param":
        for k, v in action.signature.parameters.items():
            if not v.has_default():
                req.append([k, v.qiime_type])
            else:
                non_req.append(["."+k, v.qiime_type])
    else:
        for k, v in action.signature.outputs.items():
            if not v.has_default():
                req.append([k, v.qiime_type])
            else:
                non_req.append(["."+k, v.qiime_type])

    return req, non_req

def unravel(l):
    """
    Unravel Union node to get all permutations of types for each action

    Parameters
    ----------
    list : list of Qiime2.types

    Returns
    -------
    list of lists - list of permuations of types for each action

    """

    result = [l]
    for i, x in enumerate(l):
        if len(list(x[1])) > 1:
            members = list(x[1])
            temp = copy.deepcopy(result)

            # update result with first element of types in member
            for each_list in result:
                each_list[i][1] = members[0]

            # add in other permutations of types in member
            for n in range(1, len(members)):
                copy_result = copy.deepcopy(temp)
                for each_list in copy_result:
                    each_list[i][1] = members[n]
                result += copy_result
    return result

def generate_nodes_by_action(action, opt=False):
    """
    Given a method, generates all combinations of inputs and
    outputs for that particular method and and stores the combinations
    as dictionaries in a resulting list.

    Parameters
    ----------
    method : Qiime2.action
    opt : {True, False}
        Delineates if optional types should be included

    Returns
    -------
    list of dictionaries - each dictionary is a combination inputs and
        outputs for particular node

    """

    input, input_nr = get_next_arguments(action, "input")
    param, param_nr = get_next_arguments(action, "param")
    output, output_nr = get_next_arguments(action, "output")

    input = unravel(input)
    param = unravel(param)

    opt_in_list = []

    if opt:
        opt_in_list += input_nr
        opt_in_list += param_nr
        opt_in_list = unravel(opt_in_list)
        ins = [dict(x) for x in
               [list(chain.from_iterable(i)) for i in
                list(product(input, param, opt_in_list))]]
        outs = dict(output + output_nr)
        results = [{'inputs': i, 'outputs': outs} for i in ins]
        return results

    ins = [dict(x) for x in
           [list(chain.from_iterable(i)) for i in list(product(input, param))]]
    outs = dict(output)
    results = [{'inputs': i, 'outputs': outs} for i in ins]
    return results

def build_graph(action_list=[], opt=False):
    """
    Constructs a networkx graph with different semantic types
    and actions as nodes

    Parameters
    ----------
    action_list : list of Qiime2.action
        If list is empty, will pull from all methods in the Qiime2 plugin
    opt : {True, False}
        Delineates if optional types should be included in the graph

    Returns
    -------
    nx.DiGraph - networkx graph connected based on all or specified methods

    """

    G = nx.DiGraph()
    G.edges(data=True)

    # get all actions or specifc actions if specified in sigs
    pm = qiime2.sdk.PluginManager()
    if not action_list:
        for _, pg in pm.plugins.items():
            action_list += list(pg.actions.values())

    for action in action_list:
        results = generate_nodes_by_action(action, opt)
        for dict_ in results:
            for k, v in dict_.items():
                if not v:
                    continue

                # renaming dictionary to remove '.'
                action_node = {}
                for x, y in v.items():
                    if x[0] == '.':
                        action_node[x[1:]] = y
                    else:
                        action_node[x] = y
                dict_[k] = action_node

                if not G.has_node(str(dict_)):
                    G.add_node(str(dict_), value=action, node='action')

                if k == 'inputs':
                    for in_k, in_v in v.items():
                        if not in_v:
                            continue
                        if in_k[0] == '.':
                            name = "opt_"+str(in_v)
                            G.add_edge(name, str(dict_))
                            G[name][str(dict_)]['name'] = in_k[1:]
                            G.nodes[name]['type'] = in_v
                            G.nodes[name]['optional'] = True
                            G.nodes[name]['node'] = 'type'
                        else:
                            G.add_edge(in_v, str(dict_))
                            G[in_v][str(dict_)]['name'] = in_k
                            G.nodes[in_v]['type'] = in_v
                            G.nodes[in_v]['optional'] = False
                            G.nodes[in_v]['node'] = 'type'
                else:
                    for out_k, out_v in v.items():
                        if not out_v:
                            continue
                        if out_k[0] == '.':
                            name = "opt_"+str(out_v)
                            G.add_edge("opt_"+str(out_v), str(dict_))
                            G[str(dict_)][name]['name'] = out_k[1:]
                            G.nodes[name]['type'] = in_v
                            G.nodes[name]['optional'] = True
                            G.nodes[name]['node'] = 'type'
                        else:
                            G.add_edge(str(dict_), out_v)
                            G[str(dict_)][out_v]['name'] = out_k
                            G.nodes[out_v]['type'] = out_v
                            G.nodes[out_v]['optional'] = False
                            G.nodes[out_v]['node'] = 'type'
    return G
<|endoftext|>


<commit_before>
# Sample code implementing LeNet-5 from Liu Liu

import tensorflow as tf
import numpy as np
import time
import h5py
# import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix
import itertools
from copy import deepcopy

import os
import os.path

# from tensorflow.examples.tutorials.mnist import input_data
# mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)

class cnnMNIST(object):
    def __init__(self):
        self.use_gpu = True
        self.lr = 1e-3
        self.epochs = 400
        self.runname = 'cnndetandsid_{}'.format(self.epochs)
        self.dataset_filename = 'sequential_dataset_relabel_allseconds.h5'
        self.build_graph()

    def onehot_labels(self, labels):
        out = np.zeros((labels.shape[0], 7))
        for i in range(labels.shape[0]):
            out[i, :] = np.eye(7)[labels[i]]
        return out

    def onenothot_labels(self, labels):
        out = np.zeros((labels.shape[0],))
        for i in range(labels.shape[0]):
            out[i] = np.argmax(labels[i, :])
        return out

    def get_data(self):
        # data_norm = True
        # data_augmentation = False
        try:
            f = h5py.File(self.dataset_filename, 'r')
        except:
            # f = h5py.File('/home/holiestcow/Documents/2017_fall/ne697_hayward/lecture/datacompetition/sequential_dataset_balanced.h5', 'r')
            f = h5py.File('../data/{}'.format(self.dataset_filename), 'r')

        training = f['train']
        testing = f['test']

        training_dataset = []
        training_labels = []
        for item in training:
            training_dataset += [np.array(training[item]['measured_spectra'])]
            training_labels += [np.array(training[item]['labels'])]
        training_dataset = np.concatenate(training_dataset, axis=0)
        training_labels = np.array(training_labels)
        training_labels = np.concatenate(training_labels, axis=0)

        testing_dataset = []
        testing_labels = []
        for item in testing:
            testing_dataset += [np.array(testing[item]['measured_spectra'])]
            testing_labels += [np.array(testing[item]['labels'])]
        testing_dataset = np.concatenate(testing_dataset, axis=0)
        testing_labels = np.concatenate(testing_labels, axis=0)

        self.x_train = training_dataset
        self.y_train = self.onehot_labels(training_labels)
        self.x_test = testing_dataset
        self.y_test = self.onehot_labels(testing_labels)
        return

    def naive_get_data(self):
        # data_norm = True
        # data_augmentation = False

        f = h5py.File('naive_dataset.h5', 'r')
        g = f['training']
        X = np.array(g['spectra'])
        Y = self.onehot_labels(np.array(g['labels'], dtype=np.int32))

        g = f['testing']
        X_test = np.array(g['spectra'])
        Y_test = self.onehot_labels(np.array(g['labels'], dtype=np.int32))

        self.x_train = X
        self.y_train = Y

        self.x_test = X_test
        self.y_test = Y_test

        f.close()

        return

    def batch(self, iterable, n=1):
        # l = len(iterable)
        l = iterable.shape[0]
        for ndx in range(0, l, n):
            data = iterable[ndx:min(ndx + n, l), :]
            # normalization = np.linalg.norm(data, 1, axis=1)
            # for j in range(data.shape[0]):
            #     data[j, :] = np.divide(data[j, :], normalization[j])
            yield data

    def validation_batcher(self):
        try:
            f = h5py.File(self.dataset_filename, 'r')
        except:
            f = h5py.File('../data/{}'.format('sequential_dataset_relabel_validationonly.h5'), 'r')
        g = f['validate']
        samplelist = list(g.keys())

        for i in range(len(samplelist)):
            data = np.array(g[samplelist[i]])
            yield data, samplelist[i]

    def build_graph(self):
        feature_map1 = 32
        feature_map2 = 64

        final_hidden_nodes = 512

        self.x = tf.placeholder(tf.float32, shape=[None, 1024])
        self.y_ = tf.placeholder(tf.float32, shape=[None, 7])
        self.keep_prob = tf.placeholder(tf.float32)

        x_image = self.hack_1dreshape(self.x)
        # define conv-layer variables
        W_conv1 = self.weight_variable([1, 9, 1, feature_map1])    # first conv-layer has 32 kernels, size=5
        b_conv1 = self.bias_variable([feature_map1])
        W_conv2 = self.weight_variable([1, 3, feature_map1, feature_map2])
        b_conv2 = self.bias_variable([feature_map2])

        # x_image = tf.reshape(self.x, [-1, 28, 28, 1])
        h_conv1 = tf.nn.relu(self.conv2d(x_image, W_conv1) + b_conv1)
        h_pool1 = self.max_pool_2x2(h_conv1)
        h_conv2 = tf.nn.relu(self.conv2d(h_pool1, W_conv2) + b_conv2)
        h_pool2 = self.max_pool_2x2(h_conv2)

        # densely/fully connected layer
        W_fc1 = self.weight_variable([256 * feature_map2, final_hidden_nodes])
        b_fc1 = self.bias_variable([final_hidden_nodes])

        h_pool2_flat = tf.reshape(h_pool2, [-1, 256 * feature_map2])
        h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)

# dropout regularization
        h_fc1_drop = tf.nn.dropout(h_fc1, self.keep_prob)

        # linear classifier
        W_fc2 = self.weight_variable([final_hidden_nodes, 7])
        b_fc2 = self.bias_variable([7])

        y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2
        self.y_conv = y_conv

        # Now I have to weight to logits
        # class_weights = tf.constant([0.1, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0])
        # class_weights = tf.constant([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0])
        # self.y_conv = tf.multiply(y_conv, class_weights)
        cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=self.y_, logits=self.y_conv))
        # reg = tf.nn.l2_loss(W_conv1) + tf.nn.l2_loss(W_conv2) + tf.nn.l2_loss(W_fc1)
        # beta = 0.01
        # cross_entropy = tf.reduce_mean(cross_entropy + reg * beta)
        self.train_step = tf.train.AdamOptimizer(self.lr).minimize(cross_entropy)

    def shuffle(self):
        rng_state = np.random.get_state()
        np.random.set_state(rng_state)
        np.random.shuffle(self.x_train)
        np.random.set_state(rng_state)
        np.random.shuffle(self.y_train)
        # permutation = np.random.permutation(self.x_train.shape[0])
        # self.x_train = self.x_train[permutation, :]
        # self.y_train = self.y_train[permutation, :]
        return

    def train(self):
        if self.use_gpu:
            # use half of  the gpu memory
            gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.3)
            self.sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
        else:
            self.sess = tf.Session()
        # self.sess = tf.Session()
        init = tf.global_variables_initializer()
        self.sess.run(init)
        self.eval() # creating evaluation
        a = time.time()
        for i in range(self.epochs):
            # batch = mnist.train.next_batch(50)
            x_generator = self.batch(self.x_train, n=128)
            y_generator = self.batch(self.y_train, n=128)
            # print(batch[0].shape)
            # print(batch[1].shape)
            if i % 10 == 0 and i != 0:
                test_acc = self.sess.run(self.accuracy,feed_dict={self.x: self.x_test[:200, :],
                    self.y_: self.y_test[:200, :],
                    self.keep_prob: 1.0})
                train_acc = self.sess.run(self.accuracy, feed_dict={self.x: current_x,
                                                                   self.y_: current_y,
                                                                   self.keep_prob: 1.0})
                print('step %d, training accuracy %g, testing accuracy %g, elapsed time %f' % (i, train_acc, test_acc, time.time()-a))
            for current_x in x_generator:
               current_y = next(y_generator)
               self.sess.run([self.train_step], feed_dict={self.x: current_x,
                                                           self.y_: current_y,
                                                           self.keep_prob: 0.05})

            self.shuffle()

    def eval(self):
        # self.time_index = np.arange(self.y_conv.get_shape()[0])
        self.prediction = tf.argmax(self.y_conv, 1)
        truth = tf.argmax(self.y_, 1)
        correct_prediction = tf.equal(self.prediction, truth)
        self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    def test_eval(self):
        self.eval()
        x_generator = self.batch(self.x_test, n=100)
        y_generator = self.batch(self.y_test, n=100)
        test_acc = []
        counter = 0
        for data in x_generator:
            test_acc += [self.sess.run(self.accuracy, feed_dict={
            self.x: data, self.y_: next(y_generator), self.keep_prob: 1.0})]
        total_test_acc = sum(test_acc) / float(len(test_acc))
        print('test accuracy %g' % total_test_acc)

    def weight_variable(self, shape):
        initial = tf.truncated_normal(shape, stddev=0.1)
        return tf.Variable(initial)

    def bias_variable(self, shape):
        initial = tf.constant(0.1, shape=shape)
        return tf.Variable(initial)

    def hack_1dreshape(self, x):
        # expand its dimensionality to fit into conv2d
        tensor_expand = tf.expand_dims(x, 1)
        tensor_expand = tf.expand_dims(tensor_expand, -1)
        return tensor_expand

    def conv2d(self, x, W):
        return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')

    def max_pool_2x2(self, x):
        return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],
                                strides=[1, 2, 2, 1], padding='SAME')

    def get_label_predictions(self):
        x_batcher = self.batch(self.x_test, n=256)
        predictions = np.zeros((0, 1))
        for data in x_batcher:
            temp_predictions = self.sess.run(
            self.prediction,
            feed_dict={self.x: data,
                       self.keep_prob: 1.0})
            temp_predictions = temp_predictions.reshape((temp_predictions.shape[0], 1))
            predictions = np.vstack((predictions, temp_predictions))
        return predictions

def main():
    cnn = cnnMNIST()
    a = time.time()
    print('Retrieving data')
    cnn.get_data()
    b = time.time()
    print('Built the data in {} s'.format(b-a))

    validation_data = cnn.validation_batcher()

    a = time.time()
    cnn.train()
    b = time.time()
    print('Training time: {} s'.format(b-a))
    cnn.test_eval()

    predictions = cnn.get_label_predictions()

    predictions_decode = predictions
    labels_decode = cnn.onenothot_labels(cnn.y_test)

    np.save('{}_{}_predictions.npy'.format(cnn.runname, cnn.dataset_filename[:-4]), predictions_decode)
    np.save('{}_{}_ground_truth.npy'.format(cnn.runname, cnn.dataset_filename[:-4]), labels_decode)

    answers = open('approach1_answers_{}_{}.csv'.format(cnn.runname, cnn.dataset_filename[:-4]), 'w')
    answers.write('RunID,SourceID,SourceTime,Comment\n')
    counter = 0
    for sample, runname in validation_data:
        x = sample
        x = x[30:, :]
        predictions = cnn.sess.run(
            cnn.prediction,
            feed_dict = {cnn.x: x,
                         cnn.keep_prob: 1.0})
        time_index = np.arange(predictions.shape[0])
        mask = predictions >= 0.5

        # runname = sample.name.split('/')[-1]
        # runname = sample.name
        if np.sum(mask) != 0:
            counts = np.sum(x, axis=1)
            # fig = plt.figure()
            t = time_index[mask]
            t = [int(i) for i in t]
            index_guess = np.argmax(counts[t])

            current_predictions = predictions[mask]

            answers.write('{},{},{},\n'.format(
                runname, current_predictions[index_guess], t[index_guess] + 30))
        else:
            answers.write('{},{},{},\n'.format(
                runname, 0, 0))

        if counter % 1000 == 0:
            print('{} validation samples complete'.format(counter))
        counter += 1
    answers.close()
    return

main()

<commit_msg>added  debug plotting for the validation set<commit_after>
# Sample code implementing LeNet-5 from Liu Liu

import tensorflow as tf
import numpy as np
import time
import h5py
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix
import itertools
from copy import deepcopy

import os
import os.path

# from tensorflow.examples.tutorials.mnist import input_data
# mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)

class cnnMNIST(object):
    def __init__(self):
        self.use_gpu = True
        self.lr = 1e-3
        self.epochs = 400
        self.runname = 'cnndetandsid_{}'.format(self.epochs)
        self.dataset_filename = 'sequential_dataset_relabel_allseconds.h5'
        self.build_graph()

    def onehot_labels(self, labels):
        out = np.zeros((labels.shape[0], 7))
        for i in range(labels.shape[0]):
            out[i, :] = np.eye(7)[labels[i]]
        return out

    def onenothot_labels(self, labels):
        out = np.zeros((labels.shape[0],))
        for i in range(labels.shape[0]):
            out[i] = np.argmax(labels[i, :])
        return out

    def get_data(self):
        # data_norm = True
        # data_augmentation = False
        try:
            f = h5py.File(self.dataset_filename, 'r')
        except:
            # f = h5py.File('/home/holiestcow/Documents/2017_fall/ne697_hayward/lecture/datacompetition/sequential_dataset_balanced.h5', 'r')
            f = h5py.File('../data/{}'.format(self.dataset_filename), 'r')

        training = f['train']
        testing = f['test']

        training_dataset = []
        training_labels = []
        for item in training:
            training_dataset += [np.array(training[item]['measured_spectra'])]
            training_labels += [np.array(training[item]['labels'])]
        training_dataset = np.concatenate(training_dataset, axis=0)
        training_labels = np.array(training_labels)
        training_labels = np.concatenate(training_labels, axis=0)

        testing_dataset = []
        testing_labels = []
        for item in testing:
            testing_dataset += [np.array(testing[item]['measured_spectra'])]
            testing_labels += [np.array(testing[item]['labels'])]
        testing_dataset = np.concatenate(testing_dataset, axis=0)
        testing_labels = np.concatenate(testing_labels, axis=0)

        self.x_train = training_dataset
        self.y_train = self.onehot_labels(training_labels)
        self.x_test = testing_dataset
        self.y_test = self.onehot_labels(testing_labels)
        return

    def naive_get_data(self):
        # data_norm = True
        # data_augmentation = False

        f = h5py.File('naive_dataset.h5', 'r')
        g = f['training']
        X = np.array(g['spectra'])
        Y = self.onehot_labels(np.array(g['labels'], dtype=np.int32))

        g = f['testing']
        X_test = np.array(g['spectra'])
        Y_test = self.onehot_labels(np.array(g['labels'], dtype=np.int32))

        self.x_train = X
        self.y_train = Y

        self.x_test = X_test
        self.y_test = Y_test

        f.close()

        return

    def batch(self, iterable, n=1):
        # l = len(iterable)
        l = iterable.shape[0]
        for ndx in range(0, l, n):
            data = iterable[ndx:min(ndx + n, l), :]
            # normalization = np.linalg.norm(data, 1, axis=1)
            # for j in range(data.shape[0]):
            #     data[j, :] = np.divide(data[j, :], normalization[j])
            yield data

    def validation_batcher(self, testing=False):
        if testing:
            f = h5py.File('../data/{}'.format(self.dataset_filename), 'r')
        else:
            f = h5py.File('../data/{}'.format('sequential_dataset_relabel_validationonly.h5'), 'r')
        g = f['validate']
        samplelist = list(g.keys())

        for i in range(len(samplelist)):
            data = np.array(g[samplelist[i]])
            yield data, samplelist[i]

    def build_graph(self):
        feature_map1 = 32
        feature_map2 = 64

        final_hidden_nodes = 512

        self.x = tf.placeholder(tf.float32, shape=[None, 1024])
        self.y_ = tf.placeholder(tf.float32, shape=[None, 7])
        self.keep_prob = tf.placeholder(tf.float32)

        x_image = self.hack_1dreshape(self.x)
        # define conv-layer variables
        W_conv1 = self.weight_variable([1, 9, 1, feature_map1])    # first conv-layer has 32 kernels, size=5
        b_conv1 = self.bias_variable([feature_map1])
        W_conv2 = self.weight_variable([1, 3, feature_map1, feature_map2])
        b_conv2 = self.bias_variable([feature_map2])

        # x_image = tf.reshape(self.x, [-1, 28, 28, 1])
        h_conv1 = tf.nn.relu(self.conv2d(x_image, W_conv1) + b_conv1)
        h_pool1 = self.max_pool_2x2(h_conv1)
        h_conv2 = tf.nn.relu(self.conv2d(h_pool1, W_conv2) + b_conv2)
        h_pool2 = self.max_pool_2x2(h_conv2)

        # densely/fully connected layer
        W_fc1 = self.weight_variable([256 * feature_map2, final_hidden_nodes])
        b_fc1 = self.bias_variable([final_hidden_nodes])

        h_pool2_flat = tf.reshape(h_pool2, [-1, 256 * feature_map2])
        h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)

# dropout regularization
        h_fc1_drop = tf.nn.dropout(h_fc1, self.keep_prob)

        # linear classifier
        W_fc2 = self.weight_variable([final_hidden_nodes, 7])
        b_fc2 = self.bias_variable([7])

        y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2
        self.y_conv = y_conv

        # Now I have to weight to logits
        # class_weights = tf.constant([0.1, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0])
        # class_weights = tf.constant([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0])
        # self.y_conv = tf.multiply(y_conv, class_weights)
        cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=self.y_, logits=self.y_conv))
        # reg = tf.nn.l2_loss(W_conv1) + tf.nn.l2_loss(W_conv2) + tf.nn.l2_loss(W_fc1)
        # beta = 0.01
        # cross_entropy = tf.reduce_mean(cross_entropy + reg * beta)
        self.train_step = tf.train.AdamOptimizer(self.lr).minimize(cross_entropy)

    def shuffle(self):
        rng_state = np.random.get_state()
        np.random.set_state(rng_state)
        np.random.shuffle(self.x_train)
        np.random.set_state(rng_state)
        np.random.shuffle(self.y_train)
        # permutation = np.random.permutation(self.x_train.shape[0])
        # self.x_train = self.x_train[permutation, :]
        # self.y_train = self.y_train[permutation, :]
        return

    def train(self):
        if self.use_gpu:
            # use half of  the gpu memory
            gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.3)
            self.sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
        else:
            self.sess = tf.Session()
        # self.sess = tf.Session()
        init = tf.global_variables_initializer()
        self.sess.run(init)
        self.eval() # creating evaluation
        a = time.time()
        for i in range(self.epochs):
            # batch = mnist.train.next_batch(50)
            x_generator = self.batch(self.x_train, n=128)
            y_generator = self.batch(self.y_train, n=128)
            # print(batch[0].shape)
            # print(batch[1].shape)
            if i % 10 == 0 and i != 0:
                test_acc = self.sess.run(self.accuracy,feed_dict={self.x: self.x_test[:200, :],
                    self.y_: self.y_test[:200, :],
                    self.keep_prob: 1.0})
                train_acc = self.sess.run(self.accuracy, feed_dict={self.x: current_x,
                                                                   self.y_: current_y,
                                                                   self.keep_prob: 1.0})
                print('step %d, training accuracy %g, testing accuracy %g, elapsed time %f' % (i, train_acc, test_acc, time.time()-a))
            for current_x in x_generator:
               current_y = next(y_generator)
               self.sess.run([self.train_step], feed_dict={self.x: current_x,
                                                           self.y_: current_y,
                                                           self.keep_prob: 0.05})

            self.shuffle()

    def eval(self):
        # self.time_index = np.arange(self.y_conv.get_shape()[0])
        self.prediction = tf.argmax(self.y_conv, 1)
        truth = tf.argmax(self.y_, 1)
        correct_prediction = tf.equal(self.prediction, truth)
        self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    def test_eval(self):
        self.eval()
        x_generator = self.batch(self.x_test, n=100)
        y_generator = self.batch(self.y_test, n=100)
        test_acc = []
        counter = 0
        for data in x_generator:
            test_acc += [self.sess.run(self.accuracy, feed_dict={
            self.x: data, self.y_: next(y_generator), self.keep_prob: 1.0})]
        total_test_acc = sum(test_acc) / float(len(test_acc))
        print('test accuracy %g' % total_test_acc)

    def weight_variable(self, shape):
        initial = tf.truncated_normal(shape, stddev=0.1)
        return tf.Variable(initial)

    def bias_variable(self, shape):
        initial = tf.constant(0.1, shape=shape)
        return tf.Variable(initial)

    def hack_1dreshape(self, x):
        # expand its dimensionality to fit into conv2d
        tensor_expand = tf.expand_dims(x, 1)
        tensor_expand = tf.expand_dims(tensor_expand, -1)
        return tensor_expand

    def conv2d(self, x, W):
        return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')

    def max_pool_2x2(self, x):
        return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],
                                strides=[1, 2, 2, 1], padding='SAME')

    def get_label_predictions(self):
        x_batcher = self.batch(self.x_test, n=256)
        predictions = np.zeros((0, 1))
        for data in x_batcher:
            temp_predictions = self.sess.run(
            self.prediction,
            feed_dict={self.x: data,
                       self.keep_prob: 1.0})
            temp_predictions = temp_predictions.reshape((temp_predictions.shape[0], 1))
            predictions = np.vstack((predictions, temp_predictions))
        return predictions

def label_datasets():

    targetfile = '/home/holiestcow/Documents/zephyr/datasets/muse/trainingData/answers.csv'
    head, tail = os.path.split(targetfile)

    # filename = []
    source_labels = {}

    id2string = {0: 'Background',
                 1: 'HEU',
                 2: 'WGPu',
                 3: 'I131',
                 4: 'Co60',
                 5: 'Tc99',
                 6: 'HEUandTc99'}

    f = open(targetfile, 'r')
    a = f.readlines()
    for i in range(len(a)):
        line = a[i].strip()
        if line[0] == 'R':
            continue
        parsed = line.split(',')
        filename = parsed[0]
        source = parsed[1]
        source_time = parsed[2]
        source_labels[filename] = {'source': id2string[int(source)],
                                   'time': float(source_time)}
    f.close()
    return source_labels

def main():
    cnn = cnnMNIST()
    a = time.time()
    print('Retrieving data')
    cnn.get_data()
    b = time.time()
    print('Built the data in {} s'.format(b-a))

    validation_data = cnn.validation_batcher()
    isTest = True
    testing_data = cnn.validation_batcher(testing=True)
    label_dict = label_datasets()

    a = time.time()
    cnn.train()
    b = time.time()
    print('Training time: {} s'.format(b-a))
    cnn.test_eval()

    predictions = cnn.get_label_predictions()

    predictions_decode = predictions
    labels_decode = cnn.onenothot_labels(cnn.y_test)

    np.save('{}_{}_predictions.npy'.format(cnn.runname, cnn.dataset_filename[:-4]), predictions_decode)
    np.save('{}_{}_ground_truth.npy'.format(cnn.runname, cnn.dataset_filename[:-4]), labels_decode)

    answers = open('approach1_answers_{}_{}.csv'.format(cnn.runname, cnn.dataset_filename[:-4]), 'w')
    answers.write('RunID,SourceID,SourceTime,Comment\n')
    counter = 0
    if isTest:
        for sample, runname in testing_data:
            x = sample
            x = x[30:, :]
            predictions = cnn.sess.run(
                cnn.prediction,
                feed_dict = {cnn.x: x,
                             cnn.keep_prob: 1.0})
            time_index = np.arange(predictions.shape[0])
            mask = predictions >= 0.5

            # runname = sample.name.split('/')[-1]
            # runname = sample.name
            if np.sum(mask) != 0:
                counts = np.sum(x, axis=1)
                # fig = plt.figure()
                t = time_index[mask]
                t = [int(i) for i in t]
                index_guess = np.argmax(counts[t])

                current_predictions = predictions[mask]

            if counter < 30 and np.sum(mask) != 0:
                fig = plt.figure()
                plt.plot(counts, 'b.')
                plt.plot(counts[mask], 'r.')
                plt.plot(counts[index_guess], 'g*')
                plt.plot(counts[int(label_dict[runname]['time']) - 30], 'm*')
                fig.savefig('hitcounts_{}.png'.format(counter))
            else:
                break
            counter += 1
        answers.close()
        return

    for sample, runname in validation_data:
        x = sample
        x = x[30:, :]
        predictions = cnn.sess.run(
            cnn.prediction,
            feed_dict = {cnn.x: x,
                         cnn.keep_prob: 1.0})
        time_index = np.arange(predictions.shape[0])
        mask = predictions >= 0.5

        # runname = sample.name.split('/')[-1]
        # runname = sample.name
        if np.sum(mask) != 0:
            counts = np.sum(x, axis=1)
            # fig = plt.figure()
            t = time_index[mask]
            t = [int(i) for i in t]
            index_guess = np.argmax(counts[t])

            current_predictions = predictions[mask]

            answers.write('{},{},{},\n'.format(
                runname, current_predictions[index_guess], t[index_guess] + 30))
        else:
            answers.write('{},{},{},\n'.format(
                runname, 0, 0))

        if counter % 1000 == 0:
            print('{} validation samples complete'.format(counter))
        counter += 1
    answers.close()
    return

main()

<|endoftext|>


<commit_before>from pathlib import PureWindowsPath, PurePosixPath
from buildbot.status.results import SUCCESS
from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand
from buildbot.steps.transfer import FileUpload
from buildbot.steps.master import MasterShellCommand
from buildbot.process.factory import BuildFactory
from buildbot.process.properties import Property, Interpolate
from infostore import web_url

# Constants
python_exe_property_name = 'python_exe'
dir_command_property_name = 'dir_command'
run_cpp_builds_property_name = 'run_cpp_builds'
run_release_builds_property_name = 'run_release_builds'
hide_cpp_builds_property_name = 'hide_cpp_builds'
hide_release_builds_property_name = 'hide_release_builds'

python_exe = Property('python_exe', default='python')

def step_has_property(property_name, default=None, takesResults=False):
    def check_for_property(step):
        result = step.getProperty(property_name)
        if result is None:
            return default
        else:
            return result
    if takesResults:
        return lambda results, s: check_for_property(s)
    else:
        return check_for_property

def step_has_properties(property_names, default=None, takesResults=False,
                        sentinal=None):
    def check_for_property(step):
        for name in property_names:
            result = step.getProperty(name)
            if result is not sentinal:
                return result
        return default
    if takesResults:
        return lambda results, s: check_for_property(s)
    else:
        return check_for_property

# Git Urls
nim_git_url = 'git://github.com/Araq/Nimrod'
csources_git_url = 'git://github.com/nimrod-code/csources'
scripts_git_url = 'git://github.com/nimrod-code/nim-buildbot'

# Resource directories
resource_dirs = {
    'current_dir': './',
    'nim_dir': 'build/',
    'csources_dir': 'build/csources/',
    'scripts_dir': 'scripts/',
    'tester_dir': 'tests/testament/',
    'compiler_dir': 'compiler'
}

class PlatformPaths:
    pass
windows_directories = PlatformPaths()
posix_directories = PlatformPaths()
for key, value in resource_dirs.iteritems():
    setattr(windows_directories, key, PureWindowsPath(value))
    setattr(posix_directories, key, PurePosixPath(value))

# Utility Functions

def inject_paths(func):
    def wrapper(platform, *args, **kwargs):
        platform_directories = posix_directories
        if platform == 'windows':
            platform_directories = windows_directories
        return func(platform_directories, *args, **kwargs)
    return wrapper

has_passed = lambda results, step: results == SUCCESS

# Build Steps

# Common build step parameters
common_git_parameters = {
    'haltOnFailure': True,
    'description': 'Updating',
    'descriptionDone': 'Updated',

    'mode': 'incremental',
    'retry': (10, 20),

    'progress': True,
    'clobberOnFailure': True
}

# Steps

@inject_paths
def update_utility_scripts(platform):
    """
    Updates the utility scripts used by other steps.
    """
    common_parameters = common_git_parameters.copy()
    common_parameters['haltOnFailure'] = False

    return [
        Git(
            name="Update Utility Scripts",
            descriptionSuffix=' Utility Scripts',

            repourl=scripts_git_url,
            alwaysUseLatest=True,

            workdir=str(platform.scripts_dir),
            hideStepIf=True,
            **common_parameters
        )
    ]

@inject_paths
def update_repositories(platform):
    """
    Adds the steps needed to update the csources and Nimrod repositories.
    """
    return [
        Git(
            name="Update Local Nim Repository",
            descriptionSuffix=' Local Nim Repository',

            repourl=nim_git_url,

            workdir=str(platform.nim_dir),
            **common_git_parameters
        ),
        Git(
            name="Update Local CSources Repository",
            descriptionSuffix=' Local CSources Repository',

            repourl=csources_git_url,
            alwaysUseLatest=True,

            workdir=str(platform.csources_dir),
            **common_git_parameters
        )
    ]

@inject_paths
def clean_repositories(platform):
    """
    Adds the steps needed to clean the Nimrod and CSources repositories.
    Requires that the repositories already exist, in the layout done by
    update_repositories.
    """
    csources_filter = str(platform.csources_dir / '*')
    return [
        ShellCommand(
            name='Clean Local Nim Repository',
            description='Cleaning',
            descriptionDone='Cleaned',
            descriptionSuffix=' Local Nim Repository',

            command=[
                'git', 'clean', '-x', '-f', '-e', csources_filter, '-d'],
            workdir=str(platform.nim_dir),
            haltOnFailure=True,
        ),

        ShellCommand(
            name='Clean Local CSources Repository',
            description='Cleaning',
            descriptionDone='Cleaned',
            descriptionSuffix=' Local CSources Repository',

            command=['git', 'clean', '-x', '-f', '-d'],
            workdir=str(platform.nim_dir),
            haltOnFailure=True,
        )
    ]

@inject_paths
def build_csources(platform, csources_script_cmd):
    """
    Builds the csources binary. Requires that the csources repository be
    present and that a suitable C compiler be present on the system path.
    """
    return [
        ShellCommand(
            name='Build Basic Nim Binary',
            description='Building',
            descriptionDone='Built',
            descriptionSuffix=' Basic CSources Binary',

            command=csources_script_cmd,
            workdir=str(platform.csources_dir),
            haltOnFailure=True,
        )
    ]

@inject_paths
def normalize_nim_names(platform):
    """
    Makes sure that both a 'nim' and 'nimrod' binary are present.
    """
    bin_dir = str(platform.nim_dir / 'bin')
    return [
        ShellCommand(
            name='Normalizing Binary Names',
            description='Normalizing',
            descriptionDone='Normalized',
            descriptionSuffix=' Binary Names',

            command=[python_exe, str(
                platform.scripts_dir / 'normalize_nim.py'), bin_dir],
            workdir=str(platform.current_dir),
            hideStepIf=False
        )
    ]

@inject_paths
def compile_koch(platform):
    """
    Compiles the koch utility.
    """
    bin_dir = str(platform.nim_dir / 'bin')
    base_env = {
        'PATH': [
            str(platform.current_dir),
            bin_dir,
            'bin',
            "${PATH}"
        ]
    }

    return [
        ShellCommand(
            name='Compile Koch',
            description='Compiling',
            descriptionDone='Compiled',
            descriptionSuffix=' Koch Binary',

            command=['nim', 'c', 'koch.nim'],
            env=base_env,
            workdir=str(platform.nim_dir),
            haltOnFailure=True,
        )
    ]

@inject_paths
def boot_nimrod(platform):
    nimfile_dir = str(platform.compiler_dir / 'nim.nim')
    base_env = {
        'PATH': [
            str(platform.current_dir),
            str(platform.current_dir / 'bin'),
            'bin',
            "${PATH}"
        ]
    }

    return [
        ShellCommand(
            name='Bootstrap Debug Version of Nim Compiler (With C Backend)',
            description='Booting',
            descriptionDone='Booted',
            descriptionSuffix=' Debug Nim Compiler (With C Backend)',

            command=['koch', 'boot'],
            workdir=str(platform.nim_dir),
            env=base_env,
            haltOnFailure=True,
        ),

        ShellCommand(
            name='Bootstrap Release Version of Nim Compiler (With C Backend)',
            description='Booting',
            descriptionDone='Booted',
            descriptionSuffix=' Release Nim Compiler (With C Backend)',

            command=['nim', 'c', '-d:release', nimfile_dir],
            workdir=str(platform.nim_dir),
            env=base_env,

            doStepIf=step_has_property(
                property_name=run_release_builds_property_name,
                default=True),
            hideStepIf=step_has_property(
                property_name=hide_release_builds_property_name,
                default=False,
                takesResults=True),
            haltOnFailure=False
        ),

        ShellCommand(
            name='Bootstrap Debug Version of Nim Compiler (With C++ Backend)',
            description='Booting',
            descriptionDone='Booted',
            descriptionSuffix=' Debug Nim Compiler (With C++ Backend)',

            command=['nim', 'cpp', nimfile_dir],
            workdir=str(platform.nim_dir),
            env=base_env,

            doStepIf=step_has_property(
                property_name=run_cpp_builds_property_name,
                default=True
            ),
            hideStepIf=step_has_property(
                property_name=hide_cpp_builds_property_name,
                default=False,
                takesResults=True
            ),
            haltOnFailure=True
        ),

        ShellCommand(
            name='Bootstrap Release Version of Nim Compiler (With C++ Backend)',
            description='Booting',
            descriptionDone='Booted',
            descriptionSuffix=' Release Nim Compiler (With C++ Backend)',

            command=['nim', 'cpp', '-d:release', nimfile_dir],
            workdir=str(platform.nim_dir),
            env=base_env,

            doStepIf=step_has_properties(
                property_names=[
                    run_cpp_builds_property_name,
                    run_release_builds_property_name
                ],
                default=True
            ),
            hideStepIf=step_has_properties(
                property_names=[
                    hide_cpp_builds_property_name,
                    hide_release_builds_property_name
                ],
                default=False,
                takesResults=True
            ),
            haltOnFailure=False,
        )
    ]

@inject_paths
def run_testament(platform):
    base_env = {
        'PATH': [
            str(platform.current_dir),
            str(platform.current_dir / 'bin'),
            'bin',
            "${PATH}"
        ]
    }
    test_destination = 'build_tests/%(prop:buildername)s/'
    test_file = 'test-%(prop:buildnumber)s.html'

    return [
        ShellCommand(
            name='Run Testament',
            description='Running',
            descriptionDone='Run',
            descriptionSuffix=' Testament',

            command=['koch', 'test'],
            workdir=str(platform.nim_dir),
            env=base_env,
            haltOnFailure=True,
            timeout=21600
        ),

        MasterShellCommand(
            command=['mkdir', '-p', Interpolate('"{0}"'.format(test_destination))],
            path="public_html"
        ),

        FileUpload(
            slavesrc=str(platform.nim_dir / 'testresults.html'),
            masterdest=Interpolate(test_destination + test_file),
            url=Interpolate(web_url + test_destination + test_file)
        )
    ]

@inject_paths
def generate_csources(platform):
    base_env = {
        'PATH': [
            str(platform.current_dir),
            str(platform.current_dir / 'bin'),
            'bin',
            "${PATH}"
        ]
    }

    return [
        ShellCommand(
            name='Generate CSources',
            description='Generating',
            descriptionDone='Generated',
            descriptionSuffix=' CSources',

            command=['koch', 'csources'],
            workdir=str(platform.nim_dir),
            env=base_env,
            haltOnFailure=True,
        )
    ]

@inject_paths
def generate_zip(platform):
    base_env = {
        'PATH': [
            str(platform.current_dir),
            str(platform.current_dir / 'bin'),
            'bin',
            "${PATH}"
        ]
    }

    return [
        ShellCommand(
            name='Generate CSources',
            description='Generating',
            descriptionDone='Generated',
            descriptionSuffix=' CSources',

            command=['koch', 'csources'],
            workdir=str(platform.nim_dir),
            env=base_env,
            haltOnFailure=True,
        )
    ]

@inject_paths
def generate_installer(platform):
    base_env = {
        'PATH': [
            str(platform.current_dir),
            str(platform.current_dir / 'bin'),
            'bin',
            "${PATH}"
        ]
    }

    return [
        ShellCommand(
            name='Generate CSources',
            description='Generating',
            descriptionDone='Generated',
            descriptionSuffix=' CSources',

            command=['koch', 'nsis'],
            workdir=str(platform.nim_dir),
            env=base_env,
            haltOnFailure=True,
        )
    ]

# Build Configurations
def construct_nim_build(platform, csources_script_cmd, f=None):
    if f is None:
        f = BuildFactory()

    steps = []
    steps.extend(update_utility_scripts(platform))
    steps.extend(update_repositories(platform))
    steps.extend(clean_repositories(platform))
    steps.extend(build_csources(platform, csources_script_cmd))
    steps.extend(normalize_nim_names(platform))
    steps.extend(compile_koch(platform))
    steps.extend(boot_nimrod(platform))
    steps.extend(run_testament(platform))
    for step in steps:
        f.addStep(step)

    return f
<commit_msg>Fix #5<commit_after>from pathlib import PureWindowsPath, PurePosixPath
from buildbot.status.results import SUCCESS
from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand
from buildbot.steps.transfer import FileUpload
from buildbot.steps.master import MasterShellCommand
from buildbot.process.factory import BuildFactory
from buildbot.process.properties import Property, Interpolate
from infostore import web_url

# Constants
python_exe_property_name = 'python_exe'
dir_command_property_name = 'dir_command'
run_cpp_builds_property_name = 'run_cpp_builds'
run_release_builds_property_name = 'run_release_builds'
hide_cpp_builds_property_name = 'hide_cpp_builds'
hide_release_builds_property_name = 'hide_release_builds'

python_exe = Property('python_exe', default='python')

def step_has_property(property_name, default=None, takesResults=False):
    def check_for_property(step):
        result = step.getProperty(property_name)
        if result is None:
            return default
        else:
            return result
    if takesResults:
        return lambda results, s: check_for_property(s)
    else:
        return check_for_property

def step_has_properties(property_names, default=None, takesResults=False,
                        sentinal=None):
    def check_for_property(step):
        for name in property_names:
            result = step.getProperty(name)
            if result is not sentinal:
                return result
        return default
    if takesResults:
        return lambda results, s: check_for_property(s)
    else:
        return check_for_property

# Git Urls
nim_git_url = 'git://github.com/Araq/Nimrod'
csources_git_url = 'git://github.com/nimrod-code/csources'
scripts_git_url = 'git://github.com/nimrod-code/nim-buildbot'

# Resource directories
resource_dirs = {
    'current_dir': './',
    'nim_dir': 'build/',
    'csources_dir': 'build/csources/',
    'scripts_dir': 'scripts/',
    'tester_dir': 'tests/testament/',
    'compiler_dir': 'compiler'
}

class PlatformPaths:
    pass
windows_directories = PlatformPaths()
posix_directories = PlatformPaths()
for key, value in resource_dirs.iteritems():
    setattr(windows_directories, key, PureWindowsPath(value))
    setattr(posix_directories, key, PurePosixPath(value))

# Utility Functions

def inject_paths(func):
    def wrapper(platform, *args, **kwargs):
        platform_directories = posix_directories
        if platform == 'windows':
            platform_directories = windows_directories
        return func(platform_directories, *args, **kwargs)
    return wrapper

has_passed = lambda results, step: results == SUCCESS

# Build Steps

# Common build step parameters
common_git_parameters = {
    'haltOnFailure': True,
    'description': 'Updating',
    'descriptionDone': 'Updated',

    'mode': 'incremental',
    'retry': (10, 20),

    'progress': True,
    'clobberOnFailure': True
}

# Steps

@inject_paths
def update_utility_scripts(platform):
    """
    Updates the utility scripts used by other steps.
    """
    common_parameters = common_git_parameters.copy()
    common_parameters['haltOnFailure'] = False

    return [
        Git(
            name="Update Utility Scripts",
            descriptionSuffix=' Utility Scripts',

            repourl=scripts_git_url,
            alwaysUseLatest=True,

            workdir=str(platform.scripts_dir),
            hideStepIf=True,
            **common_parameters
        )
    ]

@inject_paths
def update_repositories(platform):
    """
    Adds the steps needed to update the csources and Nimrod repositories.
    """
    return [
        Git(
            name="Update Local Nim Repository",
            descriptionSuffix=' Local Nim Repository',

            repourl=nim_git_url,

            workdir=str(platform.nim_dir),
            **common_git_parameters
        ),
        Git(
            name="Update Local CSources Repository",
            descriptionSuffix=' Local CSources Repository',

            repourl=csources_git_url,
            alwaysUseLatest=True,

            workdir=str(platform.csources_dir),
            **common_git_parameters
        )
    ]

@inject_paths
def clean_repositories(platform):
    """
    Adds the steps needed to clean the Nimrod and CSources repositories.
    Requires that the repositories already exist, in the layout done by
    update_repositories.
    """
    csources_filter = str(platform.csources_dir / '*')
    return [
        ShellCommand(
            name='Clean Local Nim Repository',
            description='Cleaning',
            descriptionDone='Cleaned',
            descriptionSuffix=' Local Nim Repository',

            command=[
                'git', 'clean', '-x', '-f', '-e', csources_filter, '-d'],
            workdir=str(platform.nim_dir),
            haltOnFailure=True,
        ),

        ShellCommand(
            name='Clean Local CSources Repository',
            description='Cleaning',
            descriptionDone='Cleaned',
            descriptionSuffix=' Local CSources Repository',

            command=['git', 'clean', '-x', '-f', '-d'],
            workdir=str(platform.nim_dir),
            haltOnFailure=True,
        )
    ]

@inject_paths
def build_csources(platform, csources_script_cmd):
    """
    Builds the csources binary. Requires that the csources repository be
    present and that a suitable C compiler be present on the system path.
    """
    return [
        ShellCommand(
            name='Build Basic Nim Binary',
            description='Building',
            descriptionDone='Built',
            descriptionSuffix=' Basic CSources Binary',

            command=csources_script_cmd,
            workdir=str(platform.csources_dir),
            haltOnFailure=True,
        )
    ]

@inject_paths
def normalize_nim_names(platform):
    """
    Makes sure that both a 'nim' and 'nimrod' binary are present.
    """
    bin_dir = str(platform.nim_dir / 'bin')
    return [
        ShellCommand(
            name='Normalizing Binary Names',
            description='Normalizing',
            descriptionDone='Normalized',
            descriptionSuffix=' Binary Names',

            command=[python_exe, str(
                platform.scripts_dir / 'normalize_nim.py'), bin_dir],
            workdir=str(platform.current_dir),
            hideStepIf=False
        )
    ]

@inject_paths
def compile_koch(platform):
    """
    Compiles the koch utility.
    """
    bin_dir = str(platform.nim_dir / 'bin')
    base_env = {
        'PATH': [
            str(platform.current_dir),
            bin_dir,
            'bin',
            "${PATH}"
        ]
    }

    return [
        ShellCommand(
            name='Compile Koch',
            description='Compiling',
            descriptionDone='Compiled',
            descriptionSuffix=' Koch Binary',

            command=['nim', 'c', 'koch.nim'],
            env=base_env,
            workdir=str(platform.nim_dir),
            haltOnFailure=True,
        )
    ]

@inject_paths
def boot_nimrod(platform):
    nimfile_dir = str(platform.compiler_dir / 'nim.nim')
    base_env = {
        'PATH': [
            str(platform.current_dir),
            str(platform.current_dir / 'bin'),
            'bin',
            "${PATH}"
        ]
    }

    return [
        ShellCommand(
            name='Bootstrap Debug Version of Nim Compiler (With C Backend)',
            description='Booting',
            descriptionDone='Booted',
            descriptionSuffix=' Debug Nim Compiler (With C Backend)',

            command=['koch', 'boot'],
            workdir=str(platform.nim_dir),
            env=base_env,
            haltOnFailure=True,
        ),

        ShellCommand(
            name='Bootstrap Release Version of Nim Compiler (With C Backend)',
            description='Booting',
            descriptionDone='Booted',
            descriptionSuffix=' Release Nim Compiler (With C Backend)',

            command=['nim', 'c', '-d:release', nimfile_dir],
            workdir=str(platform.nim_dir),
            env=base_env,

            doStepIf=step_has_property(
                property_name=run_release_builds_property_name,
                default=True),
            hideStepIf=step_has_property(
                property_name=hide_release_builds_property_name,
                default=False,
                takesResults=True),
            haltOnFailure=False
        ),

        ShellCommand(
            name='Bootstrap Debug Version of Nim Compiler (With C++ Backend)',
            description='Booting',
            descriptionDone='Booted',
            descriptionSuffix=' Debug Nim Compiler (With C++ Backend)',

            command=['nim', 'cpp', nimfile_dir],
            workdir=str(platform.nim_dir),
            env=base_env,

            doStepIf=step_has_property(
                property_name=run_cpp_builds_property_name,
                default=True
            ),
            hideStepIf=step_has_property(
                property_name=hide_cpp_builds_property_name,
                default=False,
                takesResults=True
            ),
            haltOnFailure=True
        ),

        ShellCommand(
            name='Bootstrap Release Version of Nim Compiler (With C++ Backend)',
            description='Booting',
            descriptionDone='Booted',
            descriptionSuffix=' Release Nim Compiler (With C++ Backend)',

            command=['nim', 'cpp', '-d:release', nimfile_dir],
            workdir=str(platform.nim_dir),
            env=base_env,

            doStepIf=step_has_properties(
                property_names=[
                    run_cpp_builds_property_name,
                    run_release_builds_property_name
                ],
                default=True
            ),
            hideStepIf=step_has_properties(
                property_names=[
                    hide_cpp_builds_property_name,
                    hide_release_builds_property_name
                ],
                default=False,
                takesResults=True
            ),
            haltOnFailure=False,
        )
    ]

@inject_paths
def run_testament(platform):
    base_env = {
        'PATH': [
            str(platform.current_dir),
            str(platform.current_dir / 'bin'),
            'bin',
            "${PATH}"
        ]
    }
    test_destination = 'build_tests/%(prop:buildername)s/'
    test_file = 'test-%(prop:buildnumber)s.html'

    return [
        ShellCommand(
            name='Run Testament',
            description='Running',
            descriptionDone='Run',
            descriptionSuffix=' Testament',

            command=['koch', 'test'],
            workdir=str(platform.nim_dir),
            env=base_env,
            haltOnFailure=True,
            timeout=21600
        ),

        MasterShellCommand(
            command=['mkdir', '-p', Interpolate(test_destination)],
            path="public_html",
            hideStepIf=True
        ),

        FileUpload(
            slavesrc=str(platform.nim_dir / 'testresults.html'),
            masterdest=Interpolate(test_destination + test_file),
            url=Interpolate(web_url + test_destination + test_file)
        )
    ]

@inject_paths
def generate_csources(platform):
    base_env = {
        'PATH': [
            str(platform.current_dir),
            str(platform.current_dir / 'bin'),
            'bin',
            "${PATH}"
        ]
    }

    return [
        ShellCommand(
            name='Generate CSources',
            description='Generating',
            descriptionDone='Generated',
            descriptionSuffix=' CSources',

            command=['koch', 'csources'],
            workdir=str(platform.nim_dir),
            env=base_env,
            haltOnFailure=True,
        )
    ]

@inject_paths
def generate_zip(platform):
    base_env = {
        'PATH': [
            str(platform.current_dir),
            str(platform.current_dir / 'bin'),
            'bin',
            "${PATH}"
        ]
    }

    return [
        ShellCommand(
            name='Generate CSources',
            description='Generating',
            descriptionDone='Generated',
            descriptionSuffix=' CSources',

            command=['koch', 'csources'],
            workdir=str(platform.nim_dir),
            env=base_env,
            haltOnFailure=True,
        )
    ]

@inject_paths
def generate_installer(platform):
    base_env = {
        'PATH': [
            str(platform.current_dir),
            str(platform.current_dir / 'bin'),
            'bin',
            "${PATH}"
        ]
    }

    return [
        ShellCommand(
            name='Generate CSources',
            description='Generating',
            descriptionDone='Generated',
            descriptionSuffix=' CSources',

            command=['koch', 'nsis'],
            workdir=str(platform.nim_dir),
            env=base_env,
            haltOnFailure=True,
        )
    ]

# Build Configurations
def construct_nim_build(platform, csources_script_cmd, f=None):
    if f is None:
        f = BuildFactory()

    steps = []
    steps.extend(update_utility_scripts(platform))
    steps.extend(update_repositories(platform))
    steps.extend(clean_repositories(platform))
    steps.extend(build_csources(platform, csources_script_cmd))
    steps.extend(normalize_nim_names(platform))
    steps.extend(compile_koch(platform))
    steps.extend(boot_nimrod(platform))
    steps.extend(run_testament(platform))
    for step in steps:
        f.addStep(step)

    return f
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
 Abtract: These classes are the specifications of the core concepts, adapted from the Haskell.
          The classes are written in an object-oriented style.
"""

__author__ = "Werner Kuhn and Andrea Ballatore"
__copyright__ = "Copyright 2014"
__credits__ = ["Werner Kuhn", "Andrea Ballatore"]
__license__ = ""
__version__ = "0.1"
__maintainer__ = "Andrea Ballatore"
__email__ = ""
__date__ = "August 2014"
__status__ = "Development"

from utils import _init_log

log = _init_log("coreconcepts_oo")

class ALocate(object):
    """
    IGNORE THIS CLASS FOR THE MOMENT.
    Class defining abstract location relations
    """

    @staticmethod
    def isAt( figure, ground ):
        """
        @return Bool
        """
        raise NotImplementedError("isAt")

    @staticmethod
    def isIn( figure, ground ):
        """
        @return Bool
        """
        raise NotImplementedError("isIn")

    @staticmethod
    def isPart( figure, ground ):
        """
        @return Bool
        """
        raise NotImplementedError("isPart")

class CcField(object):
    """
    Class defining abstract field.
    Based on Field.hs
    """

    def __init__(self):
        pass

    def getValue( self, position ):
        """ @return the value of field at position """
        raise NotImplementedError("getValue")

    def setValue( self, position, value ):
        """ @return the position of new value in field """
        raise NotImplementedError("setValue")

    def domain( self, position, value ):
        """ @return Domains can be described as intervals, rectangles, corner points, convex hulls or boundaries """
        raise NotImplementedError("domain")

    def neigh( self, position ):
        """
        Map algebra: neighborhood function
        @return Geometry
        """
        raise NotImplementedError("neigh")

    def zone( self, position ):
        """
        Map algebra: zone function
        @return Geometry
        """
        raise NotImplementedError("zone")

    def local( self, fun ):
        """
        Map algebra's local operations, with a function to compute the new values
        @return new field
        """
        raise NotImplementedError("local")

    def focal( self, fun ):
        """
        Map algebra's focal operations, with a kernel function to compute the new values based on the neighborhood of the position
        @return new field
        """
        raise NotImplementedError("focal")

    def zonal( self, fun ):
        """
        Map algebra's zonal operations, with a function to compute the new values based on zones containing the positions.
        @return new field
        """
        raise NotImplementedError("zonal")

class CcObject(object):
    """
    Abstract class for core concept 'object'
    Based on Object.hs
    """

    def bounds( self ):
        raise NotImplementedError("bounds")

    def relation( self, obj, relType ):
        """ @return Boolean True if self and obj are in a relationship of type relType
                    False otherwise
        """
        raise NotImplementedError("relation")

    def property( self, prop ):
        """
        @param prop the property name
        @return value of property in obj
        """
        raise NotImplementedError("property")

    def identity( self, obj ):
        """
        @param an object
        @return Boolean True if self and obj are identical
        """
        raise NotImplementedError("identity")

class CcNetwork(object):
    """
    Abstract class for core concept 'network'
    Based on Network.hs
    """

    def __init__(self):
        pass

    def nodes( self ):
        """ @return all nodes part of this network """
        raise NotImplementedError("nodes")

    def edges( self ):
        """ @return all edges part of this network """
        raise NotImplementedError("edges")

    def addNode( self, node ):
        """ adds given node to this network """
        raise NotImplementedError("addNode")

    def addEdge( self, edge ):
        """ adds given edge to this network """
        raise NotImplementedError("addEdge")

    def connected( self, nodeA, nodeB ):
        """ @return whether B can be reached from A in this network """
        raise NotImplementedError("connected")

    def shortestPath( self, nodeA, nodeB ):
        """ @return list of edges """
        raise NotImplementedError("shortestPath")

    def degree( self, node ):
        """ @return the number of edges to other nodes in this network """
        raise NotImplementedError("degree")

    def linking( netw, edge ):
        """ @return the nodes encapsulating given edge in this network """
        raise NotImplementedError("linking")

    def distance( self, nodeA, nodeB ):
        """ @return the length of shortest path from A to B in this network """
        raise NotImplementedError("distance")

    def breadthFirst( self, node, distance ):
        """ @return all nodes within the distance from node in this network """
        raise NotImplementedError("breadthFirst")

class CcEvent(object):
    """
    Abstract class for core concept 'event'.
    Based on Event.hs
    """

    def __init__(self):
        pass

    def within( self ):
        """
        @return a Period
        """
        raise NotImplementedError("within")

    def when( self ):
        """
        @return a Period
        """
        raise NotImplementedError("when")

    def during( self, event ):
        """
        @param event an event
        @return boolean
        """
        raise NotImplementedError("during")

    @staticmethod
    def before( self, event ):
        """
        @param event an event
        @return Boolean
        """
        raise NotImplementedError("before")

    @staticmethod
    def after( self, event ):
        """
        @param event an event
        @return Boolean
        """
        raise NotImplementedError("after")

    @staticmethod
    def overlap( self, event ):
        """
        @param event an event
        @return Boolean
        """
        raise NotImplementedError("overlap")

class Period(object):
    """ Simple period class. TODO: implement"""
    def __init__(self):
        pass
<commit_msg>Addapted new version of network core concept in Python to NetworkX<commit_after>#!/usr/bin/env python

"""
 Abtract: These classes are the specifications of the core concepts, adapted from the Haskell.
          The classes are written in an object-oriented style.
"""

__author__ = "Werner Kuhn and Andrea Ballatore"
__copyright__ = "Copyright 2014"
__credits__ = ["Werner Kuhn", "Andrea Ballatore"]
__license__ = ""
__version__ = "0.1"
__maintainer__ = "Andrea Ballatore"
__email__ = ""
__date__ = "August 2014"
__status__ = "Development"

from utils import _init_log

log = _init_log("coreconcepts_oo")

class ALocate(object):
    """
    IGNORE THIS CLASS FOR THE MOMENT.
    Class defining abstract location relations
    """

    @staticmethod
    def isAt( figure, ground ):
        """
        @return Bool
        """
        raise NotImplementedError("isAt")

    @staticmethod
    def isIn( figure, ground ):
        """
        @return Bool
        """
        raise NotImplementedError("isIn")

    @staticmethod
    def isPart( figure, ground ):
        """
        @return Bool
        """
        raise NotImplementedError("isPart")

class CcField(object):
    """
    Class defining abstract field.
    Based on Field.hs
    """

    def __init__(self):
        pass

    def getValue( self, position ):
        """ @return the value of field at position """
        raise NotImplementedError("getValue")

    def setValue( self, position, value ):
        """ @return the position of new value in field """
        raise NotImplementedError("setValue")

    def domain( self, position, value ):
        """ @return Domains can be described as intervals, rectangles, corner points, convex hulls or boundaries """
        raise NotImplementedError("domain")

    def neigh( self, position ):
        """
        Map algebra: neighborhood function
        @return Geometry
        """
        raise NotImplementedError("neigh")

    def zone( self, position ):
        """
        Map algebra: zone function
        @return Geometry
        """
        raise NotImplementedError("zone")

    def local( self, fun ):
        """
        Map algebra's local operations, with a function to compute the new values
        @return new field
        """
        raise NotImplementedError("local")

    def focal( self, fun ):
        """
        Map algebra's focal operations, with a kernel function to compute the new values based on the neighborhood of the position
        @return new field
        """
        raise NotImplementedError("focal")

    def zonal( self, fun ):
        """
        Map algebra's zonal operations, with a function to compute the new values based on zones containing the positions.
        @return new field
        """
        raise NotImplementedError("zonal")

class CcObject(object):
    """
    Abstract class for core concept 'object'
    Based on Object.hs
    """

    def bounds( self ):
        raise NotImplementedError("bounds")

    def relation( self, obj, relType ):
        """ @return Boolean True if self and obj are in a relationship of type relType
                    False otherwise
        """
        raise NotImplementedError("relation")

    def property( self, prop ):
        """
        @param prop the property name
        @return value of property in obj
        """
        raise NotImplementedError("property")

    def identity( self, obj ):
        """
        @param an object
        @return Boolean True if self and obj are identical
        """
        raise NotImplementedError("identity")

class CcNetwork(object):
    """
    Abstract class for core concept 'network'
    Based on Network.hs
    """

    def __init__(self):
        pass

    def nodes( self ):
        """ @return a copy of the graph nodes in a list """
        raise NotImplementedError("nodes")

    def edges( self ):
        """ @return list of edges """
        raise NotImplementedError("edges")

    def addNode( self, n ):
        """ Add a single node n """
        raise NotImplementedError("addNode")

    def addEdge( self, u, v ):
        """ Add an edge between u and v """
        raise NotImplementedError("addEdge")

    def connected( self, u, v ):
        """ @return whether node v can be reached from node u """
        raise NotImplementedError("connected")

    def shortestPath( self, source, target ):
        """ @return shortest path in the graph """
        raise NotImplementedError("shortestPath")

    def degree( self, n ):
        """ @return number of the nodes connected to the node n """
        raise NotImplementedError("degree")

    def distance( self, source, target ):
        """ @return the length of the shortest path from the source to the target """
        raise NotImplementedError("distance")

    def breadthFirst( self, node, distance ):
        """ @return all nodes within the distance from node in this network """
        raise NotImplementedError("breadthFirst")

class CcEvent(object):
    """
    Abstract class for core concept 'event'.
    Based on Event.hs
    """

    def __init__(self):
        pass

    def within( self ):
        """
        @return a Period
        """
        raise NotImplementedError("within")

    def when( self ):
        """
        @return a Period
        """
        raise NotImplementedError("when")

    def during( self, event ):
        """
        @param event an event
        @return boolean
        """
        raise NotImplementedError("during")

    @staticmethod
    def before( self, event ):
        """
        @param event an event
        @return Boolean
        """
        raise NotImplementedError("before")

    @staticmethod
    def after( self, event ):
        """
        @param event an event
        @return Boolean
        """
        raise NotImplementedError("after")

    @staticmethod
    def overlap( self, event ):
        """
        @param event an event
        @return Boolean
        """
        raise NotImplementedError("overlap")

class Period(object):
    """ Simple period class. TODO: implement"""
    def __init__(self):
        pass
<|endoftext|>


<commit_before>from haystack import indexes

from parliament.hansards.models import Statement

class StatementIndex(indexes.SearchIndex, indexes.Indexable):
    text = indexes.CharField(document=True, model_attr='text_plain')
    searchtext = indexes.CharField(stored=False, use_template=True)
    date = indexes.DateTimeField(model_attr='time')
    politician = indexes.CharField(use_template=True)
    politician_id = indexes.CharField(model_attr='member__politician__identifier', null=True)
    who_hocid = indexes.IntegerField(model_attr='who_hocid', null=True)
    party = indexes.CharField(model_attr='member__party__short_name', null=True)
    province = indexes.CharField(model_attr='member__riding__province', null=True)
    topic = indexes.CharField(model_attr='h2', null=True)
    url = indexes.CharField(model_attr='get_absolute_url', indexed=False)
    doc_url = indexes.CharField(model_attr='document__get_absolute_url')
    committee = indexes.CharField(model_attr='committee_name')
    committee_slug = indexes.CharField(model_attr='committee_slug')
    doctype = indexes.CharField(null=True)

    def get_model(self):
        return Statement

    def index_queryset(self, using=None):
        return Statement.objects.all().prefetch_related(
            'member__politician', 'member__party', 'member__riding', 'document',
            'document__committeemeeting__committee'
        ).order_by('-date')

    def prepare_doctype(self, obj):
        if obj.committee_name:
            return 'committee'
        else:
            return 'debate'
<commit_msg>In French, don't index empty statements<commit_after>from django.conf import settings

from haystack import indexes

from parliament.hansards.models import Statement

class StatementIndex(indexes.SearchIndex, indexes.Indexable):
    text = indexes.CharField(document=True, model_attr='text_plain')
    searchtext = indexes.CharField(stored=False, use_template=True)
    date = indexes.DateTimeField(model_attr='time')
    politician = indexes.CharField(use_template=True)
    politician_id = indexes.CharField(model_attr='member__politician__identifier', null=True)
    who_hocid = indexes.IntegerField(model_attr='who_hocid', null=True)
    party = indexes.CharField(model_attr='member__party__short_name', null=True)
    province = indexes.CharField(model_attr='member__riding__province', null=True)
    topic = indexes.CharField(model_attr='h2', null=True)
    url = indexes.CharField(model_attr='get_absolute_url', indexed=False)
    doc_url = indexes.CharField(model_attr='document__get_absolute_url')
    committee = indexes.CharField(model_attr='committee_name')
    committee_slug = indexes.CharField(model_attr='committee_slug')
    doctype = indexes.CharField(null=True)

    def get_model(self):
        return Statement

    def index_queryset(self, using=None):
        qs = Statement.objects.all().prefetch_related(
            'member__politician', 'member__party', 'member__riding', 'document',
            'document__committeemeeting__committee'
        ).order_by('-date')
        if settings.LANGUAGE_CODE.startswith('fr'):
            qs = qs.exclude(content_fr='')
        return qs

    def prepare_doctype(self, obj):
        if obj.committee_name:
            return 'committee'
        else:
            return 'debate'
<|endoftext|>