<commit_before># tools.py - Tools that can be used everywhere.
# Author: Franck Michea < franck.michea@gmail.com >
# License: New BSD License (See LICENSE)

'''This module provides some tools needed almost everywhere in the code.'''

import os
import re
import shutil
import sys

# pylint: disable=W0402
import string

ROOT = os.path.join(os.path.dirname(__file__), '..')

def join(*args):
    '''Joins a some paths with ROOT as base path.'''
    return os.path.realpath(os.path.join(*args))

def filter_files(filter_fun, *path):
    '''Yields every filenames that match filter_fun in directories.'''
    root = join(*path)
    root_len = len(root)
    for subroot, directories, files in os.walk(root):
        for filename in files:
            if filter_fun(filename):
                yield (root, os.path.join(subroot[root_len:], filename))

def mkdir_p(path):
    '''
    Emulates the behaviour of `mkdir -p` in shell (makes all the directories
    of the path specified.
    '''
    try:
        os.makedirs(path)
    except OSError:
        pass

def empty_directory(directory):
    for root, dirs, files in os.walk(directory, topdown=False):
        for f in files:
            os.unlink(join(root, f))
        for d in dirs:
            os.rmdir(join(root, d))

def move_content(src, dst):
    for root, dirs, files in os.walk(src):
        for f in (files + dirs):
            shutil.move(join(root, f), dst)
        while dirs:
            dirs.pop()

def copy(src, dst):
    '''Copies the file from src to dst.'''
    mkdir_p(os.path.dirname(dst))
    shutil.copyfile(src, dst)

_SLUG_CHARS = string.ascii_lowercase + string.digits + '-'
_SLUG_SIZE = 50

def slugify(text):
    '''Returns the slug of a string (that can be used in an URL for example.'''
    slug = text.lower().replace(' ', '-')
    slug = ''.join(c for c in slug if c in _SLUG_CHARS)
    return slug[:_SLUG_SIZE]

def input_default(prompt, default, required=False, is_valid=None):
    '''
    Prompts the user for input, with a default value if nothing is given from
    the user.
    '''
    try:
        while True:
            tmp = input('{prompt}{required} [{default}]: '.format(
                prompt = prompt,
                default = default if default is not None else '',
                required = ' (required)' if required else '',
            ))
            if not tmp:
                tmp = default
            if (is_valid is None or is_valid(tmp)) and (not required or tmp):
                break
    except (KeyboardInterrupt, EOFError):
        sys.exit(1)
    return tmp

def ask_sure(default=False):
    '''
    Makes sure the user wants to proceed the following action. It returns True
    if user answers yes or y, else False.
    '''
    try:
        sure = input('Are you sure you want to proceed? [{default}] '.format(
            default = 'Y/n' if default else 'y/N',
        ))
    except (KeyboardInterrupt, EOFError):
        sys.exit('')
    return (not sure and default) or (sure.lower() in ['y', 'yes'])

def join_names(names):
    '''
    Returns a well formated list of names, ready for printing.

    Examples:
      - Trevor Reznik
      - Teddy and Leonard
      - Rita, Astor, Cody and Dexter
    '''
    if 1 < len(names):
        return ', '.join(sorted(names[:-1])) + ' and ' + names[-1]
    elif len(names) == 0:
        return '[deleted]'
    return names[0]

def split_on_comma(authors):
    '''Splits a string on commas and retrusn a set of the values.'''
    return (set(re.split(',[ \t]*', authors)) - set(['']))
<commit_msg>add a way to delete field content when editing object.<commit_after># tools.py - Tools that can be used everywhere.
# Author: Franck Michea < franck.michea@gmail.com >
# License: New BSD License (See LICENSE)

'''This module provides some tools needed almost everywhere in the code.'''

import os
import re
import shutil
import sys

# pylint: disable=W0402
import string

ROOT = os.path.join(os.path.dirname(__file__), '..')

def join(*args):
    '''Joins a some paths with ROOT as base path.'''
    return os.path.realpath(os.path.join(*args))

def filter_files(filter_fun, *path):
    '''Yields every filenames that match filter_fun in directories.'''
    root = join(*path)
    root_len = len(root)
    for subroot, directories, files in os.walk(root):
        for filename in files:
            if filter_fun(filename):
                yield (root, os.path.join(subroot[root_len:], filename))

def mkdir_p(path):
    '''
    Emulates the behaviour of `mkdir -p` in shell (makes all the directories
    of the path specified.
    '''
    try:
        os.makedirs(path)
    except OSError:
        pass

def empty_directory(directory):
    for root, dirs, files in os.walk(directory, topdown=False):
        for f in files:
            os.unlink(join(root, f))
        for d in dirs:
            os.rmdir(join(root, d))

def move_content(src, dst):
    for root, dirs, files in os.walk(src):
        for f in (files + dirs):
            shutil.move(join(root, f), dst)
        while dirs:
            dirs.pop()

def copy(src, dst):
    '''Copies the file from src to dst.'''
    mkdir_p(os.path.dirname(dst))
    shutil.copyfile(src, dst)

_SLUG_CHARS = string.ascii_lowercase + string.digits + '-'
_SLUG_SIZE = 50

def slugify(text):
    '''Returns the slug of a string (that can be used in an URL for example.'''
    slug = text.lower().replace(' ', '-')
    slug = ''.join(c for c in slug if c in _SLUG_CHARS)
    return slug[:_SLUG_SIZE]

def input_default(prompt, default, required=False, is_valid=None):
    '''
    Prompts the user for input, with a default value if nothing is given from
    the user.
    '''
    try:
        while True:
            tmp = input('{prompt}{required} [{default}]: '.format(
                prompt = prompt,
                default = default if default is not None else '',
                required = ' (required)' if required else '',
            ))
            if not tmp:
                tmp = default
            if (is_valid is None or is_valid(tmp)) and (not required or tmp):
                break
    except KeyboardInterrupt:
        sys.exit(1)
    except EOFError:
        if not required:
            print('Deleting field content.')
            if ask_sure(default=True):
                return None
        sys.exit(1)
    return tmp

def ask_sure(default=False):
    '''
    Makes sure the user wants to proceed the following action. It returns True
    if user answers yes or y, else False.
    '''
    try:
        sure = input('Are you sure you want to proceed? [{default}] '.format(
            default = 'Y/n' if default else 'y/N',
        ))
    except (KeyboardInterrupt, EOFError):
        sys.exit('')
    return (not sure and default) or (sure.lower() in ['y', 'yes'])

def join_names(names):
    '''
    Returns a well formated list of names, ready for printing.

    Examples:
      - Trevor Reznik
      - Teddy and Leonard
      - Rita, Astor, Cody and Dexter
    '''
    if 1 < len(names):
        return ', '.join(sorted(names[:-1])) + ' and ' + names[-1]
    elif len(names) == 0:
        return '[deleted]'
    return names[0]

def split_on_comma(authors):
    '''Splits a string on commas and retrusn a set of the values.'''
    return (set(re.split(',[ \t]*', authors)) - set(['']))
<|endoftext|>


<commit_before># Copyright (C) 2017 Zhixian MA <zxma_sjtu@qq.com>

"""
A convolutional autoencoder (CAE) constructor based on theano, lasagne, and
nolearn

Reference
=========
[1] convolutional_autoencoder (mikesj)
    https://github.com/mikesj-public/convolutional_autoencoder
[2] nolearn.lasagne
    http://pythonhosted.org/nolearn/lasagne.html
"""

import lasagne
from lasagne.layers import get_output
from lasagne.layers import InputLayer, DenseLayer, Upscale2DLayer, ReshapeLayer
from lasagne.layers import Conv2DLayer, MaxPool2DLayer
from nolearn.lasagne import NeuralNet, BatchIterator, PrintLayerInfo

class ConvAE():
    """
    A convolutional autoencoder (CAE) constructor

    inputs
    ======
    X_in: np.ndarray
        The sample matrix, whose size is (s,d,r,c).
        s: number of samples, d: dimensions
        r: rows, c: cols
    X_out: np.ndarray
        It usually equals to X_in.
    kernel_size: list
        Box sizes of the kernels in each ConvLayer
    kernel_num: list
        Number of kernels in each ConvLayer
    pool_flag: list of bool values
        Flags of pooling layer w.r.t. to the ConvLayer
    fc_nodes: list
        The dense layers after the full connected layer
        of last ConvLayer or pooling layer.
    encode_nodes: int
        Number of nodes in the final encoded layer

    methods
    =======
    gen_layers: construct the layers
    gen_encoder: generate the encoder
    gen_decoder: generate the decoder
    cae_build: build the cae network
    cae_train: train the cae network
    cae_eval: evaluate the cae network
    cae_save: save the network
    """

    def __init__(self, X_in, X_out, kernel_size=[3,5,3],
                 kernel_num=[12,12,24], pool_flag=[True,True,True],
                fc_nodes=[128], encode_nodes=16):
        """
        The initializer
        """
        self.X_in = X_in
        self.X_out = X_out
        self.kernel_size = kernel_size
        self.kernel_num = kernel_num
        self.pool_flag = pool_flag
        self.pool_size = 2
        self.fc_nodes = fc_nodes
        self.encode_nodes = encode_nodes

    def gen_layers(self):
        """Construct the layers"""

        # Init <TODO>
        pad_in = 'valid'
        pad_out = 'full'
        self.layers = []
        # input layer
        l_input = (InputLayer,
                   {'shape': (None, self.X_in.shape[1],
                              self.X_in.shape[2],
                              self.X_in.shape[3])})
        self.layers.append(l_input)
        # Encoder: Conv and pool layers
        rows,cols = self.X_in.shape[2:]
        for i in range(len(self.kernel_size)):
            # conv
            l_en_conv = (Conv2DLayer,
                         {'num_filters': self.kernel_num[i],
                          'filter_size': self.kernel_size[i],
                          'pad': pad_in})
            self.layers.append(l_en_conv)
            rows = rows - self.kernel_size[i] + 1
            cols = cols - self.kernel_size[i] + 1
            # pool
            if self.pool_flag[i]:
                l_en_pool = (MaxPool2DLayer,
                             {'pool_size': self.pool_size})
                self.layers.append(l_en_pool)
                rows = rows // 2
                cols = cols // 2
        # full connected layer
        num_en_fc = rows * cols * self.kernel_num[-1]
        l_en_fc = (ReshapeLayer, {'shape': (([0], -1))})
        self.layers.append(l_en_fc)
        # dense
        for i in range(len(self.fc_nodes)):
            l_en_dense = (DenseLayer, {'num_units': self.fc_nodes[i]})
            self.layers.append(l_en_dense)
        # encoder layer
        l_en = (DenseLayer,
                {'name': 'encode', 'num_units': self.encode_nodes})
        self.layers.append(l_en)

        # Decoder: reverse
        # dense
        for i in range(len(self.fc_nodes)-1, -1, -1):
            l_de_dense = (DenseLayer, {'num_units': self.fc_nodes[i]})
            self.layers.append(l_de_dense)
        # fc
        l_de_fc = (DenseLayer, {'num_units': num_en_fc})
        self.layers.append(l_de_fc)
        # fc to kernels
        l_de_fc_m = (ReshapeLayer,
                     {'shape': (([0], self.kernel_num[-1], rows, cols))})
        self.layers.append(l_de_fc_m)
        # Conv and pool
        for i in range(len(self.kernel_size)-1, -1, -1):
            # pool
            if self.pool_flag[i]:
                l_de_pool = (Upscale2DLayer,
                             {'scale_factor': self.pool_size})
                self.layers.append(l_de_pool)
            # conv
            if i > 0:
                l_de_conv = (Conv2DLayer,
                             {'num_filters': self.kernel_num[i],
                              'filter_size': self.kernel_size[i],
                              'pad': pad_out})
            else:
                l_de_conv = (Conv2DLayer,
                             {'num_filters': 1,
                              'filter_size': self.kernel_size[i],
                              'pad': pad_out})
            self.layers.append(l_de_conv)
        # output
        self.layers.append((ReshapeLayer, {'shape': (([0], -1))}))

    def cae_build(self, max_epochs=20,
                  learning_rate=0.001, momentum=0.9,
                  verbose=1):
        """Build the network"""
        self.cae = NeuralNet(
            layers = self.layers,
            max_epochs = max_epochs,
            update=lasagne.updates.nesterov_momentum,
            update_learning_rate = learning_rate,
            update_momentum = momentum,
            regression = True,
            verbose = verbose)

    def cae_train(self):
        """Train the cae net"""
        print("Training the network...")
        self.cae.fit(self.X_in, self.X_out)
        print("Trainong done.")

    def cae_eval(self):
        """Draw evaluation lines
        <TODO>
        """
        from nolearn.lasagne.visualize import plot_loss
        plot_loss(self.cae)

    def cae_predict(self,img):
        """
        Predict the output of the input image

        input
        =====
        img: np.ndarray
            The image matrix, (r,c)

        output
        ======
        img_pred: np.ndarray
            The predicted image matrix
        """
        if len(img.shape) == 4:
            rows = img.shape[2]
            cols = img.shape[3]
        elif len(img.shape) == 3:
            rows = img.shape[1]
            cols = img.shape[2]
            img = img.reshape(img.shape[0],1,rows,cols)
        elif len(img.shape) == 2:
            rows,cols = img.shape
            img = img.reshape(1,1,rows,cols)
        else:
            print("The shape of image should be 2 or 3 d")
        img_pred = self.cae.precidt(img).reshape(-1, rows, cols)

        return img_pred

    def get_encode(self, img):
        """Encode or compress on the sample

        input
        =====
        img: np.ndarray
            The sample matrix

        output
        ======
        img_en: np.ndarray
            The encoded matrix
        """
        if len(img.shape) == 4:
            rows = img.shape[2]
            cols = img.shape[3]
        elif len(img.shape) == 3:
            rows = img.shape[1]
            cols = img.shape[2]
            img = img.reshape(img.shape[0],1,rows,cols)
        elif len(img.shape) == 2:
            rows,cols = img.shape
            img = img.reshape(1,1,rows,cols)
        else:
            print("The shape of image should be 2 or 3 d")

        def get_layer_by_name(net, name):
            for i, layer in enumerate(net.get_all_layers()):
                if layer.name == name:
                    return layer, i
            return None, None

        encode_layer, encode_layer_index = get_layer_by_name(self.cae,
                                                             'encode')
        img_en =  get_output(encode_layer, inputs=img).eval()

        return img_en

    def get_decode(self, img_en):
        """Decode to output the recovered image

        input
        =====
        img_en: np.ndarray
            The encoded matrix

        output
        ======
        img_de: np.ndarray
            The recovered or predicted image matrix
        """
        def get_layer_by_name(net, name):
            for i, layer in enumerate(net.get_all_layers()):
                if layer.name == name:
                    return layer, i
            return None, None

        encode_layer, encode_layer_index = get_layer_by_name(self.cae,
                                                             'encode')
        # decoder
        new_input = InputLayer(shape=(None,encode_layer.num_units))
        layer_de_input = self.cae.get_all_layers()[encode_layer_index + 1]
        layer_de_input.input_layer = new_input
        layer_de_output = self.cae.get_all_layers()[-1]

        img_en = get_output(layer_de_output, img_en).eval()

        return img_en

    def cae_save(self,savepath='cae.pkl'):
        """Save the trained network

        input
        =====
        savepath: str
            Path of the net to be saved
        """
        import pickle
        fp = open(savepath, 'wb')
        # write
        pickle.dump(self.cae, fp)
        fp.close()
<commit_msg>Add a batchiterator to train batchly.<commit_after># Copyright (C) 2017 Zhixian MA <zxma_sjtu@qq.com>

"""
A convolutional autoencoder (CAE) constructor based on theano, lasagne, and
nolearn

Reference
=========
[1] convolutional_autoencoder (mikesj)
    https://github.com/mikesj-public/convolutional_autoencoder
[2] nolearn.lasagne
    http://pythonhosted.org/nolearn/lasagne.html
"""

import lasagne
from lasagne.layers import get_output
from lasagne.layers import InputLayer, DenseLayer, Upscale2DLayer, ReshapeLayer
from lasagne.layers import Conv2DLayer, MaxPool2DLayer
from nolearn.lasagne import NeuralNet, BatchIterator, PrintLayerInfo

class ConvAE():
    """
    A convolutional autoencoder (CAE) constructor

    inputs
    ======
    X_in: np.ndarray
        The sample matrix, whose size is (s,d,r,c).
        s: number of samples, d: dimensions
        r: rows, c: cols
    X_out: np.ndarray
        It usually equals to X_in.
    kernel_size: list
        Box sizes of the kernels in each ConvLayer
    kernel_num: list
        Number of kernels in each ConvLayer
    pool_flag: list of bool values
        Flags of pooling layer w.r.t. to the ConvLayer
    fc_nodes: list
        The dense layers after the full connected layer
        of last ConvLayer or pooling layer.
    encode_nodes: int
        Number of nodes in the final encoded layer

    methods
    =======
    gen_layers: construct the layers
    gen_encoder: generate the encoder
    gen_decoder: generate the decoder
    cae_build: build the cae network
    cae_train: train the cae network
    cae_eval: evaluate the cae network
    cae_save: save the network
    """

    def __init__(self, X_in, X_out, kernel_size=[3,5,3],
                 kernel_num=[12,12,24], pool_flag=[True,True,True],
                fc_nodes=[128], encode_nodes=16):
        """
        The initializer
        """
        self.X_in = X_in
        self.X_out = X_out
        self.kernel_size = kernel_size
        self.kernel_num = kernel_num
        self.pool_flag = pool_flag
        self.pool_size = 2
        self.fc_nodes = fc_nodes
        self.encode_nodes = encode_nodes

    def gen_BatchIterator(self,batch_size=100):
        """Generate the batch iterator"""
        B = BatchIterator(batch_size=batch_size, shuffle=True)
        return B

    def gen_layers(self):
        """Construct the layers"""

        # Init <TODO>
        pad_in = 'valid'
        pad_out = 'full'
        self.layers = []
        # input layer
        l_input = (InputLayer,
                   {'shape': (None, self.X_in.shape[1],
                              self.X_in.shape[2],
                              self.X_in.shape[3])})
        self.layers.append(l_input)
        # Encoder: Conv and pool layers
        rows,cols = self.X_in.shape[2:]
        for i in range(len(self.kernel_size)):
            # conv
            l_en_conv = (Conv2DLayer,
                         {'num_filters': self.kernel_num[i],
                          'filter_size': self.kernel_size[i],
                          'pad': pad_in})
            self.layers.append(l_en_conv)
            rows = rows - self.kernel_size[i] + 1
            cols = cols - self.kernel_size[i] + 1
            # pool
            if self.pool_flag[i]:
                l_en_pool = (MaxPool2DLayer,
                             {'pool_size': self.pool_size})
                self.layers.append(l_en_pool)
                rows = rows // 2
                cols = cols // 2
        # full connected layer
        num_en_fc = rows * cols * self.kernel_num[-1]
        l_en_fc = (ReshapeLayer, {'shape': (([0], -1))})
        self.layers.append(l_en_fc)
        # dense
        for i in range(len(self.fc_nodes)):
            l_en_dense = (DenseLayer, {'num_units': self.fc_nodes[i]})
            self.layers.append(l_en_dense)
        # encoder layer
        l_en = (DenseLayer,
                {'name': 'encode', 'num_units': self.encode_nodes})
        self.layers.append(l_en)

        # Decoder: reverse
        # dense
        for i in range(len(self.fc_nodes)-1, -1, -1):
            l_de_dense = (DenseLayer, {'num_units': self.fc_nodes[i]})
            self.layers.append(l_de_dense)
        # fc
        l_de_fc = (DenseLayer, {'num_units': num_en_fc})
        self.layers.append(l_de_fc)
        # fc to kernels
        l_de_fc_m = (ReshapeLayer,
                     {'shape': (([0], self.kernel_num[-1], rows, cols))})
        self.layers.append(l_de_fc_m)
        # Conv and pool
        for i in range(len(self.kernel_size)-1, -1, -1):
            # pool
            if self.pool_flag[i]:
                l_de_pool = (Upscale2DLayer,
                             {'scale_factor': self.pool_size})
                self.layers.append(l_de_pool)
            # conv
            if i > 0:
                l_de_conv = (Conv2DLayer,
                             {'num_filters': self.kernel_num[i],
                              'filter_size': self.kernel_size[i],
                              'pad': pad_out})
            else:
                l_de_conv = (Conv2DLayer,
                             {'num_filters': 1,
                              'filter_size': self.kernel_size[i],
                              'pad': pad_out})
            self.layers.append(l_de_conv)
        # output
        self.layers.append((ReshapeLayer, {'shape': (([0], -1))}))

    def cae_build(self, max_epochs=20, batch_size=100,
                  learning_rate=0.001, momentum=0.9,
                  verbose=1):
        """Build the network"""
        if batch_size is None:
            self.cae = NeuralNet(
                layers = self.layers,
                max_epochs = max_epochs,
                update=lasagne.updates.nesterov_momentum,
                update_learning_rate = learning_rate,
                update_momentum = momentum,
                regression = True,
                verbose = verbose)
        else:
            # batch iterator
            batch_iterator = self.gen_BatchIterator(batch_size=batch_size)
            self.cae = NeuralNet(
                layers = self.layers,
                batch_iterator_train = batch_iterator,
                max_epochs = max_epochs,
                update=lasagne.updates.nesterov_momentum,
                update_learning_rate = learning_rate,
                update_momentum = momentum,
                regression = True,
                verbose = verbose)

    def cae_train(self):
        """Train the cae net"""
        print("Training the network...")
        self.cae.fit(self.X_in, self.X_out)
        print("Training done.")
    def cae_eval(self):
        """Draw evaluation lines
        <TODO>
        """
        from nolearn.lasagne.visualize import plot_loss
        plot_loss(self.cae)

    def cae_predict(self,img):
        """
        Predict the output of the input image

        input
        =====
        img: np.ndarray
            The image matrix, (r,c)

        output
        ======
        img_pred: np.ndarray
            The predicted image matrix
        """
        if len(img.shape) == 4:
            rows = img.shape[2]
            cols = img.shape[3]
        elif len(img.shape) == 3:
            rows = img.shape[1]
            cols = img.shape[2]
            img = img.reshape(img.shape[0],1,rows,cols)
        elif len(img.shape) == 2:
            rows,cols = img.shape
            img = img.reshape(1,1,rows,cols)
        else:
            print("The shape of image should be 2 or 3 d")
        img_pred = self.cae.precidt(img).reshape(-1, rows, cols)

        return img_pred

    def get_encode(self, img):
        """Encode or compress on the sample

        input
        =====
        img: np.ndarray
            The sample matrix

        output
        ======
        img_en: np.ndarray
            The encoded matrix
        """
        if len(img.shape) == 4:
            rows = img.shape[2]
            cols = img.shape[3]
        elif len(img.shape) == 3:
            rows = img.shape[1]
            cols = img.shape[2]
            img = img.reshape(img.shape[0],1,rows,cols)
        elif len(img.shape) == 2:
            rows,cols = img.shape
            img = img.reshape(1,1,rows,cols)
        else:
            print("The shape of image should be 2 or 3 d")

        def get_layer_by_name(net, name):
            for i, layer in enumerate(net.get_all_layers()):
                if layer.name == name:
                    return layer, i
            return None, None

        encode_layer, encode_layer_index = get_layer_by_name(self.cae,
                                                             'encode')
        img_en =  get_output(encode_layer, inputs=img).eval()

        return img_en

    def get_decode(self, img_en):
        """Decode to output the recovered image

        input
        =====
        img_en: np.ndarray
            The encoded matrix

        output
        ======
        img_de: np.ndarray
            The recovered or predicted image matrix
        """
        def get_layer_by_name(net, name):
            for i, layer in enumerate(net.get_all_layers()):
                if layer.name == name:
                    return layer, i
            return None, None

        encode_layer, encode_layer_index = get_layer_by_name(self.cae,
                                                             'encode')
        # decoder
        new_input = InputLayer(shape=(None,encode_layer.num_units))
        layer_de_input = self.cae.get_all_layers()[encode_layer_index + 1]
        layer_de_input.input_layer = new_input
        layer_de_output = self.cae.get_all_layers()[-1]

        img_en = get_output(layer_de_output, img_en).eval()

        return img_en

    def cae_save(self,savepath='cae.pkl'):
        """Save the trained network

        input
        =====
        savepath: str
            Path of the net to be saved
        """
        import pickle
        fp = open(savepath, 'wb')
        # write
        pickle.dump(self.cae, fp)
        fp.close()
<|endoftext|>


<commit_before>from sys import exit

def gold_room():
    """This is a function for the room of gold!"""
    print "This room is full of gold. How much do you take?"

    next = raw_input("> ")
    try:
        how_much = int(next)
    except ValueError:
        dead("Man, learn to type a number.")

    if how_much < 50:
        print "Nice, you're not greedy, you win!"
        exit(0)
    else:
        dead("You greedy bastard!")

def bear_room():
    """This is a function for the bear room. RAWR!"""
    print "There is a bear here."
    print "The bear has a bunch of honey."
    print "The fat bear is in front of another door."
    print "How are you going to move the bear?"
    bear_moved = False

    # This loops forever until it calls another function or breaks
    while True:
        next = raw_input("> ")

        if next == "take honey":
            dead("The bear looks at you then slaps your face off.")
        elif next == "taunt bear" and not bear_moved:
            print "The bear has moved from the door. You can go through now."
            bear_moved = True
        elif next == "taunt bear" and bear_moved:
            dead("The bear gets pissed off and chews your leg off.")
        elif next == "open door" and not bear_moved:
            dead("YOU IDIOT!")
        elif next == "open door" and bear_moved:
            gold_room()
        else:
            print "I got no idea what that means."

def cthulhu_room():
    print "Here you see the great evil Cthulhu."
    print "He, it, whatever stares at you and you go insane."
    print "Do you flee for your life or eat your head?"

    next = raw_input("> ")

    if "flee" in next:
        start()
    elif "head" in next:
        dead("Well that was tasty!")
    else:
        cthulhu_room()

def dead(why):
    print why, "Good job!"
    exit(0)

def start():
    print "You are in a dark room."
    print "There is a door to your right and left."
    print "Which one do you take?"

    next = raw_input("> ")

    if next == "left":
        bear_room()
    elif next == "right":
        cthulhu_room()
    else:
        dead("You stumble around the room until you starve.")

start()
<commit_msg>Add function descriptor thingies under functions.<commit_after>from sys import exit

def gold_room():
    """This is a function for the room of gold!"""
    print "This room is full of gold. How much do you take?"

    next = raw_input("> ")
    try:
        how_much = int(next)
    except ValueError:
        dead("Man, learn to type a number.")

    if how_much < 50:
        print "Nice, you're not greedy, you win!"
        exit(0)
    else:
        dead("You greedy bastard!")

def bear_room():
    """This is a function for the bear room. RAWR!"""
    print "There is a bear here."
    print "The bear has a bunch of honey."
    print "The fat bear is in front of another door."
    print "How are you going to move the bear?"
    bear_moved = False

    # This loops forever until it calls another function or breaks
    while True:
        next = raw_input("> ")

        if next == "take honey":
            dead("The bear looks at you then slaps your face off.")
        elif next == "taunt bear" and not bear_moved:
            print "The bear has moved from the door. You can go through now."
            bear_moved = True
        elif next == "taunt bear" and bear_moved:
            dead("The bear gets pissed off and chews your leg off.")
        elif next == "open door" and not bear_moved:
            dead("YOU IDIOT!")
        elif next == "open door" and bear_moved:
            gold_room()
        else:
            print "I got no idea what that means."

def cthulhu_room():
    """This function herlads the Great Old One"""
    print "Here you see the great evil Cthulhu."
    print "He, it, whatever stares at you and you go insane."
    print "Do you flee for your life or eat your head?"

    next = raw_input("> ")

    if "flee" in next:
        start()
    elif "head" in next:
        dead("Well that was tasty!")
    else:
        cthulhu_room()

def dead(why):
    """This function reports your death and kills the program."""
    print why, "Good job!"
    exit(0)

def start():
    """This fucntion is the start of the game"""
    print "You are in a dark room."
    print "There is a door to your right and left."
    print "Which one do you take?"

    next = raw_input("> ")

    if next == "left":
        bear_room()
    elif next == "right":
        cthulhu_room()
    else:
        dead("You stumble around the room until you starve.")

start()
<|endoftext|>


<commit_before>import elfi
import numpy as np

from functools import partial

from elfi.core import simulator_operation
from elfi.core import Node
from elfi.core import ObservedMixin

from mocks import MockSimulator, MockSequentialSimulator
from mocks import MockSummary, MockSequentialSummary
from mocks import MockDiscrepancy, MockSequentialDiscrepancy

class TestSimulatorOperation():

    def test_normal_use(self):
        ret1 = np.array([5])
        ret2 = np.array([6])
        mock = MockSimulator([ret1, ret2])
        prng = np.random.RandomState(1234)
        input_dict = {
                "n": 2,
                "data": [np.atleast_2d([[1], [2]]),
                         np.atleast_2d([[3], [4]])],
                "random_state": prng.get_state()
                }
        output_dict = simulator_operation(mock, True, input_dict)
        prng.rand()
        print(output_dict)
        assert mock.n_calls == 1
        assert output_dict["n"] == 2
        assert np.array_equal(output_dict["data"], np.vstack((ret1, ret2)))
        new_state = prng.get_state()
        assert output_dict["random_state"][0] == new_state[0]
        assert np.array_equal(output_dict["random_state"][1], new_state[1])
        assert output_dict["random_state"][2] == new_state[2]
        assert output_dict["random_state"][3] == new_state[3]
        assert output_dict["random_state"][4] == new_state[4]

class Test_vectorization():
    """Test operation vectorization operations 'as_vectrized_*'
    """

    def test_as_vectorized_simulator(self):
        ret1 = np.array([5])
        ret2 = np.array([6])
        mock_seq = elfi.core.as_vectorized_simulator(MockSequentialSimulator([ret1, ret2]))
        mock_vec = MockSimulator([ret1, ret2])
        input_data = [np.atleast_2d([[1], [2]]), np.atleast_2d([[3], [4]])]
        output_seq = mock_seq(*input_data, n_sim=2)
        output_vec = mock_vec(*input_data, n_sim=2)
        assert np.array_equal(output_seq, output_vec)

    def test_as_vectorized_summary(self):
        ret1 = np.array([5])
        ret2 = np.array([6])
        mock_seq = elfi.core.as_vectorized_summary(MockSequentialSummary([ret1, ret2]))
        mock_vec = MockSummary([ret1, ret2])
        input_data = [np.atleast_2d([[1], [2]]), np.atleast_2d([[3], [4]])]
        output_seq = mock_seq(*input_data)
        output_vec = mock_vec(*input_data)
        assert np.array_equal(output_seq, output_vec)

    def test_as_vectorized_discrepancy(self):
        ret1 = np.array([5])
        ret2 = np.array([6])
        mock_seq = elfi.core.as_vectorized_discrepancy(MockSequentialDiscrepancy([ret1, ret2]))
        mock_vec = MockDiscrepancy([ret1, ret2])
        x = (np.atleast_2d([[1], [2]]), np.atleast_2d([[3], [4]]))
        y = (np.atleast_2d([[5]]), np.atleast_2d([[6]]))
        output_seq = mock_seq(x, y)
        output_vec = mock_vec(x, y)
        assert np.array_equal(output_seq, output_vec)

def test_node_data_sub_slicing():
    mu = elfi.Prior('mu', 'uniform', 0, 4)
    ar1 = mu.acquire(10).compute()
    ar2 = mu.acquire(5).compute()
    assert np.array_equal(ar1[0:5], ar2)

    ar3 = mu.acquire(20).compute()
    assert np.array_equal(ar1, ar3[0:10])

def test_generate_vs_acquire():
    mu = elfi.Prior('mu', 'uniform', 0, 4)
    ar1 = mu.acquire(10).compute()
    ar2 = mu.generate(5).compute()
    ar12 = mu.acquire(15).compute()
    assert np.array_equal(np.vstack((ar1, ar2)), ar12)

class TestNode():
    def test_construction1(self):
        a = Node('a')
        assert a.name == 'a'
        assert a.parents == []
        assert a.children == []
        assert a.is_root()
        assert a.is_leaf()

    def test_construction(self):
        a = Node('a')
        b = Node('b')
        c = Node('c', b)
        d = Node('d', a, c)
        e = Node('e', d)
        f = Node('f', d, c)
        g = Node('g', e, f)
        assert b.is_root()
        assert c.parents == [b]
        assert d.parents == [a, c]
        assert g.parents == [e, f]
        assert b.children == [c]
        assert c.children == [d, f]
        assert g.is_leaf()

    def test_construction_add(self):
        b = Node('b')
        c = Node('c')
        d = Node('d')
        c.add_parent(b)
        assert c.parents == [b]
        assert b.children == [c]
        b._add_child(d)
        assert b.children == [c, d]
        assert d.parents == []  # _add_child doesn't call add_parent

    def test_construction_unique(self):
        b = Node('b')
        c = Node('c', b)
        c.add_parent(b)
        assert c.parents == [b]
        assert b.children == [c]

    def test_construction_add_index(self):
        b = Node('b')
        c = Node('c', b)
        d = Node('d')
        e = Node('e')
        c.add_parent(d, index=0, index_child=0)
        assert c.parents == [d, b]
        assert b.children == [c]
        assert d.children == [c]
        b._add_child(e, index=0)
        assert b.children == [e, c]
        assert e.parents == []  # _add_child doesn't call add_parent

    def test_construction_index_out_of_bounds(self):
        b = Node('b')
        c = Node('c')
        try:
            c.add_parent(b, index=13)
            b._add_child(c, index=13)
            assert False
        except ValueError:
            assert True

    def test_construction_acyclic(self):
        a = Node('a')
        b = Node('b', a)
        c = Node('c', b)
        try:
            a.add_parent(c)
            assert False
        except ValueError:
            assert True

    def test_family_tree(self):
        a = Node('a')
        b = Node('b')
        c = Node('c', b)
        d = Node('d', a, c)
        e = Node('e', d)
        f = Node('f', d, c)
        g = Node('g', e, f)
        assert a.descendants == [d, e, f, g]
        assert e.descendants == [g]
        assert g.descendants == []
        assert g.ancestors == [e, f, d, a, c, b]
        assert c.ancestors == [b]
        assert a.ancestors == []
        assert d.neighbours == [e, f, a, c]

    def test_component(self):
        a = Node('a')
        b = Node('b')
        c = Node('c', b)
        d = Node('d', a, c)
        e = Node('e', d)
        f = Node('f', d, c)
        g = Node('g', e, f)
        assert c.component == [c, b, d, f, e, g]
        assert g.component == [g, e, f, d, a, c, b]

    def test_remove_parent(self):
        a = Node('a')
        b = Node('b', a)
        c = Node('c', b)
        c.remove_parent(0)
        assert c.parents == []
        assert b.children == []
        b.remove_parent(a)
        assert b.parents == []
        assert a.children == []

    def test_remove(self):
        a = Node('a')
        b = Node('b')
        c = Node('c', a, b)
        d = Node('d', c)
        e = Node('e', c)
        c.remove(keep_parents=False, keep_children=False)
        assert c.children == []
        assert c.parents == []
        assert b.children == []
        assert e.parents == []

    def test_change_to(self):
        a = Node('a')
        b = Node('b')
        c = Node('c', a, b)
        d = Node('d', c)
        e = Node('e', c)
        f = Node('f')
        c_new = Node('c_new', a, f)
        c = c.change_to(c_new, transfer_parents=False, transfer_children=True)
        assert c.parents == [a, f]
        assert c.children == [d, e]
        c_new2 = Node('c_new2', a)
        c = c.change_to(c_new2, transfer_parents=True, transfer_children=False)
        assert c.parents == [a, f]
        assert c.children == []

    def test_none_parent(self):
        a = Node('a', None)
        assert a.parents == []

class TestObservedMixin():

    def test_numpy_array_observation(self):
        np.random.seed(21273632)
        for i in range(20):
            n_dims = np.random.randint(1,5)
            dims = [np.random.randint(1,5) for i in range(n_dims)]
            dims[0] = max(2, dims[0])
            dims = tuple(dims)
            obs = np.zeros(dims)
            o = ObservedMixin("o", lambda x: x, None, observed=obs)
            assert o.observed.shape == (1, ) + dims
            assert o.observed.dtype == obs.dtype
            np.testing.assert_array_equal(o.observed[0], obs)

    def test_list_observation(self):
        # if list or tuple is passed as observation, it will be auto-converted to numpy array
        class MockClass():
            pass
        for obs, dtype, shape in [
                ([False], bool, (1,)),
                ((1, 2, 3), int, (3,)),
                ([1.2, 2.3], float, (2,)),
                ([(1,2), (2.3,3.4)], float, (2,2)),
                ([set((1,2))], object, (1,)),
                ([{1:2}, {2:4}], object, (2,)),
                ([1, MockClass()], object, (2,)),
                           ]:
            o = ObservedMixin("o", lambda x: x, None, observed=obs)
            assert o.observed.shape == (1,) + shape
            assert o.observed.dtype == dtype
            np.testing.assert_array_equal(o.observed[0], obs)

    def test_value_observation(self):
        # raw types are converted to at least 2d arrays
        class MockClass():
            pass
        for obs, dtype in [
                (123, int),
                (123.0, float),
                (False, bool),
                ({3:4}, object),
                (set((5,6)), object),
                (MockClass(), object),
                ("string", object),
                    ]:
            o = ObservedMixin("o", lambda x: x, None, observed=obs)
            assert o.observed.shape == (1,1)
            assert o.observed.dtype == dtype
            assert o.observed[0] == obs

class TestNumpyInterfaces():

    def test_simulator_summary_input_dimensions(self):
        np.random.seed(438763)
        for i in range(20):
            # dimensions
            n_samples = np.random.randint(1,5)
            n_in_dims = np.random.randint(1,5)
            in_dims = [np.random.randint(1,5) for i in range(n_in_dims)]
            in_dims[0] = max(2, in_dims[0])
            in_dims = tuple(in_dims)
            n_out_dims = np.random.randint(1,5)
            out_dims = tuple([np.random.randint(1,5) for i in range(n_out_dims)])
            # data
            ret = np.zeros((n_samples, ) + in_dims)
            obs = ret[0]
            # summary
            def mock_summary(x):
                exp_in_dims = in_dims
                if len(exp_in_dims) == 0:
                    exp_in_dims = (1,)
                if x.shape == (n_samples, ) + exp_in_dims:
                    # simulation data
                    return np.zeros((n_samples,) + out_dims)
                elif x.shape == (1,) + exp_in_dims:
                    # observation data
                    return np.zeros((1,) + out_dims)
                assert False
            # model
            mock = MockSimulator(ret)
            si = elfi.Simulator("si", mock, None, observed=obs)
            su = elfi.Summary("su", mock_summary, si)
            res = su.generate(n_samples).compute()
            exp_out_dims = out_dims
            if len(exp_out_dims) == 0:
                exp_out_dims = (1,)
            assert res.shape == (n_samples,) + exp_out_dims

    def test_summary_discrepancy_input_dimensions(self):
        np.random.seed(23876123)
        for i in range(20):
            # dimensions
            n_samples = np.random.randint(1,5)
            n_sum = np.random.randint(1,5)
            n_dims = [np.random.randint(1,5) for i in range(n_sum)]
            dims = [tuple([np.random.randint(1,5) for j in range(n_dims[i])]) for i in range(n_sum)]
            # data
            ret = np.zeros((n_samples, 1))
            obs = ret[0]
            # summary
            def mock_summary(i, x):
                return np.zeros((x.shape[0], ) + dims[i])
            # discrepancy
            def mock_discrepancy(x, y):
                assert len(x) == len(y) == n_sum
                for i in range(n_sum):
                    exp_dims = dims[i]
                    if len(exp_dims) == 0:
                        exp_dims = (1,)
                    assert y[i].shape == (1,) + exp_dims
                    assert x[i].shape == (n_samples,) + exp_dims
                return np.zeros((n_samples, 1))
            # model
            mock = MockSimulator(ret)
            si = elfi.Simulator("si", mock, None, observed=obs)
            su = [elfi.Summary("su{}".format(i), partial(mock_summary, i), si) for i in range(n_sum)]
            di = elfi.Discrepancy("di", mock_discrepancy, *su)
            res = di.generate(n_samples).compute()
            assert res.shape == (n_samples, 1)

<commit_msg>Add tests for vectorization annotations<commit_after>import elfi
import numpy as np

from functools import partial

from elfi.core import simulator_operation
from elfi.core import Node
from elfi.core import ObservedMixin

from mocks import MockSimulator, MockSequentialSimulator
from mocks import MockSummary, MockSequentialSummary
from mocks import MockDiscrepancy, MockSequentialDiscrepancy

class TestSimulatorOperation():

    def test_normal_use(self):
        ret1 = np.array([5])
        ret2 = np.array([6])
        mock = MockSimulator([ret1, ret2])
        prng = np.random.RandomState(1234)
        input_dict = {
                "n": 2,
                "data": [np.atleast_2d([[1], [2]]),
                         np.atleast_2d([[3], [4]])],
                "random_state": prng.get_state()
                }
        output_dict = simulator_operation(mock, True, input_dict)
        prng.rand()
        print(output_dict)
        assert mock.n_calls == 1
        assert output_dict["n"] == 2
        assert np.array_equal(output_dict["data"], np.vstack((ret1, ret2)))
        new_state = prng.get_state()
        assert output_dict["random_state"][0] == new_state[0]
        assert np.array_equal(output_dict["random_state"][1], new_state[1])
        assert output_dict["random_state"][2] == new_state[2]
        assert output_dict["random_state"][3] == new_state[3]
        assert output_dict["random_state"][4] == new_state[4]

class Test_vectorization():
    """Test operation vectorization operations 'as_vectorized_*'
    """

    @staticmethod
    @elfi.core.as_vectorized_simulator
    def mock_simulator1(*data, prng=None):
        return np.array((1,))

    @staticmethod
    def mock_simulator2(*data, n_sim=1, prng=None):
        return np.ones((n_sim, 1))

    @staticmethod
    @elfi.core.as_vectorized_summary
    def mock_summary1(*data):
        return np.ones((1,))

    @staticmethod
    def mock_summary2(*data):
        n_sim = data[0].shape[0]
        return np.ones((n_sim, 1))

    @staticmethod
    @elfi.core.as_vectorized_discrepancy
    def mock_discrepancy1(x, y):
        return np.ones((1,))

    @staticmethod
    def mock_discrepancy2(x, y):
        n_sim = x[0].shape[0]
        return np.ones((n_sim, 1))

    def test_as_vectorized_simulator(self):
        ret1 = np.array([5])
        ret2 = np.array([6])
        mock_seq = elfi.core.as_vectorized_simulator(MockSequentialSimulator([ret1, ret2]))
        mock_vec = MockSimulator([ret1, ret2])
        input_data = [np.atleast_2d([[1], [2]]), np.atleast_2d([[3], [4]])]
        output_seq = mock_seq(*input_data, n_sim=2)
        output_vec = mock_vec(*input_data, n_sim=2)
        assert np.array_equal(output_seq, output_vec)

    def test_annotated_simulator(self):
        input_data = [np.atleast_2d([[1], [2]]), np.atleast_2d([[3], [4]])]
        output1 = self.mock_simulator1(*input_data, n_sim=2)
        output2 = self.mock_simulator2(*input_data, n_sim=2)
        assert np.array_equal(output1, output2)

    def test_as_vectorized_summary(self):
        ret1 = np.array([5])
        ret2 = np.array([6])
        mock_seq = elfi.core.as_vectorized_summary(MockSequentialSummary([ret1, ret2]))
        mock_vec = MockSummary([ret1, ret2])
        input_data = [np.atleast_2d([[1], [2]]), np.atleast_2d([[3], [4]])]
        output_seq = mock_seq(*input_data)
        output_vec = mock_vec(*input_data)
        assert np.array_equal(output_seq, output_vec)

    def test_annotated_summary(self):
        input_data = [np.atleast_2d([[1], [2]]), np.atleast_2d([[3], [4]])]
        output1 = self.mock_summary1(*input_data)
        output2 = self.mock_summary2(*input_data)
        assert np.array_equal(output1, output2)

    def test_as_vectorized_discrepancy(self):
        ret1 = np.array([5])
        ret2 = np.array([6])
        mock_seq = elfi.core.as_vectorized_discrepancy(MockSequentialDiscrepancy([ret1, ret2]))
        mock_vec = MockDiscrepancy([ret1, ret2])
        x = (np.atleast_2d([[1], [2]]), np.atleast_2d([[3], [4]]))
        y = (np.atleast_2d([[5]]), np.atleast_2d([[6]]))
        output_seq = mock_seq(x, y)
        output_vec = mock_vec(x, y)
        assert np.array_equal(output_seq, output_vec)

    def test_annotated_discrepancy(self):
        x = (np.atleast_2d([[1], [2]]), np.atleast_2d([[3], [4]]))
        y = (np.atleast_2d([[5]]), np.atleast_2d([[6]]))
        output1 = self.mock_discrepancy1(x, y)
        output2 = self.mock_discrepancy2(x, y)
        assert np.array_equal(output1, output2)

def test_node_data_sub_slicing():
    mu = elfi.Prior('mu', 'uniform', 0, 4)
    ar1 = mu.acquire(10).compute()
    ar2 = mu.acquire(5).compute()
    assert np.array_equal(ar1[0:5], ar2)

    ar3 = mu.acquire(20).compute()
    assert np.array_equal(ar1, ar3[0:10])

def test_generate_vs_acquire():
    mu = elfi.Prior('mu', 'uniform', 0, 4)
    ar1 = mu.acquire(10).compute()
    ar2 = mu.generate(5).compute()
    ar12 = mu.acquire(15).compute()
    assert np.array_equal(np.vstack((ar1, ar2)), ar12)

class TestNode():
    def test_construction1(self):
        a = Node('a')
        assert a.name == 'a'
        assert a.parents == []
        assert a.children == []
        assert a.is_root()
        assert a.is_leaf()

    def test_construction(self):
        a = Node('a')
        b = Node('b')
        c = Node('c', b)
        d = Node('d', a, c)
        e = Node('e', d)
        f = Node('f', d, c)
        g = Node('g', e, f)
        assert b.is_root()
        assert c.parents == [b]
        assert d.parents == [a, c]
        assert g.parents == [e, f]
        assert b.children == [c]
        assert c.children == [d, f]
        assert g.is_leaf()

    def test_construction_add(self):
        b = Node('b')
        c = Node('c')
        d = Node('d')
        c.add_parent(b)
        assert c.parents == [b]
        assert b.children == [c]
        b._add_child(d)
        assert b.children == [c, d]
        assert d.parents == []  # _add_child doesn't call add_parent

    def test_construction_unique(self):
        b = Node('b')
        c = Node('c', b)
        c.add_parent(b)
        assert c.parents == [b]
        assert b.children == [c]

    def test_construction_add_index(self):
        b = Node('b')
        c = Node('c', b)
        d = Node('d')
        e = Node('e')
        c.add_parent(d, index=0, index_child=0)
        assert c.parents == [d, b]
        assert b.children == [c]
        assert d.children == [c]
        b._add_child(e, index=0)
        assert b.children == [e, c]
        assert e.parents == []  # _add_child doesn't call add_parent

    def test_construction_index_out_of_bounds(self):
        b = Node('b')
        c = Node('c')
        try:
            c.add_parent(b, index=13)
            b._add_child(c, index=13)
            assert False
        except ValueError:
            assert True

    def test_construction_acyclic(self):
        a = Node('a')
        b = Node('b', a)
        c = Node('c', b)
        try:
            a.add_parent(c)
            assert False
        except ValueError:
            assert True

    def test_family_tree(self):
        a = Node('a')
        b = Node('b')
        c = Node('c', b)
        d = Node('d', a, c)
        e = Node('e', d)
        f = Node('f', d, c)
        g = Node('g', e, f)
        assert a.descendants == [d, e, f, g]
        assert e.descendants == [g]
        assert g.descendants == []
        assert g.ancestors == [e, f, d, a, c, b]
        assert c.ancestors == [b]
        assert a.ancestors == []
        assert d.neighbours == [e, f, a, c]

    def test_component(self):
        a = Node('a')
        b = Node('b')
        c = Node('c', b)
        d = Node('d', a, c)
        e = Node('e', d)
        f = Node('f', d, c)
        g = Node('g', e, f)
        assert c.component == [c, b, d, f, e, g]
        assert g.component == [g, e, f, d, a, c, b]

    def test_remove_parent(self):
        a = Node('a')
        b = Node('b', a)
        c = Node('c', b)
        c.remove_parent(0)
        assert c.parents == []
        assert b.children == []
        b.remove_parent(a)
        assert b.parents == []
        assert a.children == []

    def test_remove(self):
        a = Node('a')
        b = Node('b')
        c = Node('c', a, b)
        d = Node('d', c)
        e = Node('e', c)
        c.remove(keep_parents=False, keep_children=False)
        assert c.children == []
        assert c.parents == []
        assert b.children == []
        assert e.parents == []

    def test_change_to(self):
        a = Node('a')
        b = Node('b')
        c = Node('c', a, b)
        d = Node('d', c)
        e = Node('e', c)
        f = Node('f')
        c_new = Node('c_new', a, f)
        c = c.change_to(c_new, transfer_parents=False, transfer_children=True)
        assert c.parents == [a, f]
        assert c.children == [d, e]
        c_new2 = Node('c_new2', a)
        c = c.change_to(c_new2, transfer_parents=True, transfer_children=False)
        assert c.parents == [a, f]
        assert c.children == []

    def test_none_parent(self):
        a = Node('a', None)
        assert a.parents == []

class TestObservedMixin():

    def test_numpy_array_observation(self):
        np.random.seed(21273632)
        for i in range(20):
            n_dims = np.random.randint(1,5)
            dims = [np.random.randint(1,5) for i in range(n_dims)]
            dims[0] = max(2, dims[0])
            dims = tuple(dims)
            obs = np.zeros(dims)
            o = ObservedMixin("o", lambda x: x, None, observed=obs)
            assert o.observed.shape == (1, ) + dims
            assert o.observed.dtype == obs.dtype
            np.testing.assert_array_equal(o.observed[0], obs)

    def test_list_observation(self):
        # if list or tuple is passed as observation, it will be auto-converted to numpy array
        class MockClass():
            pass
        for obs, dtype, shape in [
                ([False], bool, (1,)),
                ((1, 2, 3), int, (3,)),
                ([1.2, 2.3], float, (2,)),
                ([(1,2), (2.3,3.4)], float, (2,2)),
                ([set((1,2))], object, (1,)),
                ([{1:2}, {2:4}], object, (2,)),
                ([1, MockClass()], object, (2,)),
                           ]:
            o = ObservedMixin("o", lambda x: x, None, observed=obs)
            assert o.observed.shape == (1,) + shape
            assert o.observed.dtype == dtype
            np.testing.assert_array_equal(o.observed[0], obs)

    def test_value_observation(self):
        # raw types are converted to at least 2d arrays
        class MockClass():
            pass
        for obs, dtype in [
                (123, int),
                (123.0, float),
                (False, bool),
                ({3:4}, object),
                (set((5,6)), object),
                (MockClass(), object),
                ("string", object),
                    ]:
            o = ObservedMixin("o", lambda x: x, None, observed=obs)
            assert o.observed.shape == (1,1)
            assert o.observed.dtype == dtype
            assert o.observed[0] == obs

class TestNumpyInterfaces():

    def test_simulator_summary_input_dimensions(self):
        np.random.seed(438763)
        for i in range(20):
            # dimensions
            n_samples = np.random.randint(1,5)
            n_in_dims = np.random.randint(1,5)
            in_dims = [np.random.randint(1,5) for i in range(n_in_dims)]
            in_dims[0] = max(2, in_dims[0])
            in_dims = tuple(in_dims)
            n_out_dims = np.random.randint(1,5)
            out_dims = tuple([np.random.randint(1,5) for i in range(n_out_dims)])
            # data
            ret = np.zeros((n_samples, ) + in_dims)
            obs = ret[0]
            # summary
            def mock_summary(x):
                exp_in_dims = in_dims
                if len(exp_in_dims) == 0:
                    exp_in_dims = (1,)
                if x.shape == (n_samples, ) + exp_in_dims:
                    # simulation data
                    return np.zeros((n_samples,) + out_dims)
                elif x.shape == (1,) + exp_in_dims:
                    # observation data
                    return np.zeros((1,) + out_dims)
                assert False
            # model
            mock = MockSimulator(ret)
            si = elfi.Simulator("si", mock, None, observed=obs)
            su = elfi.Summary("su", mock_summary, si)
            res = su.generate(n_samples).compute()
            exp_out_dims = out_dims
            if len(exp_out_dims) == 0:
                exp_out_dims = (1,)
            assert res.shape == (n_samples,) + exp_out_dims

    def test_summary_discrepancy_input_dimensions(self):
        np.random.seed(23876123)
        for i in range(20):
            # dimensions
            n_samples = np.random.randint(1,5)
            n_sum = np.random.randint(1,5)
            n_dims = [np.random.randint(1,5) for i in range(n_sum)]
            dims = [tuple([np.random.randint(1,5) for j in range(n_dims[i])]) for i in range(n_sum)]
            # data
            ret = np.zeros((n_samples, 1))
            obs = ret[0]
            # summary
            def mock_summary(i, x):
                return np.zeros((x.shape[0], ) + dims[i])
            # discrepancy
            def mock_discrepancy(x, y):
                assert len(x) == len(y) == n_sum
                for i in range(n_sum):
                    exp_dims = dims[i]
                    if len(exp_dims) == 0:
                        exp_dims = (1,)
                    assert y[i].shape == (1,) + exp_dims
                    assert x[i].shape == (n_samples,) + exp_dims
                return np.zeros((n_samples, 1))
            # model
            mock = MockSimulator(ret)
            si = elfi.Simulator("si", mock, None, observed=obs)
            su = [elfi.Summary("su{}".format(i), partial(mock_summary, i), si) for i in range(n_sum)]
            di = elfi.Discrepancy("di", mock_discrepancy, *su)
            res = di.generate(n_samples).compute()
            assert res.shape == (n_samples, 1)

<|endoftext|>


<commit_before>import asyncio
import aiomas
import random
import re
import math
import logging
import json
import traceback

import sqlalchemy

import common.utils
import common.rpc
from common import utils
from common.config import config
from common import game_data
from common import twitch
from lrrbot import storage
import lrrbot.docstring

log = logging.getLogger('serverevents')

class Server(common.rpc.Server):
	router = aiomas.rpc.Service(['cardviewer', 'link_spam', 'spam', 'static'])

	def __init__(self, lrrbot, loop):
		super().__init__()
		self.lrrbot = lrrbot
		self.loop = loop

		self.cardviewer = None
		self.link_spam = None
		self.spam = None
		self.static = None

	@aiomas.expose
	def get_game_id(self):
		return self.lrrbot.get_game_id()

	@aiomas.expose
	def get_data(self, key):
		if not isinstance(key, (list, tuple)):
			key = [key]
		node = storage.data
		for subkey in key:
			node = node.get(subkey, {})
		return node

	@aiomas.expose
	def set_data(self, key, value):
		if not isinstance(key, (list, tuple)):
			key = [key]
		log.info("Setting storage %s to %r" % (user, '.'.join(key), value))
		# if key is, eg, ["a", "b", "c"]
		# then we want to effectively do:
		# storage.data["a"]["b"]["c"] = value
		# But in case one of those intermediate dicts doesn't exist:
		# storage.data.setdefault("a", {}).setdefault("b", {})["c"] = value
		node = storage.data
		for subkey in key[:-1]:
			node = node.setdefault(subkey, {})
		node[key[-1]] = value
		storage.save()

	@aiomas.expose
	def get_commands(self):
		ret = []
		for command in self.lrrbot.commands.commands.values():
			doc = lrrbot.docstring.parse_docstring(command['func'].__doc__)
			for cmd in doc.walk():
				if cmd.get_content_maintype() == "multipart":
					continue
				if cmd.get_all("command") is None:
					continue
				ret += [{
					"aliases": cmd.get_all("command"),
					"mod-only": cmd.get("mod-only") == "true",
					"sub-only": cmd.get("sub-only") == "true",
					"public-only": cmd.get("public-only") == "true",
					"throttled": (int(cmd.get("throttle-count", 1)), int(cmd.get("throttled"))) if "throttled" in cmd else None,
					"literal-response": cmd.get("literal-response") == "true",
					"section": cmd.get("section"),
					"description": cmd.get_payload(),
				}]
		return ret

	@aiomas.expose
	def get_header_info(self):
		live = twitch.is_stream_live()
		game_id = self.lrrbot.get_game_id()

		data = {
			"is_live": live,
			"channel": config['channel'],
		}

		if live and game_id is not None:
			data['current_game'] = {
				"id": game_id,
				"is_override": self.lrrbot.game_override is not None,
			}
			data['current_show'] = {
				"id": self.lrrbot.get_show_id(),
				"is_override": self.lrrbot.show_override is not None,
			}

		if 'advice' in storage.data['responses']:
			data['advice'] = random.choice(storage.data['responses']['advice']['response'])

		return data

	@aiomas.expose
	def set_show(self, show):
		import lrrbot.commands
		lrrbot.commands.show.set_show(self.lrrbot, show)

	@aiomas.expose
	def get_show_id(self):
		return self.lrrbot.get_show_id()

	@aiomas.expose
	def get_tweet(self):
		tweet = self.generate_tweet()
		while len(tweet) > 280:
			tweet = self.generate_tweet()
		return tweet

	def generate_tweet(self):
		import lrrbot.commands
		mode = utils.weighted_choice([(0, 10), (1, 4)])
		if mode == 0: # get random !advice
			return random.choice(storage.data['responses']['advice']['response'])
		elif mode == 1: # get a random !quote
			quotes = self.lrrbot.metadata.tables["quotes"]
			with self.lrrbot.engine.begin() as conn:
				query = sqlalchemy.select([quotes.c.quote, quotes.c.attrib_name, quotes.c.context]).where(~quotes.c.deleted)
				row = common.utils.pick_random_elements(conn.execute(query), 1)[0]
			if row is None:
				return None

			quote, name, context = row

			quote_msg = "\"{quote}\"".format(quote=quote)
			if name:
				quote_msg += " —{name}".format(name=name)
				if context:
					quote_msg += ", {context}".format(context=context)
			return quote_msg

	@aiomas.expose
	def disconnect_from_chat(self):
		self.lrrbot.disconnect()

	@aiomas.expose
	def get_polls(self):
		from lrrbot.commands import strawpoll
		return strawpoll.get_polls_data(self.lrrbot)
<commit_msg>Add a couple of other things that could come out in tweets<commit_after>import asyncio
import aiomas
import random
import re
import math
import logging
import json
import traceback

import sqlalchemy

import common.utils
import common.rpc
from common import utils
from common.config import config
from common import game_data
from common import twitch
from lrrbot import storage
import lrrbot.docstring

log = logging.getLogger('serverevents')

class Server(common.rpc.Server):
	router = aiomas.rpc.Service(['cardviewer', 'link_spam', 'spam', 'static'])

	def __init__(self, lrrbot, loop):
		super().__init__()
		self.lrrbot = lrrbot
		self.loop = loop

		self.cardviewer = None
		self.link_spam = None
		self.spam = None
		self.static = None

	@aiomas.expose
	def get_game_id(self):
		return self.lrrbot.get_game_id()

	@aiomas.expose
	def get_data(self, key):
		if not isinstance(key, (list, tuple)):
			key = [key]
		node = storage.data
		for subkey in key:
			node = node.get(subkey, {})
		return node

	@aiomas.expose
	def set_data(self, key, value):
		if not isinstance(key, (list, tuple)):
			key = [key]
		log.info("Setting storage %s to %r" % (user, '.'.join(key), value))
		# if key is, eg, ["a", "b", "c"]
		# then we want to effectively do:
		# storage.data["a"]["b"]["c"] = value
		# But in case one of those intermediate dicts doesn't exist:
		# storage.data.setdefault("a", {}).setdefault("b", {})["c"] = value
		node = storage.data
		for subkey in key[:-1]:
			node = node.setdefault(subkey, {})
		node[key[-1]] = value
		storage.save()

	@aiomas.expose
	def get_commands(self):
		ret = []
		for command in self.lrrbot.commands.commands.values():
			doc = lrrbot.docstring.parse_docstring(command['func'].__doc__)
			for cmd in doc.walk():
				if cmd.get_content_maintype() == "multipart":
					continue
				if cmd.get_all("command") is None:
					continue
				ret += [{
					"aliases": cmd.get_all("command"),
					"mod-only": cmd.get("mod-only") == "true",
					"sub-only": cmd.get("sub-only") == "true",
					"public-only": cmd.get("public-only") == "true",
					"throttled": (int(cmd.get("throttle-count", 1)), int(cmd.get("throttled"))) if "throttled" in cmd else None,
					"literal-response": cmd.get("literal-response") == "true",
					"section": cmd.get("section"),
					"description": cmd.get_payload(),
				}]
		return ret

	@aiomas.expose
	def get_header_info(self):
		live = twitch.is_stream_live()
		game_id = self.lrrbot.get_game_id()

		data = {
			"is_live": live,
			"channel": config['channel'],
		}

		if live and game_id is not None:
			data['current_game'] = {
				"id": game_id,
				"is_override": self.lrrbot.game_override is not None,
			}
			data['current_show'] = {
				"id": self.lrrbot.get_show_id(),
				"is_override": self.lrrbot.show_override is not None,
			}

		if 'advice' in storage.data['responses']:
			data['advice'] = random.choice(storage.data['responses']['advice']['response'])

		return data

	@aiomas.expose
	def set_show(self, show):
		import lrrbot.commands
		lrrbot.commands.show.set_show(self.lrrbot, show)

	@aiomas.expose
	def get_show_id(self):
		return self.lrrbot.get_show_id()

	@aiomas.expose
	def get_tweet(self):
		tweet = self.generate_tweet()
		while len(tweet) > 280:
			tweet = self.generate_tweet()
		return tweet

	def generate_tweet(self):
		import lrrbot.commands
		QUOTE = 1
		ADVICE = 2
		SECRET = 3
		SMASH = 4
		options = [(QUOTE, 5)]
		if 'advice' in storage.data['responses']:
			options.append((ADVICE, 10))
		if 'secret' in storage.data['responses']:
			options.append((SECRET, 2))
		if 'smash' in storage.data['responses']:
			options.append((SMASH, 2))
		mode = utils.weighted_choice(options)
		if mode == ADVICE:
			return random.choice(storage.data['responses']['advice']['response'])
		elif mode == SECRET:
			return random.choice(storage.data['responses']['secret']['response'])
		elif mode == SMASH:
			return random.choice(storage.data['responses']['smash']['response'])
		elif mode == QUOTE:
			quotes = self.lrrbot.metadata.tables["quotes"]
			with self.lrrbot.engine.begin() as conn:
				query = sqlalchemy.select([quotes.c.quote, quotes.c.attrib_name, quotes.c.context]).where(~quotes.c.deleted)
				row = common.utils.pick_random_elements(conn.execute(query), 1)[0]
			if row is None:
				return None

			quote, name, context = row

			quote_msg = f'"{quote}"'
			if name:
				quote_msg += f" —{name}"
				if context:
					quote_msg += f", {context}"
			return quote_msg

	@aiomas.expose
	def disconnect_from_chat(self):
		self.lrrbot.disconnect()

	@aiomas.expose
	def get_polls(self):
		from lrrbot.commands import strawpoll
		return strawpoll.get_polls_data(self.lrrbot)
<|endoftext|>


<commit_before>import functools
import httplib2
import logging
import os

import mox
import nose

def skip(fn):
    def testNothing(self):
        raise nose.SkipTest('skip this test')
    functools.update_wrapper(testNothing, fn)
    return testNothing

def are_automated():
    return bool(os.getenv('AUTOMATED_TESTING'))

def skip_if_automated(fn):
    if are_automated():
        return skip(fn)
    return fn

def todo(fn):
    def testReverse(*args, **kwargs):
        try:
            fn(*args, **kwargs)
        except:
            pass
        else:
            raise AssertionError('test %s unexpectedly succeeded' % fn.__name__)
    functools.update_wrapper(testReverse, fn)
    return testReverse

class MockedHttp(object):
    def __init__(self, req, resp_or_content):
        self.mock = mox.MockObject(httplib2.Http)

        if not isinstance(req, dict):
            req = dict(uri=req)

        resp, content = self.make_response(resp_or_content, req['uri'])
        self.mock.request(**req).AndReturn((resp, content))

    def make_response(self, response, url):
        default_response = {
            'status':           200,
            'etag':             '7',
            'content-type':     'application/json',
            'content-location': url,
        }

        if isinstance(response, dict):
            if 'content' in response:
                content = response['content']
                del response['content']
            else:
                content = ''

            status = response.get('status', 200)
            if 200 <= status < 300:
                response_info = dict(default_response)
                response_info.update(response)
            else:
                # Homg all bets are off!! Use specified headers only.
                response_info = dict(response)
        else:
            response_info = dict(default_response)
            content = response

        return httplib2.Response(response_info), content

    def __enter__(self):
        mox.Replay(self.mock)
        return self.mock

    def __exit__(self, *exc_info):
        # don't really care about the mock if there was an exception
        if None in exc_info:
            mox.Verify(self.mock)

def log():
    import sys
    logging.basicConfig(level=logging.DEBUG, stream=sys.stderr, format="%(asctime)s %(levelname)s %(message)s")
<commit_msg>Don't use functools; it's not available in 2.4<commit_after>import httplib2
import logging
import os

import mox
import nose

def update_wrapper(wrapper, original):
    for field in ('__module__', '__name__', '__doc__'):
        setattr(wrapper, field, getattr(original, field))

def skip(fn):
    def testNothing(self):
        raise nose.SkipTest('skip this test')
    update_wrapper(testNothing, fn)
    return testNothing

def are_automated():
    return bool(os.getenv('AUTOMATED_TESTING'))

def skip_if_automated(fn):
    if are_automated():
        return skip(fn)
    return fn

def todo(fn):
    def testReverse(*args, **kwargs):
        try:
            fn(*args, **kwargs)
        except:
            pass
        else:
            raise AssertionError('test %s unexpectedly succeeded' % fn.__name__)
    update_wrapper(testReverse, fn)
    return testReverse

class MockedHttp(object):
    def __init__(self, req, resp_or_content):
        self.mock = mox.MockObject(httplib2.Http)

        if not isinstance(req, dict):
            req = dict(uri=req)

        resp, content = self.make_response(resp_or_content, req['uri'])
        self.mock.request(**req).AndReturn((resp, content))

    def make_response(self, response, url):
        default_response = {
            'status':           200,
            'etag':             '7',
            'content-type':     'application/json',
            'content-location': url,
        }

        if isinstance(response, dict):
            if 'content' in response:
                content = response['content']
                del response['content']
            else:
                content = ''

            status = response.get('status', 200)
            if 200 <= status < 300:
                response_info = dict(default_response)
                response_info.update(response)
            else:
                # Homg all bets are off!! Use specified headers only.
                response_info = dict(response)
        else:
            response_info = dict(default_response)
            content = response

        return httplib2.Response(response_info), content

    def __enter__(self):
        mox.Replay(self.mock)
        return self.mock

    def __exit__(self, *exc_info):
        # don't really care about the mock if there was an exception
        if None in exc_info:
            mox.Verify(self.mock)

def log():
    import sys
    logging.basicConfig(level=logging.DEBUG, stream=sys.stderr, format="%(asctime)s %(levelname)s %(message)s")
<|endoftext|>


<commit_before>#!/usr/bin/python

#----------------------------------------------------------------------
# Copyright (c) 2011 Raytheon BBN Technologies
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and/or hardware specification (the "Work") to
# deal in the Work without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Work, and to permit persons to whom the Work
# is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Work.
#
# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS
# IN THE WORK.
#----------------------------------------------------------------------

""" The omni-configure.py script.
    This script is meant to help new users setup their omni config in
    a standard way. Although many of the parameters can be customized using
    command line options, the user should be able to run the script
    with the default configuration and configure Omni. This script should be
    used by new user that want a default configuration of Omni. If advanced
    configuration is needed (multiple users, etc) this should still be done
    manually by editing the omni configuration file.
"""

import string
import sys, os
from subprocess import Popen, PIPE
import ConfigParser
import optparse
from sfa.trust.certificate import Certificate, Keypair

def parseArgs(argv, options=None):
    """Construct an Options Parser for parsing omni-configure command line
    arguments, and parse them.
    """

    parser = optparse.OptionParser()
    parser.add_option("-c", "--configfile", default="~/.gcf/omni_config",
                      help="Config file location", metavar="FILE")
    parser.add_option("-p", "--cert", default="~/.ssl/geni_cert",
                      help="User certificate file location", metavar="FILE")
    parser.add_option("-k", "--plkey", default="~/.ssh/geni_pl_key",
                      help="PlanetLab private key file location", metavar="FILE")
    parser.add_option("-f", "--framework", default="pg", type='choice',
                      choices=['pg', 'pl'],
                      help="Control framework that you have an account with")
    parser.add_option("-v", "--verbose", default=False, action="store_true",
                      help="Turn on verbose command summary for omni-configure script")

    if argv is None:
        # prints to stderr
        parser.print_help()
        return

    (opts, args) = parser.parse_args(argv, options)
    print opts
    return opts, args

def initialize(opts):

    #Check if directory for config file exists
    # Expand the configfile to a full path
    opts.configfile= os.path.expanduser(opts.configfile)
    configdir = os.path.dirname(opts.configfile)
    if not os.path.exists(configdir):
      # If the directory does not exist but it is the
      # default directory, create it, if not print an error
      if not cmp(os.path.expanduser('~/.gcf'), configdir):
        os.makedirs(configdir)
      else:
        sys.exit('Directory '+ configdir + ' does not exist!')

    # If the value is the default add the appropriate file extention
    # based on the framework
    if not cmp(opts.cert, "~/.ssl/geni_cert") :
        if not cmp(opts.framework,'pg'):
            opts.cert += ".pem"
        else :
            if not cmp(opts.framework,'pl'):
                opts.cert += ".gid"

    # Expand the cert file to a full path
    opts.cert= os.path.expanduser(opts.cert)

    # Expand the plkey file to a full path
    opts.plkey = os.path.expanduser(opts.plkey)

    # If framework is pgeni, check that the cert file is in the right place
    if not cmp(opts.framework,'pg'):
        if not os.path.exists(opts.cert):
            sys.exit("Geni certificate not in '"+opts.cert+"'. \nMake sure you\
place the .pem file that you downloaded from the Web UI there,\nor\
use the '-p' option to specify a custom location of the certificate.\n")

    # If framework is planetlab, check that the key are in the right place
    if not cmp(opts.framework,'pl'):
        if not os.path.exists(opts.cert):
            sys.exit("\nScript currently does not support automatic download of \
PlanetLab cert.\nIf you have a copy place it at '"+opts.cert+"', \nor \
use the '-p' option to specify a custom location of the certificate.\n")
        if not os.path.exists(opts.plkey) :
            sys.exit("\nPlanetLab private key not in '"+opts.plkey+"'. \nMake sure\
you place the private key registered with PlanetLab there or use\n\
the '-k' option to specify a custom location for the key.\n")

def createSSHKeypair(opts):
    if not cmp(opts.framework,'pg'):
      pkey = opts.cert
    else :
      if not cmp(opts.framework,'pl'):
        pkey = opts.plkey

    k = Keypair()
    k.load_from_file(pkey)

    # Figure out where to place the private key
    private_key_file = os.path.expanduser('~/.ssh/geni_key')

    if os.path.exists(private_key_file):
        k_exist = Keypair()
        k_exist.load_from_file(private_key_file)
        # If the file exists and it is not the same as the existing key ask the
        # user to replace it or not
        if not k_exist.is_same(k) :
            valid_ans=['','y', 'n']
            replace_flag = raw_input("File " + private_key_file + " exists, do you want to replace it [Y,n]?").lower()
            while replace_flag not in valid_ans:
                replace_flag = raw_input("Your input has to be 'y' or <ENTER> for yes, 'n' for no:").lower()
            if replace_flag == 'n' :
                i = 1
                tmp_pk_file = private_key_file + '_' + str(i)
                while os.path.exists(tmp_pk_file):
                    i = i+1
                    tmp_pk_file = private_key_file + '_' + str(i)
                print tmp_pk_file
                private_key_file = tmp_pk_file

    k.save_to_file(private_key_file)
    # Change the permission to something appropriate for keys
    os.chmod(private_key_file, 0600)

    args = ['ssh-keygen', '-y', '-f']
    args.append(private_key_file)
    p = Popen(args, stdout=PIPE)
    public_key = p.communicate()[0]
    if p.returncode != 0:
        raise Exception("Error creating public key")
    public_key_file = private_key_file + '.pub'
    f = open(public_key_file,'w')
    f.write("%s" % public_key)
    f.close()

    return private_key_file, public_key_file

def createConfigFile(opts, public_key_file):

    cert = Certificate(filename=opts.cert)
    # We need to get the issuer and the subject for SFA frameworks
    # issuer -> authority
    # subject -> user
    issuer = cert.get_issuer()
    subject = cert.get_subject()

    # The user URN is in the Alternate Subject Data
    cert_alt_data = cert.get_data()
    data = cert_alt_data.split(',')
    user_urn_list = [o for o in data if o.find('+user+') != -1]

    # If there is no data that has the string '+user+' this probably means that
    # the provided cert is not a user cert
    if len(user_urn_list) == 0:
      sys.exit("The certificate is probably not a user cert")

    # XXX If there are more data with the '+user+' string probably more than one
    # user URNs in the cert. For now exit, but maybe the right thing would be to
    # pick one?
    if len(user_urn_list) > 1:
      sys.exit("There are more than one user URNs in the cert. Exit!")

    urn = user_urn_list[0].lstrip('URI:')
    user = urn.split('+')[-1]

    if not cmp(opts.framework,'pg'):
        cf_section = """
[%s]
type = pg
ch = https://www.emulab.net:443/protogeni/xmlrpc/ch
sa = https://www.pgeni.gpolab.bbn.com:443/protogeni/xmlrpc/sa
cert = %s
key = %s
""" %(opts.framework, opts.cert, opts.cert)

    else:
      if not cmp(opts.framework, 'pl'):
        cf_section = """
[%s]
type = sfa
authority=%s
user=%s
cert=%s
key=%s
registry=http://www.planet-lab.org:12345
slicemgr=http://www.planet-lab.org:12347
""" %(opts.framework, issuer, subject, opts.cert, opts.plkey)

    omni_config_dict = {
                        'cf' : opts.framework,
                        'user' : user,
                        'urn' : urn,
                        'pkey' : public_key_file,
                        'cf_section' : cf_section,
                       }

    omni_config_file="""
[omni]
default_cf = %(cf)s
users = %(user)s

# ---------- Users ----------
[%(user)s]
urn = %(urn)s
keys = %(pkey)s

# ---------- Frameworks ----------
%(cf_section)s

#------AM nicknames
# Format :
# Nickname=URN, URL
# URN is optional
[aggregate_nicknames]
pg-gpo=,https://pgeni.gpolab.bbn.com/protogeni/xmlrpc/am
pg-utah=,https://www.emulab.net/protogeni/xmlrpc/am
plc=,https://www.planet-lab.org:12346
pg-ky=,https://www.uky.emulab.net/protogeni/xmlrpc/am

# Private myplc installations
plc-gpo=,http://myplc.gpolab.bbn.com:12346/
plc-clemson=,http://myplc.clemson.edu:12346/
plc-stanford=,https://myplc.stanford.edu:12346/
plc-wisconsin=,https://wings-openflow-1.wail.wisc.edu:12346/
plc-washington=,https://of.cs.washington.edu:12346/
plc-rutgers=,https://plc.orbit-lab.org:12346/
plc-indiana=,https://myplc.grnoc.iu.edu:12346/
plc-gatech=,https://myplc.cip.gatech.edu:12346/

# OpenFlow AMs
of-gpo=,https://foam.gpolab.bbn.com:3626/foam/gapi/1
of-stanford=,https://openflow4.stanford.edu:3626/foam/gapi/1
of-clemson=,https://foam.clemson.edu:3626/foam/gapi/1
of-wisconsin=,https://foam.wail.wisc.edu:3626/foam/gapi/1
of-rutgers=,https://foam.oflow.cip.gatech.edu:3626/foam/gapi/1
of-indiana=,https://foam.noc.iu.edu:3626/foam/gapi/1
of-gatech=,https://nox.orbit-lab.org:3626/foam/gapi/1
of-nlr=,https://foam.nlr.net:3626/foam/gapi/1
of-i2=,https://foam.net.internet2.edu:3626/foam/gapi/1

""" % omni_config_dict

    f = open(opts.configfile, 'w')
    print >>f, omni_config_file
    f.close()

def main(argv=None):
    # do initial setup & process the user's call
    if argv is None:
        argv = sys.argv[1:]
        (opts, args) = parseArgs(argv)
        initialize(opts)
        (pr_key_file, pub_key_file) = createSSHKeypair(opts)
        createConfigFile(opts,pub_key_file)

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>Check the URN to figure out where the users has an account with and put the appropriate sa in the omni_config. For now this has hardcoded checks for emulab.net and pgeni.gpolab.bbn.com<commit_after>#!/usr/bin/python

#----------------------------------------------------------------------
# Copyright (c) 2011 Raytheon BBN Technologies
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and/or hardware specification (the "Work") to
# deal in the Work without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Work, and to permit persons to whom the Work
# is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Work.
#
# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS
# IN THE WORK.
#----------------------------------------------------------------------

""" The omni-configure.py script.
    This script is meant to help new users setup their omni config in
    a standard way. Although many of the parameters can be customized using
    command line options, the user should be able to run the script
    with the default configuration and configure Omni. This script should be
    used by new user that want a default configuration of Omni. If advanced
    configuration is needed (multiple users, etc) this should still be done
    manually by editing the omni configuration file.
"""

import string
import sys, os
from subprocess import Popen, PIPE
import ConfigParser
import optparse
from sfa.trust.certificate import Certificate, Keypair

def parseArgs(argv, options=None):
    """Construct an Options Parser for parsing omni-configure command line
    arguments, and parse them.
    """

    parser = optparse.OptionParser()
    parser.add_option("-c", "--configfile", default="~/.gcf/omni_config",
                      help="Config file location", metavar="FILE")
    parser.add_option("-p", "--cert", default="~/.ssl/geni_cert",
                      help="User certificate file location", metavar="FILE")
    parser.add_option("-k", "--plkey", default="~/.ssh/geni_pl_key",
                      help="PlanetLab private key file location", metavar="FILE")
    parser.add_option("-f", "--framework", default="pg", type='choice',
                      choices=['pg', 'pl'],
                      help="Control framework that you have an account with")
    parser.add_option("-v", "--verbose", default=False, action="store_true",
                      help="Turn on verbose command summary for omni-configure script")

    if argv is None:
        # prints to stderr
        parser.print_help()
        return

    (opts, args) = parser.parse_args(argv, options)
    print opts
    return opts, args

def initialize(opts):

    #Check if directory for config file exists
    # Expand the configfile to a full path
    opts.configfile= os.path.expanduser(opts.configfile)
    configdir = os.path.dirname(opts.configfile)
    if not os.path.exists(configdir):
      # If the directory does not exist but it is the
      # default directory, create it, if not print an error
      if not cmp(os.path.expanduser('~/.gcf'), configdir):
        os.makedirs(configdir)
      else:
        sys.exit('Directory '+ configdir + ' does not exist!')

    # If the value is the default add the appropriate file extention
    # based on the framework
    if not cmp(opts.cert, "~/.ssl/geni_cert") :
        if not cmp(opts.framework,'pg'):
            opts.cert += ".pem"
        else :
            if not cmp(opts.framework,'pl'):
                opts.cert += ".gid"

    # Expand the cert file to a full path
    opts.cert= os.path.expanduser(opts.cert)

    # Expand the plkey file to a full path
    opts.plkey = os.path.expanduser(opts.plkey)

    # If framework is pgeni, check that the cert file is in the right place
    if not cmp(opts.framework,'pg'):
        if not os.path.exists(opts.cert):
            sys.exit("Geni certificate not in '"+opts.cert+"'. \nMake sure you\
place the .pem file that you downloaded from the Web UI there,\nor\
use the '-p' option to specify a custom location of the certificate.\n")

    # If framework is planetlab, check that the key are in the right place
    if not cmp(opts.framework,'pl'):
        if not os.path.exists(opts.cert):
            sys.exit("\nScript currently does not support automatic download of \
PlanetLab cert.\nIf you have a copy place it at '"+opts.cert+"', \nor \
use the '-p' option to specify a custom location of the certificate.\n")
        if not os.path.exists(opts.plkey) :
            sys.exit("\nPlanetLab private key not in '"+opts.plkey+"'. \nMake sure\
you place the private key registered with PlanetLab there or use\n\
the '-k' option to specify a custom location for the key.\n")

def createSSHKeypair(opts):
    if not cmp(opts.framework,'pg'):
      pkey = opts.cert
    else :
      if not cmp(opts.framework,'pl'):
        pkey = opts.plkey

    k = Keypair()
    k.load_from_file(pkey)

    # Figure out where to place the private key
    private_key_file = os.path.expanduser('~/.ssh/geni_key')

    if os.path.exists(private_key_file):
        k_exist = Keypair()
        k_exist.load_from_file(private_key_file)
        # If the file exists and it is not the same as the existing key ask the
        # user to replace it or not
        if not k_exist.is_same(k) :
            valid_ans=['','y', 'n']
            replace_flag = raw_input("File " + private_key_file + " exists, do you want to replace it [Y,n]?").lower()
            while replace_flag not in valid_ans:
                replace_flag = raw_input("Your input has to be 'y' or <ENTER> for yes, 'n' for no:").lower()
            if replace_flag == 'n' :
                i = 1
                tmp_pk_file = private_key_file + '_' + str(i)
                while os.path.exists(tmp_pk_file):
                    i = i+1
                    tmp_pk_file = private_key_file + '_' + str(i)
                print tmp_pk_file
                private_key_file = tmp_pk_file

    k.save_to_file(private_key_file)
    # Change the permission to something appropriate for keys
    os.chmod(private_key_file, 0600)

    args = ['ssh-keygen', '-y', '-f']
    args.append(private_key_file)
    p = Popen(args, stdout=PIPE)
    public_key = p.communicate()[0]
    if p.returncode != 0:
        raise Exception("Error creating public key")
    public_key_file = private_key_file + '.pub'
    f = open(public_key_file,'w')
    f.write("%s" % public_key)
    f.close()

    return private_key_file, public_key_file

def createConfigFile(opts, public_key_file):

    cert = Certificate(filename=opts.cert)
    # We need to get the issuer and the subject for SFA frameworks
    # issuer -> authority
    # subject -> user
    issuer = cert.get_issuer()
    subject = cert.get_subject()

    # The user URN is in the Alternate Subject Data
    cert_alt_data = cert.get_data()
    data = cert_alt_data.split(',')
    user_urn_list = [o for o in data if o.find('+user+') != -1]

    # If there is no data that has the string '+user+' this probably means that
    # the provided cert is not a user cert
    if len(user_urn_list) == 0:
      sys.exit("The certificate is probably not a user cert")

    # XXX If there are more data with the '+user+' string probably more than one
    # user URNs in the cert. For now exit, but maybe the right thing would be to
    # pick one?
    if len(user_urn_list) > 1:
      sys.exit("There are more than one user URNs in the cert. Exit!")

    urn = user_urn_list[0].lstrip('URI:')
    user = urn.split('+')[-1]

    if not cmp(opts.framework,'pg'):
        if urn.find('emulab.net') != -1 :
            sa = 'https://www.emulab.net:443/protogeni/xmlrpc/sa'
        else :
            if urn.find('pgeni.gpolab.bbn.com') != -1 :
                sa = 'https://www.pgeni.gpolab.bbn.com:443/protogeni/xmlrpc/sa'
            else :
                raise Exception("Creation of omni_config for users at %s is not supported. Please contact omni-users@geni.net" % urn.split('+')[-2])
        cf_section = """
[%s]
type = pg
ch = https://www.emulab.net:443/protogeni/xmlrpc/ch
sa = %s
cert = %s
key = %s
""" %(opts.framework, sa, opts.cert, opts.cert)

    else:
      if not cmp(opts.framework, 'pl'):
        cf_section = """
[%s]
type = sfa
authority=%s
user=%s
cert=%s
key=%s
registry=http://www.planet-lab.org:12345
slicemgr=http://www.planet-lab.org:12347
""" %(opts.framework, issuer, subject, opts.cert, opts.plkey)

    omni_config_dict = {
                        'cf' : opts.framework,
                        'user' : user,
                        'urn' : urn,
                        'pkey' : public_key_file,
                        'cf_section' : cf_section,
                       }

    omni_config_file="""
[omni]
default_cf = %(cf)s
users = %(user)s

# ---------- Users ----------
[%(user)s]
urn = %(urn)s
keys = %(pkey)s

# ---------- Frameworks ----------
%(cf_section)s

#------AM nicknames
# Format :
# Nickname=URN, URL
# URN is optional
[aggregate_nicknames]
pg-gpo=,https://pgeni.gpolab.bbn.com/protogeni/xmlrpc/am
pg-utah=,https://www.emulab.net/protogeni/xmlrpc/am
plc=,https://www.planet-lab.org:12346
pg-ky=,https://www.uky.emulab.net/protogeni/xmlrpc/am

# Private myplc installations
plc-gpo=,http://myplc.gpolab.bbn.com:12346/
plc-clemson=,http://myplc.clemson.edu:12346/
plc-stanford=,https://myplc.stanford.edu:12346/
plc-wisconsin=,https://wings-openflow-1.wail.wisc.edu:12346/
plc-washington=,https://of.cs.washington.edu:12346/
plc-rutgers=,https://plc.orbit-lab.org:12346/
plc-indiana=,https://myplc.grnoc.iu.edu:12346/
plc-gatech=,https://myplc.cip.gatech.edu:12346/

# OpenFlow AMs
of-gpo=,https://foam.gpolab.bbn.com:3626/foam/gapi/1
of-stanford=,https://openflow4.stanford.edu:3626/foam/gapi/1
of-clemson=,https://foam.clemson.edu:3626/foam/gapi/1
of-wisconsin=,https://foam.wail.wisc.edu:3626/foam/gapi/1
of-rutgers=,https://foam.oflow.cip.gatech.edu:3626/foam/gapi/1
of-indiana=,https://foam.noc.iu.edu:3626/foam/gapi/1
of-gatech=,https://nox.orbit-lab.org:3626/foam/gapi/1
of-nlr=,https://foam.nlr.net:3626/foam/gapi/1
of-i2=,https://foam.net.internet2.edu:3626/foam/gapi/1

""" % omni_config_dict

    f = open(opts.configfile, 'w')
    print >>f, omni_config_file
    f.close()

def main(argv=None):
    # do initial setup & process the user's call
    if argv is None:
        argv = sys.argv[1:]
        (opts, args) = parseArgs(argv)
        initialize(opts)
        (pr_key_file, pub_key_file) = createSSHKeypair(opts)
        createConfigFile(opts,pub_key_file)

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before>import numpy as np

from latent_crf import LatentCRF
from latent_structured_svm import LatentSSVM
from pystruct.learners import OneSlackSSVM
from time import time

from common import load_data
from common import compute_error

if __name__ == '__main__':
    results = np.zeros((18, 7))
    timestamps = np.zeros((18, 7))

    for dataset in xrange(1, 18):
        for j, nfull in enumerate([0, 2, 4, 10, 25, 100, 400]):
            crf = LatentCRF(n_states=10, n_features=10, n_edge_features=2,
                            inference_method='qpbo')
            base_clf = OneSlackSSVM(crf, max_iter=100, C=0.01, verbose=0,
                                    tol=0.1, n_jobs=4, inference_cache=100)
            clf = LatentSSVM(base_clf, latent_iter=5)

            X, H = load_data(dataset)

            Y = []
            for h in H:
                Y.append(np.unique(h))

            x_train = X[:400]
            y_train = Y[:400]
            h_train = H[:400]
            x_test = X[401:]
            y_test = Y[401:]
            h_test = H[401:]

            for i in xrange(nfull, len(h_train)):
                h_train[i] = None

            start = time()
            clf.fit(x_train, y_train, h_train)
            stop = time()

            h_pred = clf.predict_latent(x_test)

            results[dataset - 1, j] = compute_error(h_test, h_pred)
            timestamps[dataset - 1, j] = stop - start

            print 'Dataset: %d, nfull=%d, error=%f, time=%f' % (dataset, nfull, results[dataset - 1, j], timestamps[dataset - 1, j])

    np.savetxt('quality.csv', results, delimiter=',')
    np.savetxt('timestamps.csv', timestamps, delimiter=',')
<commit_msg>refactor ssvm testing utility<commit_after>import numpy as np

from latent_crf import LatentCRF
from edge_crf import EdgeCRF
from latent_structured_svm import LatentSSVM

from pystruct.learners import OneSlackSSVM
from time import time

from common import load_data
from common import compute_error

def latent_model():
    crf = LatentCRF(n_states=10, n_features=10, n_edge_features=2,
                    inference_method='qpbo')
    base_clf = OneSlackSSVM(crf, max_iter=100, C=0.01, verbose=0,
                            tol=0.1, n_jobs=4, inference_cache=100)
    clf = LatentSSVM(base_clf, latent_iter=5)

    return clf

def ssvm_model():
    crf = EdgeCRF(n_states=10, n_features=10, n_edge_features=2,
                  inference_method='qpbo')
    clf = OneSlackSSVM(crf, max_iter=10000, C=0.01, verbose=0,
                       tol=0.1, show_loss_every=5, n_jobs=4,
                       inference_cache=100)

    return clf

def weak_from_hidden(H):
    Y = []
    for h in H:
        Y.append(np.unique(h))
    return Y

def test_latent(full_labeled, train_size):
    results = np.zeros((18, 7))
    timestamps = np.zeros((18, 7))

    for dataset in xrange(1, 19):
        X, H = load_data(dataset)
        Y = weak_from_hidden(H)

        for j, nfull in enumerate(full_labeled):
            clf = ssvm_model()

            x_train = X[:train_size]
            y_train = Y[:train_size]
            h_train = H[:train_size]
            x_test = X[(train_size + 1):]
            h_test = H[(train_size + 1):]

            for i in xrange(nfull, len(h_train)):
                h_train[i] = None

            try:
                start = time()
                clf.fit(x_train, y_train, h_train)
                stop = time()
                h_pred = clf.predict_latent(x_test)

                results[dataset - 1, j] = compute_error(h_test, h_pred)
                timestamps[dataset - 1, j] = stop - start

                print 'dataset=%d, nfull=%d, \
                       error=%f, time=%f' % (dataset, nfull,
                                             results[dataset - 1, j],
                                             timestamps[dataset - 1, j])
            except ValueError:
                print 'dataset=%d, nfull=%d: Failed' % (dataset, nfull)

    return results, timestamps

def test_ssvm(full_labeled, train_size):
    results = np.zeros((18, 7))
    timestamps = np.zeros((18, 7))

    for dataset in xrange(1, 19):
        X, Y = load_data(dataset)

        for j, nfull in enumerate(full_labeled):
            clf = ssvm_model()

            x_train = X[:nfull]
            y_train = Y[:nfull]
            x_test = X[(train_size + 1):]
            y_test = Y[(train_size + 1):]

            try:
                start = time()
                clf.fit(x_train, y_train)
                stop = time()
                y_pred = clf.predict(x_test)

                results[dataset - 1, j] = compute_error(y_test, y_pred)
                timestamps[dataset - 1, j] = stop - start

                print 'dataset=%d, nfull=%d, \
                       error=%f, time=%f' % (dataset, nfull,
                                             results[dataset - 1, j],
                                             timestamps[dataset - 1, j])
            except ValueError:
                print 'dataset=%d, nfull=%d: Failed' % (dataset, nfull)

    return results, timestamps

if __name__ == '__main__':
    full_labeled = np.array([0, 2, 4, 10, 25, 100, 400])
    train_size = 400

    results, timestamps = test_ssvm(full_labeled, train_size)

    np.savetxt('ssvm_quality.csv', results, delimiter=',')
    np.savetxt('ssvm_timestamps.csv', timestamps, delimiter=',')
<|endoftext|>