<commit_before>'''
Multi-layer perceptrons.
'''
from copy import deepcopy

import numpy as np

from . import metrics
from .helpers import np_print_options

__all__ = ('MultilayerPerceptron',)

class _Layer:
    def __init__(self, num_inputs, num_nodes, parent=None, bias=True,
                 sigmoid=metrics.logistic, weight_init_range=.5):
        self.num_nodes = num_nodes
        self.num_inputs = num_inputs
        self.sigmoid = sigmoid
        self.bias = bias

        self.parent = parent
        self.child = None
        if parent is not None:
            parent.child = self

        if bias:
            num_nodes_nobias = num_nodes - 1
        else:
            num_nodes_nobias = num_nodes

        self._activation_scores = np.zeros(num_nodes_nobias)
        self.activations = np.zeros(num_nodes)
        if bias: self.activations[0] = 1
        self.weights = np.random.uniform(-weight_init_range, weight_init_range,
                                         size=(num_inputs, num_nodes_nobias))
        self._last_change = np.zeros((num_inputs, num_nodes_nobias))
        self._deltas = np.zeros(num_nodes_nobias)

    def __str__(self):
        layer_rep = ('[ ' +
                     self.bias*'1 ' + (1-self.bias)*'x ' +
                     ' '.join(['x' for _ in range(self.num_nodes-1)]) +
                     (self.num_nodes > 1)*' ' +
                     ']')

        with np_print_options(precision=5, suppress=True):
            result = "  {}\n{}".format(
                str(self.weights).replace('\n', '\n  '),
                layer_rep
            )
        return result

    def update_activations(self, inputs):
        self._activation_scores = np.dot(inputs, self.weights)
        if self.bias:
            if len(inputs.shape) == 1:
                self.activations = np.append(
                    [1], self.sigmoid.f(self._activation_scores))
            else:
                self.activations = np.hstack(
                    [np.ones(len(inputs)).reshape(-1,1),
                     self.sigmoid.f(self._activation_scores)])
        else:
            self.activations = self.sigmoid.f(self._activation_scores)

        if self.child is None:
            return self.activations
        else:
            return self.child.update_activations(self.activations)

    def backpropogate(self, system_inputs, errors, learn_rate, momentum):
        if self.parent is None:
            inputs = system_inputs
        else:
            inputs = self.parent.activations

        if self.bias:
            self._deltas = self.sigmoid.d(self._activation_scores)*errors[1:]
        else:
            self._deltas = self.sigmoid.d(self._activation_scores)*errors

        change = np.outer(inputs, self._deltas)
        self._last_change = change + momentum*self._last_change
        self.weights += learn_rate*self._last_change

        if self.parent is None:
            return np.dot(self._deltas, self.weights.T)
        else:
            return self.parent.backpropogate(
                system_inputs, np.dot(self._deltas, self.weights.T),
                learn_rate, momentum
            )

class MultilayerPerceptron:
    '''
    Multilayer Perceptron.

    Parameters
    ----------
    num_inputs : int, optional
        Number of input nodes to the system. Should be the same as the number
        of features of input data, +1 if you use add_constant=True in the
        fit method (default 3).
    num_outputs : int, optional
        Number of output nodes from the system. Should be the same as the number
        of variables in the training output data (default 1).
    num_hidden_layers : int, optional
        Number of hidden layers to train (default 1).
    num_hidden_nodes : int or list, optional
        Number of nodes per hidden layer. If an int is provided then all the
        hidden layers will have that number of hidden nodes. If a list-like
        is provided then they will be mapped onto the corresponding layers
        ordered from input to output (default 3).
    learn_rate : float, optional
        Learning rate for training (default 0.5).
    momentum : float, optional
        Momentum parameter for training (default 0.1).
    seed : int, optional
        Seeds the random number generator (default None).
    sigmoid : object, optional
        Sigmoid function to use. Must have f and d methods for the function
        and derivative values, respectively (default metrics.logistic).
    weight_init_range : float, optional
        Weights are initalized from a uniform distribution between plus-minus
        this value (default .5).
    '''
    params_names = (
        'num_inputs', 'num_outputs', 'num_hidden_layers', 'num_hidden_nodes',
        'learn_rate', 'momentum', 'seed', 'sigmoid',
        'weight_init_range'
    )

    def __init__(self, num_inputs=3, num_outputs=1,
                 num_hidden_layers=1, num_hidden_nodes=3,
                 learn_rate=.5, momentum=.1, seed=None,
                 sigmoid=metrics.logistic, weight_init_range=.5):

        self.num_inputs = num_inputs
        self.num_outputs = num_outputs
        self.learn_rate = learn_rate
        self.momentum = momentum
        self.seed = seed
        self.sigmoid = sigmoid
        self.weight_init_range = weight_init_range

        self.num_hidden_layers = num_hidden_layers
        if isinstance(num_hidden_nodes, int):
            num_hidden_nodes = [num_hidden_nodes
                                for i in range(num_hidden_layers)]
        elif len(num_hidden_nodes) != num_hidden_layers:
            raise ValueError(
                'Length of num_hidden_nodes ({}) must match num_hidden_layers '
                '({}).'.format(len(num_hidden_nodes), num_hidden_layers)
            )
        self.num_hidden_nodes = num_hidden_nodes

        lpars = {'sigmoid': sigmoid, 'weight_init_range': weight_init_range}

        self.layers = []
        self.layers.append(_Layer(num_inputs, num_hidden_nodes[0],
                                  bias=True, **lpars))
        for num_inp, num_nodes in zip(num_hidden_nodes[:-1],
                                      num_hidden_nodes[1:]):
            self.layers.append(
                _Layer(num_inp, num_nodes,
                       parent=self.layers[-1], bias=True, **lpars)
            )
        self.layers.append(_Layer(num_hidden_nodes[-1], num_outputs, bias=False,
                                  parent=self.layers[-1], **lpars))

    def __str__(self):
        result = ('[ 1 ' +
                  ' '.join(['x' for _ in range(self.num_inputs-1)]) +
                  ' ]')
        for l in self.layers:
            result += '\n' + str(l)
        return result

    @property
    def params(self):
        result = {}
        for name in self.params_names:
            result[name] = getattr(self, name)
        return result

    def copy(self):
        result = MultilayerPerceptron(**self.params)
        result.layers = deepcopy(self.layers)
        return result

    def fit(self, X, Y, epochnum=1000, add_constant=True, verbose=False):
        '''
        Fit the multilayer perceptron using training data.

        Parameters
        ----------
        X : array of shape (n_samples, n_features)
            Feature dataset for training. Either a constant column is required
            or add_constant must be set to True. If there is already a constant
            column then it must be the first column in X.
        Y : array of shape (n_samples, n_outputs)
            Output data for training.
        epochnum : int, optional
            Number of epochs, i.e. passes through the entire dataset
            (default 1000).
        add_constant : bool, optional
            Adds a column of ones to the front of the X data. Set to False
            if your data already has a constant column (default True).
        verbose : bool, optional
            Print status during estimation.
        '''
        if self.seed is not None:
            np.random.seed(self.seed)

        num_obs = len(X)
        if Y.ndim == 1:
            Y = np.reshape(Y, (-1, 1))
        if add_constant:
            X = np.column_stack([np.ones(num_obs), X])

        for i in range(epochnum):
            order = np.random.choice(num_obs, size=num_obs, replace=False)
            for ind in order:
                error = 0.

                targets = Y[ind]
                inputs = X[ind]

                pred = self.layers[0].update_activations(inputs)
                self.layers[-1].backpropogate(
                    inputs, targets-pred, self.learn_rate, self.momentum)

                error += np.mean(np.abs(targets - pred))
            if verbose and (i % 100) == 99:
                print('{:>4}, error: {:.3e}'.format(i+1, error/num_obs))
        return self

    def predict_prob(self, X, add_constant=True):
        if add_constant:
            X = np.column_stack([np.ones(len(X)), X])

        return self.layers[0].update_activations(X)

    def classify(self, X, add_constant=True, max_ind=False):
        prob = self.predict_prob(X, add_constant=add_constant)
        if max_ind:
            return np.argmax(prob, axis=1)
        else:
            return (prob > .5).astype(int)

<commit_msg>Changed verbose to take integer arguments.<commit_after>'''
Multi-layer perceptrons.
'''
from copy import deepcopy

import numpy as np

from . import metrics
from .helpers import np_print_options

__all__ = ('MultilayerPerceptron',)

class _Layer:
    def __init__(self, num_inputs, num_nodes, parent=None, bias=True,
                 sigmoid=metrics.logistic, weight_init_range=.5):
        self.num_nodes = num_nodes
        self.num_inputs = num_inputs
        self.sigmoid = sigmoid
        self.bias = bias

        self.parent = parent
        self.child = None
        if parent is not None:
            parent.child = self

        if bias:
            num_nodes_nobias = num_nodes - 1
        else:
            num_nodes_nobias = num_nodes

        self._activation_scores = np.zeros(num_nodes_nobias)
        self.activations = np.zeros(num_nodes)
        if bias: self.activations[0] = 1
        self.weights = np.random.uniform(-weight_init_range, weight_init_range,
                                         size=(num_inputs, num_nodes_nobias))
        self._last_change = np.zeros((num_inputs, num_nodes_nobias))
        self._deltas = np.zeros(num_nodes_nobias)

    def __str__(self):
        layer_rep = ('[ ' +
                     self.bias*'1 ' + (1-self.bias)*'x ' +
                     ' '.join(['x' for _ in range(self.num_nodes-1)]) +
                     (self.num_nodes > 1)*' ' +
                     ']')

        with np_print_options(precision=5, suppress=True):
            result = "  {}\n{}".format(
                str(self.weights).replace('\n', '\n  '),
                layer_rep
            )
        return result

    def update_activations(self, inputs):
        self._activation_scores = np.dot(inputs, self.weights)
        if self.bias:
            if len(inputs.shape) == 1:
                self.activations = np.append(
                    [1], self.sigmoid.f(self._activation_scores))
            else:
                self.activations = np.hstack(
                    [np.ones(len(inputs)).reshape(-1,1),
                     self.sigmoid.f(self._activation_scores)])
        else:
            self.activations = self.sigmoid.f(self._activation_scores)

        if self.child is None:
            return self.activations
        else:
            return self.child.update_activations(self.activations)

    def backpropogate(self, system_inputs, errors, learn_rate, momentum):
        if self.parent is None:
            inputs = system_inputs
        else:
            inputs = self.parent.activations

        if self.bias:
            self._deltas = self.sigmoid.d(self._activation_scores)*errors[1:]
        else:
            self._deltas = self.sigmoid.d(self._activation_scores)*errors

        change = np.outer(inputs, self._deltas)
        self._last_change = change + momentum*self._last_change
        self.weights += learn_rate*self._last_change

        if self.parent is None:
            return np.dot(self._deltas, self.weights.T)
        else:
            return self.parent.backpropogate(
                system_inputs, np.dot(self._deltas, self.weights.T),
                learn_rate, momentum
            )

class MultilayerPerceptron:
    '''
    Multilayer Perceptron.

    Parameters
    ----------
    num_inputs : int, optional
        Number of input nodes to the system. Should be the same as the number
        of features of input data, +1 if you use add_constant=True in the
        fit method (default 3).
    num_outputs : int, optional
        Number of output nodes from the system. Should be the same as the number
        of variables in the training output data (default 1).
    num_hidden_layers : int, optional
        Number of hidden layers to train (default 1).
    num_hidden_nodes : int or list, optional
        Number of nodes per hidden layer. If an int is provided then all the
        hidden layers will have that number of hidden nodes. If a list-like
        is provided then they will be mapped onto the corresponding layers
        ordered from input to output (default 3).
    learn_rate : float, optional
        Learning rate for training (default 0.5).
    momentum : float, optional
        Momentum parameter for training (default 0.1).
    seed : int, optional
        Seeds the random number generator (default None).
    sigmoid : object, optional
        Sigmoid function to use. Must have f and d methods for the function
        and derivative values, respectively (default metrics.logistic).
    weight_init_range : float, optional
        Weights are initalized from a uniform distribution between plus-minus
        this value (default .5).
    '''
    params_names = (
        'num_inputs', 'num_outputs', 'num_hidden_layers', 'num_hidden_nodes',
        'learn_rate', 'momentum', 'seed', 'sigmoid',
        'weight_init_range'
    )

    def __init__(self, num_inputs=3, num_outputs=1,
                 num_hidden_layers=1, num_hidden_nodes=3,
                 learn_rate=.5, momentum=.1, seed=None,
                 sigmoid=metrics.logistic, weight_init_range=.5):

        self.num_inputs = num_inputs
        self.num_outputs = num_outputs
        self.learn_rate = learn_rate
        self.momentum = momentum
        self.sigmoid = sigmoid
        self.weight_init_range = weight_init_range

        self.seed = seed
        if seed is not None:
            np.random.seed(seed)

        self.num_hidden_layers = num_hidden_layers
        if isinstance(num_hidden_nodes, int):
            num_hidden_nodes = [num_hidden_nodes
                                for i in range(num_hidden_layers)]
        elif len(num_hidden_nodes) != num_hidden_layers:
            raise ValueError(
                'Length of num_hidden_nodes ({}) must match num_hidden_layers '
                '({}).'.format(len(num_hidden_nodes), num_hidden_layers)
            )
        self.num_hidden_nodes = num_hidden_nodes

        lpars = {'sigmoid': sigmoid, 'weight_init_range': weight_init_range}

        self.layers = []
        self.layers.append(_Layer(num_inputs, num_hidden_nodes[0],
                                  bias=True, **lpars))
        for num_inp, num_nodes in zip(num_hidden_nodes[:-1],
                                      num_hidden_nodes[1:]):
            self.layers.append(
                _Layer(num_inp, num_nodes,
                       parent=self.layers[-1], bias=True, **lpars)
            )
        self.layers.append(_Layer(num_hidden_nodes[-1], num_outputs, bias=False,
                                  parent=self.layers[-1], **lpars))

    def __str__(self):
        result = ('[ 1 ' +
                  ' '.join(['x' for _ in range(self.num_inputs-1)]) +
                  ' ]')
        for l in self.layers:
            result += '\n' + str(l)
        return result

    @property
    def params(self):
        result = {}
        for name in self.params_names:
            result[name] = getattr(self, name)
        return result

    def copy(self):
        result = MultilayerPerceptron(**self.params)
        result.layers = deepcopy(self.layers)
        return result

    def fit(self, X, Y, epochnum=100, add_constant=True, verbose=False):
        '''
        Fit the multilayer perceptron using training data.

        Parameters
        ----------
        X : array of shape (n_samples, n_features)
            Feature dataset for training. Either a constant column is required
            or add_constant must be set to True. If there is already a constant
            column then it must be the first column in X.
        Y : array of shape (n_samples, n_outputs)
            Output data for training.
        epochnum : int, optional
            Number of epochs, i.e. passes through the entire dataset
            (default 1000).
        add_constant : bool, optional
            Adds a column of ones to the front of the X data. Set to False
            if your data already has a constant column (default True).
        verbose : bool/int, optional
            Print status during estimation. If an int is provided, prints
            status every verbose epochs.
        '''
        if verbose is True:
            verbose = 50

        num_obs = len(X)
        if Y.ndim == 1:
            Y = np.reshape(Y, (-1, 1))
        if add_constant:
            X = np.column_stack([np.ones(num_obs), X])

        for i in range(epochnum):
            order = np.random.choice(num_obs, size=num_obs, replace=False)
            for ind in order:
                error = 0.

                targets = Y[ind]
                inputs = X[ind]

                pred = self.layers[0].update_activations(inputs)
                self.layers[-1].backpropogate(
                    inputs, targets-pred, self.learn_rate, self.momentum)

                error += np.mean(np.abs(targets - pred))
            if verbose and (i % verbose) == (verbose-1):
                print('{:>4}, error: {:.3e}'.format(i+1, error/num_obs))
        return self

    def predict_prob(self, X, add_constant=True):
        if add_constant:
            X = np.column_stack([np.ones(len(X)), X])

        return self.layers[0].update_activations(X)

    def classify(self, X, add_constant=True, max_ind=False):
        prob = self.predict_prob(X, add_constant=add_constant)
        if max_ind:
            return np.argmax(prob, axis=1)
        else:
            return (prob > .5).astype(int)

<|endoftext|>


<commit_before>'''
Data transformations. These are used for stuff. testing.s
'''
import numpy as np
from scipy.linalg import svd

def standardize(X):
    ''' Standardizes each column of the data to mean = 0, stdev = 1. '''
    return (X - np.mean(X, axis=0)) / np.std(X, axis=0)

class PCA:
    '''
    Principle Components Analysis.

    Parameters
    ----------
    X : ndarray
        Training data to estimate PCA on.
    num_comp : int [None]
        Number of principle components to use. If not provided, will default
        to min(num_samples, num_variables).

    Attributes
    ----------
    Vt : ndarray
        The transpose of the principle component weights.
    '''
    def __init__(self, X, num_comp=None):
        if num_comp is None:
            num_comp = min(X.shape)
        else:
            self.num_comp = num_comp
        self.X = X
        self.Vt = None

    def fit(self):
        ''' Calculate the principle components. '''
        U, S, Vt = svd(self.X, full_matrices=False)
        self.Vt = Vt
        return self

    def transform(self, X=None, num_comp=None):
        ''' Apply dimensionality reduction.

        Parameters
        ----------
        X : ndarray [None]
            X matrix to apply dimensionality reduction to. Will use the
            training data if not provided.
        num_comp : ndarray [None]
            Number of principle components to use. If not provided, will use
            the number provided when initializing the PCA object.

        Returns
        -------
        X_new : ndarray
            Transformed array of shape (num_samples, num_comp).
        '''
        X, num_comp = self._check(X, num_comp)
        return np.dot(X, self.Vt[:self.num_comp].T)

    def project(self, X=None, num_comp=None):
        ''' Project onto the principle components.

        Parameters
        ----------
        X : ndarray [None]
            X matrix to project. Will use the trainig data if not provided.
        num_comp : ndarray [None]
            Number of principle components to use. If not provided, will use
            the number provided when initializing the PCA object.

        Returns
        -------
        X_new : ndarray
            Projected array of shape (num_samples, num_variables), the same
            shape as X.
        '''
        X, num_comp = self._check(X, num_comp)
        return np.dot(np.dot(X, self.Vt[:self.num_comp].T),
                      self.Vt[:self.num_comp])

    def _check(self, X, num_comp):
        if self.Vt is None:
            raise AttributeError('Model not yet fit. Use fit() first.')
        if X is None:
            X = self.X
        if num_comp is None:
            num_comp = self.num_comp

        return X, num_comp
<commit_msg>Getting rid of testing stuff.<commit_after>'''
Data transformations.
'''
import numpy as np
from scipy.linalg import svd

def standardize(X):
    ''' Standardizes each column of the data to mean = 0, stdev = 1. '''
    return (X - np.mean(X, axis=0)) / np.std(X, axis=0)

class PCA:
    '''
    Principle Components Analysis.

    Parameters
    ----------
    X : ndarray
        Training data to estimate PCA on.
    num_comp : int [None]
        Number of principle components to use. If not provided, will default
        to min(num_samples, num_variables).

    Attributes
    ----------
    Vt : ndarray
        The transpose of the principle component weights.
    '''
    def __init__(self, X, num_comp=None):
        if num_comp is None:
            num_comp = min(X.shape)
        else:
            self.num_comp = num_comp
        self.X = X
        self.Vt = None

    def fit(self):
        ''' Calculate the principle components. '''
        U, S, Vt = svd(self.X, full_matrices=False)
        self.Vt = Vt
        return self

    def transform(self, X=None, num_comp=None):
        ''' Apply dimensionality reduction.

        Parameters
        ----------
        X : ndarray [None]
            X matrix to apply dimensionality reduction to. Will use the
            training data if not provided.
        num_comp : ndarray [None]
            Number of principle components to use. If not provided, will use
            the number provided when initializing the PCA object.

        Returns
        -------
        X_new : ndarray
            Transformed array of shape (num_samples, num_comp).
        '''
        X, num_comp = self._check(X, num_comp)
        return np.dot(X, self.Vt[:self.num_comp].T)

    def project(self, X=None, num_comp=None):
        ''' Project onto the principle components.

        Parameters
        ----------
        X : ndarray [None]
            X matrix to project. Will use the trainig data if not provided.
        num_comp : ndarray [None]
            Number of principle components to use. If not provided, will use
            the number provided when initializing the PCA object.

        Returns
        -------
        X_new : ndarray
            Projected array of shape (num_samples, num_variables), the same
            shape as X.
        '''
        X, num_comp = self._check(X, num_comp)
        return np.dot(np.dot(X, self.Vt[:self.num_comp].T),
                      self.Vt[:self.num_comp])

    def _check(self, X, num_comp):
        if self.Vt is None:
            raise AttributeError('Model not yet fit. Use fit() first.')
        if X is None:
            X = self.X
        if num_comp is None:
            num_comp = self.num_comp

        return X, num_comp
<|endoftext|>


<commit_before>#!/usr/bin/env/python3
import pygame
from helpers import ScottyLabsHandler
from courseNode import courseNode
import math
import random

def getDepartmentColor():
    idPrefix=courseID[0:2]
    idCourseLevel=int(course[3])

    csBaseColor=(204,0,0)
    eceBaseColor=(51,204,51)
    divisor=max((4-idCourseLevel),1)

    if idPrefix=="15":
        for i in csBaseColor:
            i=i//divisor
            return csBaseColor
    elif idPrefix=="18":
        for i in eceBaseColor:
            i=i // divisor
            return eceBaseColor

#put in init
def getMasterDict():
    result_dict = dict()
    course_handler = ScottyLabsHandler()
    for course in course_handler.courses:
        currentNode = courseNode(course_handler.courses[course]["name"])
        currentNode.addPrereqsFor(course_handler.getPreFor(course_handler.courses,course))
        currentNode.addPrereqsNeeded(course_handler.getPreNeeded(course_handler.courses,course))
        result_dict[course] = courseNode
    return result_dict

dict_example = getMasterDict()
print(dict_example['15-112'].prereqsFor)

class mainApp(object):
    def __init__(self):
        pygame.init()
        self.resolution=2
        self.height=(1080)//2
        self.width=(1920)//2
        self.screen=pygame.display.set_mode((self.width,self.height))
        self.background=pygame.Surface((self.width,self.height))
        self.background.fill((255,25,255))
        self.background=self.background.convert()
        self.isRunning=True
    def keyPressed(self):
        pass
    def mousePressed(self):
        pass
    def timerFired(self):
        pass
    def drawAll(self):
        self.background=self.background.convert()
        self.screen.blit(self.background,(0,0))
    def run(self):
        while self.isRunning:
            (mouseX,mouseY)=pygame.mouse.get_pos()
            for event in pygame.event.get():
                if event.type==pygame.QUIT:
                    self.isRunning=False
                elif event.type==pygame.KEYDOWN and event.key==pygame.K_ESCAPE:
                    self.isRunning=False
                elif event.type==pygame.KEYDOWN:
                    self.keyPressed()
                elif event.type==pygame.MOUSEBUTTONUP:
                    seslf.mousePressed()
            self.timerFired()
            self.drawAll()
            pygame.display.update()
        pygame.quit()

def getAllNLevelCourse(courseDict,n,hasPrereqs):
    L= []
    for courseId, courseNode in courseDict.items():
        if(courseId[3] == 1 and (hasPrereqs == True or len(courseNode.prereqsNeeded) == 0)):
            L.extend(courseID)
    return L

def setNodePositions(courseDict, width, height):
    cx = width//2
    cy = height//2
    radiusScalingFactor = 20;
    for courseId, courseNode in courseDict.items():
        angle = random.uniform(0, 2*pi)
        radius = courseNode.superScore*radiusScalingFactor
        posX = radius*cos(angle)
        posY = radius*sin(angle)
        courseNode.setPosition(posX,posY)

# if __name__=='__main__':
#     testApp=mainApp()
#     testApp.run()
<commit_msg>updated run<commit_after>#!/usr/bin/env/python3
import pygame
from helpers import ScottyLabsHandler
from courseNode import courseNode
import math
import random

def getDepartmentColor():
    idPrefix=courseID[0:2]
    idCourseLevel=int(course[3])

    csBaseColor=(204,0,0)
    eceBaseColor=(51,204,51)
    divisor=max((4-idCourseLevel),1)

    if idPrefix=="15":
        for i in csBaseColor:
            i=i//divisor
        return csBaseColor
    elif idPrefix=="18":
        for i in eceBaseColor:
            i=i // divisor
        return eceBaseColor

#put in init
def getMasterDict():
    result_dict = dict()
    course_handler = ScottyLabsHandler()
    for course in course_handler.courses:
        currentNode = courseNode(course_handler.courses[course]["name"])
        currentNode.addPrereqsFor(course_handler.getPreFor(course_handler.courses,course))
        currentNode.addPrereqsNeeded(course_handler.getPreNeeded(course_handler.courses,course))
        result_dict[course] = courseNode
    return result_dict

dict_example = getMasterDict()
#print(dict_example['15-112'].prereqsFor)

class mainApp(object):
    def __init__(self):
        pygame.init()
        self.resolution=2
        self.height=(1080)//2
        self.width=(1920)//2
        self.screen=pygame.display.set_mode((self.width,self.height))
        self.background=pygame.Surface((self.width,self.height))
        self.background.fill((255,25,255))
        self.background=self.background.convert()
        self.isRunning=True
    def keyPressed(self):
        pass
    def mousePressed(self):
        pass
    def timerFired(self):
        pass
    def drawAll(self):
        self.background=self.background.convert()
        self.screen.blit(self.background,(0,0))
    def run(self):
        while self.isRunning:
            (mouseX,mouseY)=pygame.mouse.get_pos()
            for event in pygame.event.get():
                if event.type==pygame.QUIT:
                    self.isRunning=False
                elif event.type==pygame.KEYDOWN and event.key==pygame.K_ESCAPE:
                    self.isRunning=False
                elif event.type==pygame.KEYDOWN:
                    self.keyPressed()
                elif event.type==pygame.MOUSEBUTTONUP:
                    seslf.mousePressed()
            self.timerFired()
            self.drawAll()
            pygame.display.update()
        pygame.quit()

def getAllNLevelCourse(courseDict,n,hasPrereqs):
    L= []
    for courseId, courseNode in courseDict.items():
        if(courseId[3] == 1 and (hasPrereqs == True or len(courseNode.prereqsNeeded) == 0)):
            L.extend(courseID)
    return L

def setNodePositions(courseDict, width, height):
    cx = width//2
    cy = height//2
    radiusScalingFactor = 20;
    for courseId, courseNode in courseDict.items():
        angle = random.uniform(0, 2*pi)
        radius = courseNode.superScore*radiusScalingFactor
        posX = radius*cos(angle)
        posY = radius*sin(angle)
        courseNode.setPosition(posX,posY)

#if __name__=='__main__':
#     testApp=mainApp()
#     testApp.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
@author Luke Campbell <LCampbell@ASAScience.com>
@file ion/processes/data/replay/replay_process.py
@date 06/14/12 13:31
@description Implementation for a replay process.
'''

from pyon.core.exception import BadRequest
from pyon.core.object import IonObjectDeserializer
from pyon.core.bootstrap import get_obj_registry
from pyon.util.arg_check import validate_is_instance
from pyon.util.log import log

from ion.services.dm.inventory.dataset_management_service import DatasetManagementService
from ion.services.dm.utility.granule import RecordDictionaryTool
from ion.util.time_utils import TimeUtils

from coverage_model import utils

from interface.services.dm.idataset_management_service import DatasetManagementServiceProcessClient
from interface.services.dm.ipubsub_management_service import PubsubManagementServiceProcessClient
from interface.services.dm.ireplay_process import BaseReplayProcess

from gevent.event import Event
from numbers import Number
import gevent
import numpy as np

class ReplayProcess(BaseReplayProcess):

    '''
    ReplayProcess - A process spawned for the purpose of replaying data
    --------------------------------------------------------------------------------
    Configurations
    ==============
    process:
      dataset_id:      ""     # Dataset to be replayed
      delivery_format: {}     # Delivery format to be replayed back (unused for now)
      query:
        start_time: 0         # Start time (index value) to be replayed
        end_time:   0         # End time (index value) to be replayed
        parameters: []        # List of parameters to form in the granule

    '''
    process_type  = 'standalone'
    publish_limit = 10
    dataset_id    = None
    delivery_format = {}
    start_time      = None
    end_time        = None
    stride_time     = None
    parameters      = None
    stream_id       = ''
    stream_def_id   = ''

    def __init__(self, *args, **kwargs):
        super(ReplayProcess,self).__init__(*args,**kwargs)
        self.deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
        self.publishing   = Event()
        self.play         = Event()
        self.end          = Event()

    def on_start(self):
        '''
        Starts the process
        '''
        log.info('Replay Process Started')
        super(ReplayProcess,self).on_start()
        dsm_cli = DatasetManagementServiceProcessClient(process=self)
        pubsub  = PubsubManagementServiceProcessClient(process=self)

        self.dataset_id      = self.CFG.get_safe('process.dataset_id', None)
        self.delivery_format = self.CFG.get_safe('process.delivery_format',{})
        self.start_time      = self.CFG.get_safe('process.query.start_time', None)
        self.end_time        = self.CFG.get_safe('process.query.end_time', None)
        self.stride_time     = self.CFG.get_safe('process.query.stride_time', None)
        self.parameters      = self.CFG.get_safe('process.query.parameters',None)
        self.publish_limit   = self.CFG.get_safe('process.query.publish_limit', 10)
        self.tdoa            = self.CFG.get_safe('process.query.tdoa',None)
        self.stream_id       = self.CFG.get_safe('process.publish_streams.output', '')
        self.stream_def      = pubsub.read_stream_definition(stream_id=self.stream_id)
        self.stream_def_id   = self.stream_def._id

        self.publishing.clear()
        self.play.set()
        self.end.clear()

        if self.dataset_id is None:
            raise BadRequest('dataset_id not specified')

        self.dataset = dsm_cli.read_dataset(self.dataset_id)
        self.pubsub = PubsubManagementServiceProcessClient(process=self)

    @classmethod
    def get_time_idx(cls, coverage, timeval):
        temporal_variable = coverage.temporal_parameter_name
        uom = coverage.get_parameter_context(temporal_variable).uom

        units = TimeUtils.ts_to_units(uom, timeval)

        idx = TimeUtils.get_relative_time(coverage, units)
        return idx

    @classmethod
    def _coverage_to_granule(cls, coverage, start_time=None, end_time=None, stride_time=None, fuzzy_stride=True, parameters=None, stream_def_id=None, tdoa=None):
        slice_ = slice(None) # Defaults to all values

        # Validations
        if start_time is not None:
            validate_is_instance(start_time, Number, 'start_time must be a number for striding.')
        if end_time is not None:
            validate_is_instance(end_time, Number, 'end_time must be a number for striding.')
        if stride_time is not None:
            validate_is_instance(stride_time, Number, 'stride_time must be a number for striding.')

        if tdoa is not None and isinstance(tdoa,slice):
            slice_ = tdoa

        elif stride_time is not None and not fuzzy_stride: # SLOW
            ugly_range = np.arange(start_time, end_time, stride_time)
            idx_values = [cls.get_time_idx(coverage,i) for i in ugly_range]
            idx_values = list(set(idx_values)) # Removing duplicates - also mixes the order of the list!!!
            idx_values.sort()
            slice_ = [idx_values]

        elif not (start_time is None and end_time is None):
            if start_time is not None:
                start_time = cls.get_time_idx(coverage,start_time)
            if end_time is not None:
                end_time = cls.get_time_idx(coverage,end_time)

            slice_ = slice(start_time,end_time,stride_time)
            log.info('Slice: %s', slice_)

        if stream_def_id:
            rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
        else:
            rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary)
        if parameters is not None:
            # TODO: Improve efficiency here
            fields = list(set(parameters).intersection(rdt.fields))
        else:
            fields = rdt.fields

        for field in fields:
            log.info( 'Slice is %s' , slice_)
            n = coverage.get_parameter_values(field,tdoa=slice_)
            if n is None:
                rdt[field] = [n]
            elif isinstance(n,np.ndarray):
                if coverage.get_data_extents(field)[0] != coverage.num_timesteps:
                    log.error("Misformed coverage detected, padding with fill_value")
                    arr_len = utils.slice_shape(slice_, (coverage.num_timesteps,))[0]
                    fill_arr = np.empty(arr_len - n.shape[0] , dtype=n.dtype)
                    fill_arr.fill(coverage.get_parameter_context(field).fill_value)
                    n = np.append(n,fill_arr)
                rdt[field] = n
            else:
                rdt[field] = [n]
        return rdt

    def execute_retrieve(self):
        '''
        execute_retrieve Executes a retrieval and returns the result
        as a value in lieu of publishing it on a stream
        '''
        try:
            coverage = DatasetManagementService._get_coverage(self.dataset_id,mode='r')
            if coverage.num_timesteps == 0:
                log.info('Reading from an empty coverage')
                rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary)
            else:
                rdt = self._coverage_to_granule(coverage=coverage,start_time=self.start_time, end_time=self.end_time, stride_time=self.stride_time, parameters=self.parameters,tdoa=self.tdoa)
        except Exception as e:
            import traceback
            traceback.print_exc(e)
            raise BadRequest('Problems reading from the coverage')
        finally:
            coverage.close(timeout=5)
        return rdt.to_granule()

    def execute_replay(self):
        '''
        execute_replay Performs a replay and publishes the results on a stream.
        '''
        if self.publishing.is_set():
            return False
        gevent.spawn(self.replay)
        return True

    def replay(self):
        self.publishing.set() # Minimal state, supposed to prevent two instances of the same process from replaying on the same stream
        for rdt in self._replay():
            if self.end.is_set():
                return
            self.play.wait()
            self.output.publish(rdt.to_granule())

        self.publishing.clear()
        return

    def pause(self):
        self.play.clear()

    def resume(self):
        self.play.set()

    def stop(self):
        self.end.set()

    @classmethod
    def get_last_values(cls, dataset_id, number_of_points):
        coverage = DatasetManagementService._get_coverage(dataset_id,mode='r')
        if coverage.num_timesteps < number_of_points:
            if coverage.num_timesteps == 0:
                rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary)
                return rdt.to_granule()
            number_of_points = coverage.num_timesteps
        rdt = cls._coverage_to_granule(coverage,tdoa=slice(-number_of_points,None))
        coverage.close(timeout=5)

        return rdt.to_granule()

    def _replay(self):
        coverage = DatasetManagementService._get_coverage(self.dataset_id,mode='r')
        rdt = self._coverage_to_granule(coverage=coverage, start_time=self.start_time, end_time=self.end_time, stride_time=self.stride_time, parameters=self.parameters, stream_def_id=self.stream_def_id)
        elements = len(rdt)

        for i in xrange(elements / self.publish_limit):
            outgoing = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
            fields = self.parameters or outgoing.fields
            for field in fields:
                v = rdt[field]
                if v is not None:
                    outgoing[field] = v[(i*self.publish_limit) : ((i+1)*self.publish_limit)]
            yield outgoing
        coverage.close(timeout=5)
        return

<commit_msg>Better error handling for retrieving against corrupt coverages.<commit_after>#!/usr/bin/env python
'''
@author Luke Campbell <LCampbell@ASAScience.com>
@file ion/processes/data/replay/replay_process.py
@date 06/14/12 13:31
@description Implementation for a replay process.
'''

from pyon.core.exception import BadRequest
from pyon.core.object import IonObjectDeserializer
from pyon.core.bootstrap import get_obj_registry
from pyon.util.arg_check import validate_is_instance
from pyon.util.log import log

from ion.services.dm.inventory.dataset_management_service import DatasetManagementService
from ion.services.dm.utility.granule import RecordDictionaryTool
from ion.util.time_utils import TimeUtils

from coverage_model import utils

from interface.services.dm.idataset_management_service import DatasetManagementServiceProcessClient
from interface.services.dm.ipubsub_management_service import PubsubManagementServiceProcessClient
from interface.services.dm.ireplay_process import BaseReplayProcess

from pyon.core.exception import CorruptionError

from gevent.event import Event
from numbers import Number
import gevent
import numpy as np

class ReplayProcess(BaseReplayProcess):

    '''
    ReplayProcess - A process spawned for the purpose of replaying data
    --------------------------------------------------------------------------------
    Configurations
    ==============
    process:
      dataset_id:      ""     # Dataset to be replayed
      delivery_format: {}     # Delivery format to be replayed back (unused for now)
      query:
        start_time: 0         # Start time (index value) to be replayed
        end_time:   0         # End time (index value) to be replayed
        parameters: []        # List of parameters to form in the granule

    '''
    process_type  = 'standalone'
    publish_limit = 10
    dataset_id    = None
    delivery_format = {}
    start_time      = None
    end_time        = None
    stride_time     = None
    parameters      = None
    stream_id       = ''
    stream_def_id   = ''

    def __init__(self, *args, **kwargs):
        super(ReplayProcess,self).__init__(*args,**kwargs)
        self.deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
        self.publishing   = Event()
        self.play         = Event()
        self.end          = Event()

    def on_start(self):
        '''
        Starts the process
        '''
        log.info('Replay Process Started')
        super(ReplayProcess,self).on_start()
        dsm_cli = DatasetManagementServiceProcessClient(process=self)
        pubsub  = PubsubManagementServiceProcessClient(process=self)

        self.dataset_id      = self.CFG.get_safe('process.dataset_id', None)
        self.delivery_format = self.CFG.get_safe('process.delivery_format',{})
        self.start_time      = self.CFG.get_safe('process.query.start_time', None)
        self.end_time        = self.CFG.get_safe('process.query.end_time', None)
        self.stride_time     = self.CFG.get_safe('process.query.stride_time', None)
        self.parameters      = self.CFG.get_safe('process.query.parameters',None)
        self.publish_limit   = self.CFG.get_safe('process.query.publish_limit', 10)
        self.tdoa            = self.CFG.get_safe('process.query.tdoa',None)
        self.stream_id       = self.CFG.get_safe('process.publish_streams.output', '')
        self.stream_def      = pubsub.read_stream_definition(stream_id=self.stream_id)
        self.stream_def_id   = self.stream_def._id

        self.publishing.clear()
        self.play.set()
        self.end.clear()

        if self.dataset_id is None:
            raise BadRequest('dataset_id not specified')

        self.dataset = dsm_cli.read_dataset(self.dataset_id)
        self.pubsub = PubsubManagementServiceProcessClient(process=self)

    @classmethod
    def get_time_idx(cls, coverage, timeval):
        temporal_variable = coverage.temporal_parameter_name
        uom = coverage.get_parameter_context(temporal_variable).uom

        units = TimeUtils.ts_to_units(uom, timeval)

        idx = TimeUtils.get_relative_time(coverage, units)
        return idx

    @classmethod
    def _coverage_to_granule(cls, coverage, start_time=None, end_time=None, stride_time=None, fuzzy_stride=True, parameters=None, stream_def_id=None, tdoa=None):
        slice_ = slice(None) # Defaults to all values

        # Validations
        if start_time is not None:
            validate_is_instance(start_time, Number, 'start_time must be a number for striding.')
        if end_time is not None:
            validate_is_instance(end_time, Number, 'end_time must be a number for striding.')
        if stride_time is not None:
            validate_is_instance(stride_time, Number, 'stride_time must be a number for striding.')

        if tdoa is not None and isinstance(tdoa,slice):
            slice_ = tdoa

        elif stride_time is not None and not fuzzy_stride: # SLOW
            ugly_range = np.arange(start_time, end_time, stride_time)
            idx_values = [cls.get_time_idx(coverage,i) for i in ugly_range]
            idx_values = list(set(idx_values)) # Removing duplicates - also mixes the order of the list!!!
            idx_values.sort()
            slice_ = [idx_values]

        elif not (start_time is None and end_time is None):
            if start_time is not None:
                start_time = cls.get_time_idx(coverage,start_time)
            if end_time is not None:
                end_time = cls.get_time_idx(coverage,end_time)

            slice_ = slice(start_time,end_time,stride_time)
            log.info('Slice: %s', slice_)

        if stream_def_id:
            rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
        else:
            rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary)
        if parameters is not None:
            # TODO: Improve efficiency here
            fields = list(set(parameters).intersection(rdt.fields))
        else:
            fields = rdt.fields

        for field in fields:
            log.info( 'Slice is %s' , slice_)
            n = coverage.get_parameter_values(field,tdoa=slice_)
            if n is None:
                rdt[field] = [n]
            elif isinstance(n,np.ndarray):
                if coverage.get_data_extents(field)[0] < coverage.num_timesteps:
                    log.error("Misformed coverage detected, padding with fill_value")
                    arr_len = utils.slice_shape(slice_, (coverage.num_timesteps,))[0]
                    fill_arr = np.empty(arr_len - n.shape[0] , dtype=n.dtype)
                    fill_arr.fill(coverage.get_parameter_context(field).fill_value)
                    n = np.append(n,fill_arr)
                elif coverage.get_data_extents(field)[0] > coverage.num_timesteps:
                    raise CorruptionError('The coverage is corrupted:\n\tfield: %s\n\textents: %s\n\ttimesteps: %s' % (field, coverage.get_data_extents(field), coverage.num_timesteps))
                rdt[field] = n
            else:
                rdt[field] = [n]
        return rdt

    def execute_retrieve(self):
        '''
        execute_retrieve Executes a retrieval and returns the result
        as a value in lieu of publishing it on a stream
        '''
        try:
            coverage = DatasetManagementService._get_coverage(self.dataset_id,mode='r')
            if coverage.num_timesteps == 0:
                log.info('Reading from an empty coverage')
                rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary)
            else:
                rdt = self._coverage_to_granule(coverage=coverage,start_time=self.start_time, end_time=self.end_time, stride_time=self.stride_time, parameters=self.parameters,tdoa=self.tdoa)
        except Exception as e:
            import traceback
            traceback.print_exc(e)
            raise BadRequest('Problems reading from the coverage')
        finally:
            coverage.close(timeout=5)
        return rdt.to_granule()

    def execute_replay(self):
        '''
        execute_replay Performs a replay and publishes the results on a stream.
        '''
        if self.publishing.is_set():
            return False
        gevent.spawn(self.replay)
        return True

    def replay(self):
        self.publishing.set() # Minimal state, supposed to prevent two instances of the same process from replaying on the same stream
        for rdt in self._replay():
            if self.end.is_set():
                return
            self.play.wait()
            self.output.publish(rdt.to_granule())

        self.publishing.clear()
        return

    def pause(self):
        self.play.clear()

    def resume(self):
        self.play.set()

    def stop(self):
        self.end.set()

    @classmethod
    def get_last_values(cls, dataset_id, number_of_points):
        coverage = DatasetManagementService._get_coverage(dataset_id,mode='r')
        if coverage.num_timesteps < number_of_points:
            if coverage.num_timesteps == 0:
                rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary)
                return rdt.to_granule()
            number_of_points = coverage.num_timesteps
        rdt = cls._coverage_to_granule(coverage,tdoa=slice(-number_of_points,None))
        coverage.close(timeout=5)

        return rdt.to_granule()

    def _replay(self):
        coverage = DatasetManagementService._get_coverage(self.dataset_id,mode='r')
        rdt = self._coverage_to_granule(coverage=coverage, start_time=self.start_time, end_time=self.end_time, stride_time=self.stride_time, parameters=self.parameters, stream_def_id=self.stream_def_id)
        elements = len(rdt)

        for i in xrange(elements / self.publish_limit):
            outgoing = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
            fields = self.parameters or outgoing.fields
            for field in fields:
                v = rdt[field]
                if v is not None:
                    outgoing[field] = v[(i*self.publish_limit) : ((i+1)*self.publish_limit)]
            yield outgoing
        coverage.close(timeout=5)
        return

<|endoftext|>


<commit_before>import base64, re, zlib
from xmlrpc.client import ServerProxy
from domain.api.model import to_model

LANGUAGE = 'en'
NEWLINE_PATTERN = re.compile(r'(\r\n|\r|\n)')
OPENSUBTITLES_URL = 'http://api.opensubtitles.org/xml-rpc'
OPENSUBTITLES_UA = 'subvoc v1.0'

def ensure_success(resp):
    if resp.get('status').split()[0] != '200':
        raise RuntimeError("received status {}".format(resp.get('status')))

def create_client():
    return ServerProxy(OPENSUBTITLES_URL, allow_none=True)

class OpenSubtitles:

    def __init__(self, credentials, create_client=create_client):
        self.token = None
        self.credentials = credentials
        self.xmlrpc = create_client()

    def login(self):
        resp = self.xmlrpc.LogIn(self.credentials[0], self.credentials[1], LANGUAGE, OPENSUBTITLES_UA)
        ensure_success(resp)
        self.token = resp.get('token')

    def find(self, query):
        if not self.token:
            self.login()

        resp = self.xmlrpc.SearchSubtitles(self.token, [query], [ { 'limit': 500 } ])
        ensure_success(resp)

        return [to_model(item) for item in resp.get('data')]

    def find_by_query(self, query):
        return self.find({ 'query': query, 'sublanguageid': 'eng' })

    def find_subtitles_for_movie(self, movie_id):
        return self.find({ 'imdbid': movie_id, 'sublanguageid': 'eng' })

    def load_text(self, subtitle):
        if not self.token:
            self.login()

        resp = self.xmlrpc.DownloadSubtitles(self.token, [subtitle.id])
        ensure_success(resp)

        text = resp.get('data')[0].get('data')
        text = base64.standard_b64decode(text)
        text = zlib.decompress(text, 47)
        text = str(text, subtitle.encoding)
        text = re.sub(NEWLINE_PATTERN, '\n', text)

        return text
<commit_msg>remove UTF-8 BOM<commit_after>import base64, codecs, re, zlib
from xmlrpc.client import ServerProxy
from domain.api.model import to_model

LANGUAGE = 'en'
NEWLINE_PATTERN = re.compile(r'(\r\n|\r|\n)')
OPENSUBTITLES_URL = 'http://api.opensubtitles.org/xml-rpc'
OPENSUBTITLES_UA = 'subvoc v1.0'
UNICODE_BOM = u'\N{ZERO WIDTH NO-BREAK SPACE}'

def ensure_success(resp):
    if resp.get('status').split()[0] != '200':
        raise RuntimeError("received status {}".format(resp.get('status')))

def create_client():
    return ServerProxy(OPENSUBTITLES_URL, allow_none=True)

class OpenSubtitles:

    def __init__(self, credentials, create_client=create_client):
        self.token = None
        self.credentials = credentials
        self.xmlrpc = create_client()

    def login(self):
        resp = self.xmlrpc.LogIn(self.credentials[0], self.credentials[1], LANGUAGE, OPENSUBTITLES_UA)
        ensure_success(resp)
        self.token = resp.get('token')

    def find(self, query):
        if not self.token:
            self.login()

        resp = self.xmlrpc.SearchSubtitles(self.token, [query], [ { 'limit': 500 } ])
        ensure_success(resp)

        return [to_model(item) for item in resp.get('data')]

    def find_by_query(self, query):
        return self.find({ 'query': query, 'sublanguageid': 'eng' })

    def find_subtitles_for_movie(self, movie_id):
        return self.find({ 'imdbid': movie_id, 'sublanguageid': 'eng' })

    def load_text(self, subtitle):
        if not self.token:
            self.login()

        resp = self.xmlrpc.DownloadSubtitles(self.token, [subtitle.id])
        ensure_success(resp)

        text = resp.get('data')[0].get('data')
        text = base64.standard_b64decode(text)
        text = zlib.decompress(text, 47)
        text = str(text, subtitle.encoding)
        text = text.lstrip(UNICODE_BOM)
        text = re.sub(NEWLINE_PATTERN, '\n', text)

        return text
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import apsw as lite
import os as os
import tempfile
import numpy as np
import pandas as pd
import bcolz as bcz

from .Tools import log
from .Config import cf
from .Exceptions import CamocoExistsError
from apsw import ConstraintError

class Camoco(object):

    '''
        Cache Money
    '''

    def __init__(self, name, type='Camoco'):
        # Set up our base directory
        self.log = log()
        self.type = type
        # A dataset already exists, return it
        self.db = self._database(name)
        try:
            (self.ID, self.name, self.description, self.type, self.added) = \
                self._database('Camoco', type='Camoco') \
                .cursor().execute(
                "SELECT rowid, * FROM datasets WHERE name = ? AND type = ?",
                (name, type)
            ).fetchone()
            cur = self.db.cursor()
            cur.execute('''
                CREATE TABLE IF NOT EXISTS globals (
                    key TEXT,
                    val TEXT
                );
                CREATE UNIQUE INDEX IF NOT EXISTS uniqkey ON globals(key)
                ''')
        except TypeError:
            raise TypeError('{}.{} does not exist'.format(type, name))

    def _database(self, dbname, type=None):
        # This lets us grab databases for other types
        if type is None:
            type = self.type
        # return a connection if exists
        return lite.Connection(
            os.path.expanduser(
                os.path.join(
                    cf.options.basedir,
                    'databases',
                    "{}.{}.db".format(type, dbname)
                )
            )
        )

    def _bcolz(self, tblname, dbname=None, type=None, df=None, blaze=False):
    # Suppress the warning until the next wersion
        import warnings
        #from flask.exthook import ExtDeprecationWarning
        #warnings.simplefilter('ignore',ExtDeprecationWarning)
        warnings.simplefilter('ignore',FutureWarning)
        import blaze as blz

        if type is None:
            type = self.type
        if dbname is None:
            dbname = self.name
        if df is None:
            # return the dataframe if it exists
            try:
                df = bcz.open(
                    os.path.expanduser(
                        os.path.join(
                            cf.options.basedir,
                            'databases',
                            "{}.{}.{}".format(type, dbname, tblname)
                        )
                    )
                )
            except IOError:
                return None
            else:
                if len(df) == 0:
                    df = pd.DataFrame()
                    if blaze:
                        df = blz.data(df)
                else:
                    if blaze:
                        df = blz.data(df)
                    else:
                        df = df.todataframe()
                if not blaze and 'idx' in df.columns.values:
                    df.set_index('idx', drop=True, inplace=True)
                    df.index.name = None
                return df

        else:
            if not(df.index.dtype_str == 'int64') and not(df.empty):
                df = df.copy()
                df['idx'] = df.index
            if isinstance(df,pd.DataFrame):
                path = os.path.expanduser(
                        os.path.join(
                            cf.options.basedir,
                            'databases',
                            "{}.{}.{}".format(type, dbname, tblname)
                        )
                    )
                if df.empty:
                    bcz.fromiter((),dtype=np.int32,mode='w',count=0,rootdir=path)
                else:
                    bcz.ctable.fromdataframe(df,mode='w',rootdir=path)

            if 'idx' in df.columns.values:
                del df
            return

    def _raw_coex(self,scores,significance_threshold):
        path = os.path.expanduser(
                os.path.join(
                    cf.options.basedir,
                    'databases',
                    "{}.{}.{}".format(self.type, self.name, 'coex')
                )
            )
        self._global('current_significance_threshold',significance_threshold)
        sigs = scores >= significance_threshold
        return bcz.ctable(columns=[scores,sigs], names=['score','significant'], mode='w', rootdir=path)

    def _tmpfile(self):
        # returns a handle to a tmp file
        return tempfile.NamedTemporaryFile(
            dir=os.path.expanduser(
                os.path.join(
                    cf.options.basedir,
                    "tmp"
                )
            )
        )

    def _global(self, key, val=None):
        # Set the global for the dataset
        try:
            if val is not None:
                self.db.cursor().execute('''
                    INSERT OR REPLACE INTO globals
                    (key, val)VALUES (?, ?)''', (key, val)
                )
            else:
                return self.db.cursor().execute(
                    '''SELECT val FROM globals WHERE key = ?''', (key, )
                ).fetchone()[0]
        except TypeError:
            # It pains me to do, but but return none if key isn't in global
            # TODO: replace returning None with an exception
            return None

    def __getattr__(self, name):
        return self._global(name)

    def __del__(self):
        '''
            Destroy a Camoco object and associated files.
        '''
        pass

    @classmethod
    def create(cls, name, description, type='Camoco'):
        '''
            This is a class method to create a new camoco type object.
            It initializes base directory hierarchy
        '''
        basedir = os.path.realpath(
            os.path.expanduser(cf.options.basedir)
        )

        # Create the basedir if not exists
        try:
            os.makedirs(basedir, exist_ok=True)
            os.makedirs(os.path.join(basedir, "logs"), exist_ok=True)
            os.makedirs(os.path.join(basedir, "databases"), exist_ok=True)
            os.makedirs(os.path.join(basedir, "analyses"), exist_ok=True)
            os.makedirs(os.path.join(basedir, "tmp"), exist_ok=True)
        except Exception:
            log('Could not create files in {}', basedir)
            raise
        try:
            # Create the base camoco database
            lite.Connection(
                os.path.join(basedir, 'databases', 'Camoco.Camoco.db')
            ).cursor().execute('''
                CREATE TABLE IF NOT EXISTS datasets (
                    name TEXT NOT NULL,
                    description TEXT,
                    type TEXT,
                    added datetime DEFAULT CURRENT_TIMESTAMP,
                    PRIMARY KEY(name, type)
                );
                INSERT OR IGNORE INTO datasets (name, description, type)
                    VALUES ('Camoco', 'Camoco base', 'Camoco');
                INSERT OR IGNORE INTO datasets (name, description, type)
                    VALUES (?, ?, ?)''', (name, description, type)
            )
        except ConstraintError:
            raise CamocoExistsError(
                'CAUTION! {}.{} Database already exists.', name, type
            )
        self = cls(name)
        return self
<commit_msg>Add busy timeout handler<commit_after>#!/usr/bin/env python3
import apsw as lite
import os as os
import tempfile
import numpy as np
import pandas as pd
import bcolz as bcz
import time
import time

from .Tools import log
from .Config import cf
from .Exceptions import CamocoExistsError
from apsw import ConstraintError

class Camoco(object):

    '''
        Cache Money
    '''

    def __init__(self, name, type='Camoco'):
        # Set up our base directory
        self.log = log()
        self.type = type
        # A dataset already exists, return it
        self.db = self._database(name)
        try:
            (self.ID, self.name, self.description, self.type, self.added) = \
                self._database('Camoco', type='Camoco') \
                .cursor().execute(
                "SELECT rowid, * FROM datasets WHERE name = ? AND type = ?",
                (name, type)
            ).fetchone()
            cur = self.db.cursor()
            cur.execute('''
                CREATE TABLE IF NOT EXISTS globals (
                    key TEXT,
                    val TEXT
                );
                CREATE UNIQUE INDEX IF NOT EXISTS uniqkey ON globals(key)
                ''')
        except TypeError:
            raise TypeError('{}.{} does not exist'.format(type, name))

    def _database(self, dbname, type=None):
        # This lets us grab databases for other types
        if type is None:
            type = self.type
        # return a connection if exists
        def busyhandler(num_prev_calls):
            if num_prev_calls >= 10:
                return False
            else:
                time.sleep(num_prev_calls)
                return True

        con = lite.Connection(
            os.path.expanduser(
                os.path.join(
                    cf.options.basedir,
                    'databases',
                    "{}.{}.db".format(type, dbname)
                )
            )
        )
        con.setbusyhandler(busyhandler)
        return con

    def _bcolz(self, tblname, dbname=None, type=None, df=None, blaze=False):
    # Suppress the warning until the next wersion
        import warnings
        #from flask.exthook import ExtDeprecationWarning
        #warnings.simplefilter('ignore',ExtDeprecationWarning)
        warnings.simplefilter('ignore',FutureWarning)
        import blaze as blz

        if type is None:
            type = self.type
        if dbname is None:
            dbname = self.name
        if df is None:
            # return the dataframe if it exists
            try:
                df = bcz.open(
                    os.path.expanduser(
                        os.path.join(
                            cf.options.basedir,
                            'databases',
                            "{}.{}.{}".format(type, dbname, tblname)
                        )
                    )
                )
            except IOError:
                return None
            else:
                if len(df) == 0:
                    df = pd.DataFrame()
                    if blaze:
                        df = blz.data(df)
                else:
                    if blaze:
                        df = blz.data(df)
                    else:
                        df = df.todataframe()
                if not blaze and 'idx' in df.columns.values:
                    df.set_index('idx', drop=True, inplace=True)
                    df.index.name = None
                return df

        else:
            if not(df.index.dtype_str == 'int64') and not(df.empty):
                df = df.copy()
                df['idx'] = df.index
            if isinstance(df,pd.DataFrame):
                path = os.path.expanduser(
                        os.path.join(
                            cf.options.basedir,
                            'databases',
                            "{}.{}.{}".format(type, dbname, tblname)
                        )
                    )
                if df.empty:
                    bcz.fromiter((),dtype=np.int32,mode='w',count=0,rootdir=path)
                else:
                    bcz.ctable.fromdataframe(df,mode='w',rootdir=path)

            if 'idx' in df.columns.values:
                del df
            return

    def _raw_coex(self,scores,significance_threshold):
        path = os.path.expanduser(
                os.path.join(
                    cf.options.basedir,
                    'databases',
                    "{}.{}.{}".format(self.type, self.name, 'coex')
                )
            )
        self._global('current_significance_threshold',significance_threshold)
        sigs = scores >= significance_threshold
        return bcz.ctable(columns=[scores,sigs], names=['score','significant'], mode='w', rootdir=path)

    def _tmpfile(self):
        # returns a handle to a tmp file
        return tempfile.NamedTemporaryFile(
            dir=os.path.expanduser(
                os.path.join(
                    cf.options.basedir,
                    "tmp"
                )
            )
        )

    def _global(self, key, val=None):
        # Set the global for the dataset
        try:
            if val is not None:
                self.db.cursor().execute('''
                    INSERT OR REPLACE INTO globals
                    (key, val)VALUES (?, ?)''', (key, val)
                )
            else:
                return self.db.cursor().execute(
                    '''SELECT val FROM globals WHERE key = ?''', (key, )
                ).fetchone()[0]
        except TypeError:
            # It pains me to do, but but return none if key isn't in global
            # TODO: replace returning None with an exception
            return None

    def __getattr__(self, name):
        return self._global(name)

    def __del__(self):
        '''
            Destroy a Camoco object and associated files.
        '''
        pass

    @classmethod
    def create(cls, name, description, type='Camoco'):
        '''
            This is a class method to create a new camoco type object.
            It initializes base directory hierarchy
        '''
        basedir = os.path.realpath(
            os.path.expanduser(cf.options.basedir)
        )

        # Create the basedir if not exists
        try:
            os.makedirs(basedir, exist_ok=True)
            os.makedirs(os.path.join(basedir, "logs"), exist_ok=True)
            os.makedirs(os.path.join(basedir, "databases"), exist_ok=True)
            os.makedirs(os.path.join(basedir, "analyses"), exist_ok=True)
            os.makedirs(os.path.join(basedir, "tmp"), exist_ok=True)
        except Exception:
            log('Could not create files in {}', basedir)
            raise
        try:
            # Create the base camoco database
            lite.Connection(
                os.path.join(basedir, 'databases', 'Camoco.Camoco.db')
            ).cursor().execute('''
                CREATE TABLE IF NOT EXISTS datasets (
                    name TEXT NOT NULL,
                    description TEXT,
                    type TEXT,
                    added datetime DEFAULT CURRENT_TIMESTAMP,
                    PRIMARY KEY(name, type)
                );
                INSERT OR IGNORE INTO datasets (name, description, type)
                    VALUES ('Camoco', 'Camoco base', 'Camoco');
                INSERT OR IGNORE INTO datasets (name, description, type)
                    VALUES (?, ?, ?)''', (name, description, type)
            )
        except ConstraintError:
            raise CamocoExistsError(
                'CAUTION! {}.{} Database already exists.', name, type
            )
        self = cls(name)
        return self
<|endoftext|>


<commit_before># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import absolute_import, unicode_literals

import re
import sys

from simplesqlite.query import AttrList
from six.moves import zip

from .error import DatabaseError

class Model(object):
    connection = None
    hidden = False
    __attr_name_list = None

    @classmethod
    def get_table_name(cls):
        table_name = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", cls.__name__)
        table_name = re.sub(r"([a-z\d])([A-Z])", r"\1_\2", table_name)
        table_name = table_name.replace("-", "_").lower()

        if cls.hidden:
            return "_{:s}_".format(table_name)

        return table_name

    @classmethod
    def get_attr_name_list(cls):
        if cls.__attr_name_list:
            return cls.__attr_name_list

        attr_name_list = [
            attr_name
            for attr_name in cls.__dict__
            if not attr_name.startswith("__") and attr_name not in ("connection", "hidden")
        ]

        if sys.version_info[:2] >= (3, 5):
            cls.__attr_name_list = attr_name_list
        else:
            cls.__attr_name_list = sorted(attr_name_list)

        return cls.__attr_name_list

    @classmethod
    def create(cls):
        cls.__validate()

        cls.connection.create_table(
            cls.get_table_name(),
            [
                "{attr} {constraints}".format(
                    attr=attr_name, constraints=cls.__dict__.get(attr_name)
                )
                for attr_name in cls.get_attr_name_list()
            ],
        )

    @classmethod
    def select(cls, where=None, extra=None):
        cls.__validate()

        result = cls.connection.select(
            select=AttrList(cls.get_attr_name_list()),
            table_name=cls.get_table_name(),
            where=where,
            extra=extra,
        )
        for record in result.fetchall():
            yield cls(
                **{attr_name: value for attr_name, value in zip(cls.get_attr_name_list(), record)}
            )

    @classmethod
    def insert(cls, model_obj):
        cls.__validate()

        if type(model_obj).__name__ != cls.__name__:
            raise TypeError("unexpected type: expected={}".format(cls.__name__))

        cls.connection.insert(
            cls.get_table_name(),
            [getattr(model_obj, attr_name) for attr_name in cls.get_attr_name_list()],
        )

    def __init__(self, *args, **kwargs):
        for attr_name in self.get_attr_name_list():
            setattr(self, attr_name, kwargs.get(attr_name))

    def __repr__(self):
        return "{name:s}: {attributes:s}".format(
            name=type(self).__name__,
            attributes=", ".join(
                [
                    "{}={}".format(attr_name, getattr(self, attr_name))
                    for attr_name in self.get_attr_name_list()
                ]
            ),
        )

    @classmethod
    def __validate(cls):
        if cls.connection is None:
            raise DatabaseError("SimpleSQLite connection required")
<commit_msg>Fix to exclude callable from the get_attr_name_list method return value<commit_after># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import absolute_import, unicode_literals

import re
import sys

from simplesqlite.query import AttrList
from six.moves import zip

from .error import DatabaseError

class Model(object):
    connection = None
    hidden = False
    __attr_name_list = None

    @classmethod
    def get_table_name(cls):
        table_name = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", cls.__name__)
        table_name = re.sub(r"([a-z\d])([A-Z])", r"\1_\2", table_name)
        table_name = table_name.replace("-", "_").lower()

        if cls.hidden:
            return "_{:s}_".format(table_name)

        return table_name

    @classmethod
    def get_attr_name_list(cls):
        if cls.__attr_name_list:
            return cls.__attr_name_list

        attr_name_list = [attr_name for attr_name in cls.__dict__ if cls.__is_attr(attr_name)]

        if sys.version_info[:2] >= (3, 5):
            cls.__attr_name_list = attr_name_list
        else:
            cls.__attr_name_list = sorted(attr_name_list)

        return cls.__attr_name_list

    @classmethod
    def create(cls):
        cls.__validate()

        cls.connection.create_table(
            cls.get_table_name(),
            [
                "{attr} {constraints}".format(
                    attr=attr_name, constraints=cls.__dict__.get(attr_name)
                )
                for attr_name in cls.get_attr_name_list()
            ],
        )

    @classmethod
    def select(cls, where=None, extra=None):
        cls.__validate()

        result = cls.connection.select(
            select=AttrList(cls.get_attr_name_list()),
            table_name=cls.get_table_name(),
            where=where,
            extra=extra,
        )
        for record in result.fetchall():
            yield cls(
                **{attr_name: value for attr_name, value in zip(cls.get_attr_name_list(), record)}
            )

    @classmethod
    def insert(cls, model_obj):
        cls.__validate()

        if type(model_obj).__name__ != cls.__name__:
            raise TypeError("unexpected type: expected={}".format(cls.__name__))

        cls.connection.insert(
            cls.get_table_name(),
            [getattr(model_obj, attr_name) for attr_name in cls.get_attr_name_list()],
        )

    def __init__(self, *args, **kwargs):
        for attr_name in self.get_attr_name_list():
            setattr(self, attr_name, kwargs.get(attr_name))

    def __repr__(self):
        return "{name:s}: {attributes:s}".format(
            name=type(self).__name__,
            attributes=", ".join(
                [
                    "{}={}".format(attr_name, getattr(self, attr_name))
                    for attr_name in self.get_attr_name_list()
                ]
            ),
        )

    @classmethod
    def __validate(cls):
        if cls.connection is None:
            raise DatabaseError("SimpleSQLite connection required")

    @classmethod
    def __is_attr(cls, attr_name):
        return (
            not attr_name.startswith("__")
            and attr_name not in ("connection", "hidden")
            and not callable(cls.__dict__.get(attr_name))
        )
<|endoftext|>


<commit_before>"""Run Grappler optimizers in the standalone mode.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import sys
import os
import json
import numpy as np

from google.protobuf import text_format
from absl import flags
from tensorflow.python.tools import freeze_graph
from tensorflow.core.framework import attr_value_pb2
from tensorflow.core.protobuf import device_properties_pb2
from tensorflow.core.protobuf import rewriter_config_pb2
from tensorflow.python.grappler import cluster as gcluster
from tensorflow.python.grappler import tf_optimizer
from tensorflow.python.lib.io import file_io

sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'node_modules/deeplearn-src/scripts/'))
import write_weights as deeplearn

flags.DEFINE_string('saved_model_dir', '', 'The saved model directory.')
flags.DEFINE_string('output_node_names', '',
                    'The names of the output nodes, comma separated.')
flags.DEFINE_string('output_graph', '', 'The name of the output graph file')
flags.DEFINE_string(
    'saved_model_tags', 'serve',
    'Tags of the MetaGraphDef to load, in comma separated string format.'
)

FLAGS = flags.FLAGS

# grappler optimization configuration for GPU
def get_cluster():
  named_device = device_properties_pb2.NamedDevice()
  named_device.name = '/GPU:0'
  named_device.properties.type = 'GPU'
  named_device.properties.environment['architecture'] = '4'
  cluster = gcluster.Cluster(devices=[named_device])
  return cluster

def load_graph(graph_filename):
  """Loads GraphDef. Returns Python Graph object."""
  with tf.gfile.Open(graph_filename, 'rb') as f:
    graph_def = tf.GraphDef()
    graph_def.ParseFromString(f.read())

  with tf.Graph().as_default() as graph:
#Set name to empty to avoid using the default name 'import'.
    tf.import_graph_def(graph_def, name='')

  for node in FLAGS.output_node_names.split(','):
    graph.add_to_collection('train_op',graph.get_operation_by_name(node.strip()))

  return graph

# validate the graph nodes are supported
def validate(nodes):
  ops = []
  for filename in os.listdir('src/operations/op_list'):
    with open('src/operations/op_list/' + filename) as json_data:
      ops += json.load(json_data)

  names = set([x['tfOpName'] for x in ops])
  not_supported = set([x.op for x in [x for x in nodes if (x.op not in
      names)]])
  return not_supported

def optimize_graph(graph):
  """Takes a Python Graph object and optimizes the graph."""
  rewriter_config = rewriter_config_pb2.RewriterConfig()
  rewriter_config.optimizers[:] = ['pruning', 'constfold', 'arithmetic',
                                'dependency', 'pruning',
                                'constfold', 'arithmetic','dependency']
  meta_graph = tf.train.export_meta_graph(
      graph_def=graph.as_graph_def(), graph=graph)
  optimized_graph = tf_optimizer.OptimizeGraph(
      rewriter_config, meta_graph, cluster=get_cluster())

  extract_weights(graph, optimized_graph)
  return optimize_graph

def extract_weights(graph, graph_def):
  """Takes a Python GraphDef object and extract the weights."""
  constants = [node for node in graph_def.node if node.op == 'Const']
  print('Writing weight file ' + FLAGS.output_graph + '...')
  const_manifest = []
  path = os.path.dirname(FLAGS.output_graph)

  with tf.Session(graph=graph) as sess:
    for const in constants:
      tensor = graph.get_tensor_by_name(const.name + ':0')
      value = tensor.eval(session=sess)
      if not(isinstance(value, np.ndarray)):
        value = np.array(value)

      const_manifest.append({'name': const.name, 'data': value})

      """Remove the binary array from tensor and save it to the external file."""
      const.attr["value"].tensor.ClearField('tensor_content')

  deeplearn.write_weights([const_manifest], path)

  file_io.atomic_write_string_to_file(
    os.path.abspath(FLAGS.output_graph), graph_def.SerializeToString())

  file_io.atomic_write_string_to_file(
    os.path.abspath(FLAGS.output_graph+'txt'), text_format.MessageToString(graph_def))

def main(_):

#Freeze the graph
  freeze_graph.freeze_graph('', '', True, '',
                          FLAGS.output_node_names,
                          '', '',
                          FLAGS.output_graph + '.frozen', True, '',
                          saved_model_tags=FLAGS.saved_model_tags,
                          input_saved_model_dir=FLAGS.saved_model_dir)
  graph = load_graph(FLAGS.output_graph + '.frozen')
  unsupported = validate(graph.as_graph_def().node)
  if len(unsupported) > 0:
    print('Unsupported Ops in the model\n' + ', '.join(unsupported))
  else:
    optimize_graph(graph)

if __name__ == '__main__':
  FLAGS(sys.argv)
  tf.app.run(main)
<commit_msg>Locale op_list from absolute path of the script. (#28)<commit_after>"""Run Grappler optimizers in the standalone mode.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import sys
import os
import json
import numpy as np

from google.protobuf import text_format
from absl import flags
from tensorflow.python.tools import freeze_graph
from tensorflow.core.framework import attr_value_pb2
from tensorflow.core.protobuf import device_properties_pb2
from tensorflow.core.protobuf import rewriter_config_pb2
from tensorflow.python.grappler import cluster as gcluster
from tensorflow.python.grappler import tf_optimizer
from tensorflow.python.lib.io import file_io

sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'node_modules/deeplearn-src/scripts/'))
import write_weights as deeplearn

flags.DEFINE_string('saved_model_dir', '', 'The saved model directory.')
flags.DEFINE_string('output_node_names', '',
                    'The names of the output nodes, comma separated.')
flags.DEFINE_string('output_graph', '', 'The name of the output graph file')
flags.DEFINE_string(
    'saved_model_tags', 'serve',
    'Tags of the MetaGraphDef to load, in comma separated string format.'
)

FLAGS = flags.FLAGS

# grappler optimization configuration for GPU
def get_cluster():
  named_device = device_properties_pb2.NamedDevice()
  named_device.name = '/GPU:0'
  named_device.properties.type = 'GPU'
  named_device.properties.environment['architecture'] = '4'
  cluster = gcluster.Cluster(devices=[named_device])
  return cluster

def load_graph(graph_filename):
  """Loads GraphDef. Returns Python Graph object."""
  with tf.gfile.Open(graph_filename, 'rb') as f:
    graph_def = tf.GraphDef()
    graph_def.ParseFromString(f.read())

  with tf.Graph().as_default() as graph:
#Set name to empty to avoid using the default name 'import'.
    tf.import_graph_def(graph_def, name='')

  for node in FLAGS.output_node_names.split(','):
    graph.add_to_collection('train_op',graph.get_operation_by_name(node.strip()))

  return graph

# validate the graph nodes are supported
def validate(nodes):
  ops = []
  op_list_path = 'src/operations/op_list/'
  dir_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
  for filename in os.listdir(os.path.join(dir_path, op_list_path)):
    with open(os.path.join(dir_path, op_list_path, filename)) as json_data:
      ops += json.load(json_data)

  names = set([x['tfOpName'] for x in ops])
  not_supported = set([x.op for x in [x for x in nodes if (x.op not in
      names)]])
  return not_supported

def optimize_graph(graph):
  """Takes a Python Graph object and optimizes the graph."""
  rewriter_config = rewriter_config_pb2.RewriterConfig()
  rewriter_config.optimizers[:] = ['pruning', 'constfold', 'arithmetic',
                                'dependency', 'pruning',
                                'constfold', 'arithmetic','dependency']
  meta_graph = tf.train.export_meta_graph(
      graph_def=graph.as_graph_def(), graph=graph)
  optimized_graph = tf_optimizer.OptimizeGraph(
      rewriter_config, meta_graph, cluster=get_cluster())

  extract_weights(graph, optimized_graph)
  return optimize_graph

def extract_weights(graph, graph_def):
  """Takes a Python GraphDef object and extract the weights."""
  constants = [node for node in graph_def.node if node.op == 'Const']
  print('Writing weight file ' + FLAGS.output_graph + '...')
  const_manifest = []
  path = os.path.dirname(FLAGS.output_graph)

  with tf.Session(graph=graph) as sess:
    for const in constants:
      tensor = graph.get_tensor_by_name(const.name + ':0')
      value = tensor.eval(session=sess)
      if not(isinstance(value, np.ndarray)):
        value = np.array(value)

      const_manifest.append({'name': const.name, 'data': value})

      """Remove the binary array from tensor and save it to the external file."""
      const.attr["value"].tensor.ClearField('tensor_content')

  deeplearn.write_weights([const_manifest], path)

  file_io.atomic_write_string_to_file(
    os.path.abspath(FLAGS.output_graph), graph_def.SerializeToString())

  file_io.atomic_write_string_to_file(
    os.path.abspath(FLAGS.output_graph+'txt'), text_format.MessageToString(graph_def))

def main(_):

#Freeze the graph
  freeze_graph.freeze_graph('', '', True, '',
                          FLAGS.output_node_names,
                          '', '',
                          FLAGS.output_graph + '.frozen', True, '',
                          saved_model_tags=FLAGS.saved_model_tags,
                          input_saved_model_dir=FLAGS.saved_model_dir)
  graph = load_graph(FLAGS.output_graph + '.frozen')
  unsupported = validate(graph.as_graph_def().node)
  if len(unsupported) > 0:
    print('Unsupported Ops in the model\n' + ', '.join(unsupported))
  else:
    optimize_graph(graph)

if __name__ == '__main__':
  FLAGS(sys.argv)
  tf.app.run(main)
<|endoftext|>


<commit_before>import requests

from canis import oauth
from canis.song import Song
from canis.memoize import memoize

class NotFound(Exception):

	def __init__(self, song):
		self.song = song

def id_for_song(song):
	album_subq = '%20album:{}'.format(song.album) if song.album else ''
	query = 'q=track:{}%20artist:{}{}&type=track'.format(song.title, song.artist, album_subq)
	r = requests.get('https://api.spotify.com/v1/search?{}'.format(query))
	json = r.json()
	try:
		track = json['tracks']['items'][0]
	except Exception:
		raise NotFound(song)
	return track['uri']

@memoize
def playlist_id_for_name(name):
	url = 'https://api.spotify.com/v1/users/{}/playlists?limit=50'.format(oauth.user_id)
	r = requests.get(url, headers=headers())
	resp = r.json()
	for playlist in resp['items']:
		if playlist['name'] == name:
			return playlist['id']
	return create_playlist(name)

def create_playlist(name):
	url = 'https://api.spotify.com/v1/users/{}/playlists'.format(oauth.user_id)
	r = requests.post(url, json={'name': name, 'public': 'false'}, headers=headers())
	resp = r.json()
	return resp['id']

def add_song_to_playlist(song_id, playlist_id):
	url = 'https://api.spotify.com/v1/users/{}/playlists/{}/tracks'.format(oauth.user_id, playlist_id)
	r = requests.post(url, json={'uris': [song_id]}, headers=headers())

def headers():
	return {'Authorization': 'Bearer {}'.format(oauth.access_token)}<commit_msg>Add retry without album.<commit_after>import requests

from canis import oauth
from canis.song import Song
from canis.memoize import memoize

class NotFound(Exception):

	def __init__(self, song):
		self.song = song

def id_for_song(song):
	album_subq = '%20album:{}'.format(song.album) if song.album else ''
	query = 'q=track:{}%20artist:{}{}&type=track'.format(song.title, song.artist, album_subq)
	r = requests.get('https://api.spotify.com/v1/search?{}'.format(query))
	json = r.json()
	try:
		track = json['tracks']['items'][0]
	except Exception:
		# Sometimes album is incorrect, try again without album
		if song.album:
			stripped = Song(song.title, song.artist, None)
			return id_for_song(stripped)
		raise NotFound(song)
	return track['uri']

@memoize
def playlist_id_for_name(name):
	url = 'https://api.spotify.com/v1/users/{}/playlists?limit=50'.format(oauth.user_id)
	r = requests.get(url, headers=headers())
	resp = r.json()
	for playlist in resp['items']:
		if playlist['name'] == name:
			return playlist['id']
	return create_playlist(name)

def create_playlist(name):
	url = 'https://api.spotify.com/v1/users/{}/playlists'.format(oauth.user_id)
	r = requests.post(url, json={'name': name, 'public': 'false'}, headers=headers())
	resp = r.json()
	return resp['id']

def add_song_to_playlist(song_id, playlist_id):
	url = 'https://api.spotify.com/v1/users/{}/playlists/{}/tracks'.format(oauth.user_id, playlist_id)
	r = requests.post(url, json={'uris': [song_id]}, headers=headers())

def headers():
	return {'Authorization': 'Bearer {}'.format(oauth.access_token)}<|endoftext|>