<commit_before>from frankdux import FrankDux

frank = FrankDux()

# Generate the client library
# frankdux app:frank

@frank.register(int, int, returns=int)
def add(first_var, second_var):
    return first_var + second_var

@frank.register(int, int, returns=int)
def subtract(a, b):
    return a - b

if __name__ == "__main__":
    frank.run()
<commit_msg>list of strings<commit_after>from frankdux import FrankDux

frank = FrankDux()

# Generate the client library
# frankdux app:frank python

@frank.register(int, int, returns=int)
def add(first_var, second_var):
    return first_var + second_var

@frank.register(int, int, returns=int)
def subtract(a, b):
    return a - b

@frank.register(str, str, returns=str)
def concat_and_make_fun(a, b):
    return a + b + " you suck"

@frank.register([str])
def concat_all(list_of_str):
    pass

if __name__ == "__main__":
    frank.run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import tensorflow as tf
import tf_sparsenet
import numpy as np
import scipy.linalg

class TopoSparsenet(tf_sparsenet.Sparsenet):
    """Topographic Sparsenet with TensorFlow backend and a few methods for defining topologies."""

    def __init__(self, data, datatype="image", pca=None,
      dict_shape = (30,30), topo = None, lam_g=0.1,
      **kwargs):
        """
        Topographic Sparsenet inherits from Sparsenet. Its unique
        attributes give the dictionary a shape and define the relative
        weight of the topographic term in the cost function.
        The topology matrix g is defined by the topology object topo.

        Args:
        lam_g : float, defines weight of topography term
        dict_shape : tuple (len, wid) of ints specifying shape of dictionary
        """
        self.lam_g = lam_g
        self.epsilon = 0.0001 # to regularize derivative of square root
        self.dict_shape = dict_shape
        nunits = int(np.prod(self.dict_shape))
        self.topo = topo or topology((nunits, nunits))
        nunits = topo.ncomponents * nunits
        try:
            kwargs['lam']
            super().__init__(data, nunits=nunits, datatype = datatype, pca = pca, **kwargs)
        except KeyError:
            super().__init__(data, nunits=nunits, datatype = datatype, pca = pca, lam=0, **kwargs)

    def build_graph(self):
        self.g = tf.constant(self.topo.get_matrix(), dtype=tf.float32)

        self.infrate = tf.Variable(self.infrate, trainable=False)
        self.learnrate = tf.Variable(self.learnrate, trainable=False)

        self.phi = tf.Variable(tf.random_normal([self.nunits,self.stims.datasize]))
        self.acts = tf.Variable(tf.zeros([self.nunits,self.batch_size]))
        self.reset_acts = self.acts.assign(tf.zeros([self.nunits,self.batch_size]))

        self.X = tf.Variable(tf.zeros([self.batch_size, self.stims.datasize]), trainable=False)
        self.Xhat = tf.matmul(tf.transpose(self.acts), self.phi, name='Xhat')
        self.resid = self.X - self.Xhat
        self.mse = tf.reduce_sum(tf.square(self.resid))/self.batch_size/self.stims.datasize
        self.meanL1 = tf.reduce_sum(tf.abs(self.acts))/self.batch_size
        self.layer2 = tf.reduce_sum(tf.sqrt(tf.matmul(self.g,tf.square(self.acts),
            name='g_times_acts') + self.epsilon))/self.batch_size
        self.loss = 0.5*self.mse + (self.lam*self.meanL1 + self.lam_g*self.layer2)/self.stims.datasize

        inferer = tf.train.GradientDescentOptimizer(self.infrate)
        inf_step = tf.Variable(0, name='inf_step', trainable=False)
        self.inf_op = inferer.minimize(self.loss, global_step=inf_step, var_list=[self.acts])

        learner = tf.train.GradientDescentOptimizer(self.learnrate)
        learn_step = tf.Variable(0,name='learn_step', trainable=False)
        self.learn_op = learner.minimize(self.loss, global_step=learn_step, var_list=[self.phi])

        self.ma_variances = tf.Variable(tf.ones(self.nunits), trainable=False)
        self.gains = tf.Variable(tf.ones(self.nunits), trainable=False)
        _, self.variances = tf.nn.moments(self.acts, axes=[1])
        self.update_variance = self.ma_variances.assign((1.-self.var_avg_rate)*self.ma_variances + self.var_avg_rate*self.variances)
        self.update_gains = self.gains.assign(self.gains*tf.pow(self.var_goal/self.ma_variances, self.gain_rate))
        self.renorm_phi = self.phi.assign((tf.expand_dims(self.gains,dim=1)*tf.nn.l2_normalize(self.phi, dim=1,epsilon=1e-15)))

        gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.2)
        config = tf.ConfigProto(gpu_options=gpu_options)
        self.sess = tf.Session(config=config)

        self.sess.run(tf.global_variables_initializer())
        self.sess.run(self.phi.assign(tf.nn.l2_normalize(self.phi, dim=1, epsilon=1e-15)))

    def show_dict(self, cmap='RdBu_r', subset=None, layout=None, savestr=None):
        layout = layout or self.dict_shape
        super().show_dict(cmap, subset, layout, savestr)

    def show_dict(self, cmap='RdBu_r', layout=None, savestr=None):
        layout = layout or self.dict_shape
        ncomp = self.topo.ncomponents
        display = np.zeros([ncomp*layout[0],layout[1]])
        per_comp = np.prod(layout)
        for nn in range(ncomp):
            display[nn*layout[0]:(nn+1)*layout[0]] = self.stims.stimarray(Qs[nn*per_comp:(nn+1)*per_comp], layout=layout)

    def sort(self, *args, **kwargs):
        print("The topographic order is meaningful, don't sort it away!")

class topology():
    def __init__(self, shape, discs=True, torus=True, binary=True, sigma = 1.0, ncomponents = 1):
        """
        shape: (tuple) (nlayer2comp, nlayer1) shape of each component
        sigma : (float) defines stdev of default gaussian neighborhoods
        """
        self.shape = shape
        dict_side = int(np.sqrt(self.shape[1]))
        assert dict_side**2 == self.shape[1], 'Only square dictionaries supported.'
        self.dict_shape = (dict_side, dict_side)
        self.discs = discs
        self.torus = torus
        self.binary = binary
        self.sigma = sigma
        self.ncomponents = ncomponents

    def get_matrix(self):

        g = np.zeros(self.shape)

        if self.discs:
            g = self.make_discs(g, *self.shape)

        if self.ncomponents > 1:
            blocks = [g.copy() for ii in range(self.ncomponents)]
            g = scipy.linalg.block_diag(*blocks)

        if self.binary:
            g = self.binarize(g)

        return g

    def make_discs(self, g, nlayer2, nlayer1):
        sigsquared = self.sigma**2
        for i in range(nlayer2):
            for j in range(nlayer1):
                g[i, j] = np.exp(-self.distance(i, j)/(2 * sigsquared))
        return g

    def distance(self, i, j):
        """ This function measures the squared distance between element i and j. The distance
        here is the distance between element i and j once the row vector has been
        reshaped into a square matrix, treating the dictionary as a torus globally
        if torus is True."""

        rows, cols = self.dict_shape
        rowi = i // cols
        coli = i % cols
        rowj = j // cols
        colj = j % cols

        if self.torus:
            # global topology is a torus
            rowj = [rowj - rows, rowj, rowj + rows]
            colj = [colj - cols, colj, colj + cols]

            dist = []
            for r in rowj:
                for c in colj:
                    dist.append((rowi - r)**2 + (coli - c)**2)

            return np.min(dist)
        else:
            return (rowi - rowj)**2 + (coli - colj)**2

    def block_membership(self, i, j, width=5):
        """This returns 1 if j is in the ith block, otherwise 0. Currently only
        works for square dictionaries."""
        # FIXME: I think there's a bug here that makes the boundary conditions
        # and the sizes wrong
        size = self.dict_shape[0]
        if size != self.dict_shape[1]:
            raise NotImplementedError
        i = [i // size, i % size]
        j = [j // size, j % size]

        if (abs((i[0]%size)-(j[0]%size)) % (size-1) < width)  and (abs((i[1]%size)-(j[1]%size)) % (size-1) < width):
            return 1
        else:
            return 0

    def set_blocks(self, width=5):
        """Change the topography by making each second layer unit respond to
        a square block of layer one with given width. g becomes binary."""
        # FIXME: doesn't work because block_membership doesn't work
        self.g = np.zeros_like(self.g)
        nunits = np.prod(self.dict_shape)
        for i in range(nunits):
            for j in range(nunits):
                self.g[i, j] = self.block_membership(i, j, width)

    def binarize(self, g, thresh=1/2, width=None):
        if width is not None:
            thresh = np.exp(-width**2/(2*self.sigma**2))
        return np.array(g >= thresh, dtype=int)<commit_msg>fixed bug in show_dict<commit_after># -*- coding: utf-8 -*-

import tensorflow as tf
import tf_sparsenet
import numpy as np
import scipy.linalg
import matplotlib.pyplot as plt

class TopoSparsenet(tf_sparsenet.Sparsenet):
    """Topographic Sparsenet with TensorFlow backend and a few methods for defining topologies."""

    def __init__(self, data, datatype="image", pca=None,
      dict_shape = (30,30), topo = None, lam_g=0.1,
      **kwargs):
        """
        Topographic Sparsenet inherits from Sparsenet. Its unique
        attributes give the dictionary a shape and define the relative
        weight of the topographic term in the cost function.
        The topology matrix g is defined by the topology object topo.

        Args:
        lam_g : float, defines weight of topography term
        dict_shape : tuple (len, wid) of ints specifying shape of dictionary
        """
        self.lam_g = lam_g
        self.epsilon = 0.0001 # to regularize derivative of square root
        self.dict_shape = dict_shape
        nunits = int(np.prod(self.dict_shape))
        self.topo = topo or topology((nunits, nunits))
        nunits = topo.ncomponents * nunits
        try:
            kwargs['lam']
            super().__init__(data, nunits=nunits, datatype = datatype, pca = pca, **kwargs)
        except KeyError:
            super().__init__(data, nunits=nunits, datatype = datatype, pca = pca, lam=0, **kwargs)

    def build_graph(self):
        self.g = tf.constant(self.topo.get_matrix(), dtype=tf.float32)

        self.infrate = tf.Variable(self.infrate, trainable=False)
        self.learnrate = tf.Variable(self.learnrate, trainable=False)

        self.phi = tf.Variable(tf.random_normal([self.nunits,self.stims.datasize]))
        self.acts = tf.Variable(tf.zeros([self.nunits,self.batch_size]))
        self.reset_acts = self.acts.assign(tf.zeros([self.nunits,self.batch_size]))

        self.X = tf.Variable(tf.zeros([self.batch_size, self.stims.datasize]), trainable=False)
        self.Xhat = tf.matmul(tf.transpose(self.acts), self.phi, name='Xhat')
        self.resid = self.X - self.Xhat
        self.mse = tf.reduce_sum(tf.square(self.resid))/self.batch_size/self.stims.datasize
        self.meanL1 = tf.reduce_sum(tf.abs(self.acts))/self.batch_size
        self.layer2 = tf.reduce_sum(tf.sqrt(tf.matmul(self.g,tf.square(self.acts),
            name='g_times_acts') + self.epsilon))/self.batch_size
        self.loss = 0.5*self.mse + (self.lam*self.meanL1 + self.lam_g*self.layer2)/self.stims.datasize

        inferer = tf.train.GradientDescentOptimizer(self.infrate)
        inf_step = tf.Variable(0, name='inf_step', trainable=False)
        self.inf_op = inferer.minimize(self.loss, global_step=inf_step, var_list=[self.acts])

        learner = tf.train.GradientDescentOptimizer(self.learnrate)
        learn_step = tf.Variable(0,name='learn_step', trainable=False)
        self.learn_op = learner.minimize(self.loss, global_step=learn_step, var_list=[self.phi])

        self.ma_variances = tf.Variable(tf.ones(self.nunits), trainable=False)
        self.gains = tf.Variable(tf.ones(self.nunits), trainable=False)
        _, self.variances = tf.nn.moments(self.acts, axes=[1])
        self.update_variance = self.ma_variances.assign((1.-self.var_avg_rate)*self.ma_variances + self.var_avg_rate*self.variances)
        self.update_gains = self.gains.assign(self.gains*tf.pow(self.var_goal/self.ma_variances, self.gain_rate))
        self.renorm_phi = self.phi.assign((tf.expand_dims(self.gains,dim=1)*tf.nn.l2_normalize(self.phi, dim=1,epsilon=1e-15)))

        gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.2)
        config = tf.ConfigProto(gpu_options=gpu_options)
        self.sess = tf.Session(config=config)

        self.sess.run(tf.global_variables_initializer())
        self.sess.run(self.phi.assign(tf.nn.l2_normalize(self.phi, dim=1, epsilon=1e-15)))

    def show_dict(self, cmap='RdBu_r', subset=None, layout=None, savestr=None):
        layout = layout or self.dict_shape
        super().show_dict(cmap, subset, layout, savestr)

    def show_dict(self, cmap='RdBu_r', layout=None, savestr=None):
        Qs = self.Q
        layout = layout or self.dict_shape
        ncomp = self.topo.ncomponents
        display = np.zeros([ncomp*layout[0],layout[1]])
        per_comp = np.prod(layout)
        for nn in range(ncomp):
            display[nn*layout[0]:(nn+1)*layout[0]] = self.stims.stimarray(Qs[nn*per_comp:(nn+1)*per_comp], layout=layout)
        plt.figure()
        arrayplot = plt.imshow(display,interpolation='nearest', cmap=cmap, aspect='auto', origin='lower')
        plt.axis('off')
        plt.colorbar()
        if savestr is not None:
            plt.savefig(savestr, bbox_inches='tight')
        return display

    def sort(self, *args, **kwargs):
        print("The topographic order is meaningful, don't sort it away!")

class topology():
    def __init__(self, shape, discs=True, torus=True, binary=True, sigma = 1.0, ncomponents = 1):
        """
        shape: (tuple) (nlayer2comp, nlayer1) shape of each component
        sigma : (float) defines stdev of default gaussian neighborhoods
        """
        self.shape = shape
        dict_side = int(np.sqrt(self.shape[1]))
        assert dict_side**2 == self.shape[1], 'Only square dictionaries supported.'
        self.dict_shape = (dict_side, dict_side)
        self.discs = discs
        self.torus = torus
        self.binary = binary
        self.sigma = sigma
        self.ncomponents = ncomponents

    def get_matrix(self):

        g = np.zeros(self.shape)

        if self.discs:
            g = self.make_discs(g, *self.shape)

        if self.ncomponents > 1:
            blocks = [g.copy() for ii in range(self.ncomponents)]
            g = scipy.linalg.block_diag(*blocks)

        if self.binary:
            g = self.binarize(g)

        return g

    def make_discs(self, g, nlayer2, nlayer1):
        sigsquared = self.sigma**2
        for i in range(nlayer2):
            for j in range(nlayer1):
                g[i, j] = np.exp(-self.distance(i, j)/(2 * sigsquared))
        return g

    def distance(self, i, j):
        """ This function measures the squared distance between element i and j. The distance
        here is the distance between element i and j once the row vector has been
        reshaped into a square matrix, treating the dictionary as a torus globally
        if torus is True."""

        rows, cols = self.dict_shape
        rowi = i // cols
        coli = i % cols
        rowj = j // cols
        colj = j % cols

        if self.torus:
            # global topology is a torus
            rowj = [rowj - rows, rowj, rowj + rows]
            colj = [colj - cols, colj, colj + cols]

            dist = []
            for r in rowj:
                for c in colj:
                    dist.append((rowi - r)**2 + (coli - c)**2)

            return np.min(dist)
        else:
            return (rowi - rowj)**2 + (coli - colj)**2

    def block_membership(self, i, j, width=5):
        """This returns 1 if j is in the ith block, otherwise 0. Currently only
        works for square dictionaries."""
        # FIXME: I think there's a bug here that makes the boundary conditions
        # and the sizes wrong
        size = self.dict_shape[0]
        if size != self.dict_shape[1]:
            raise NotImplementedError
        i = [i // size, i % size]
        j = [j // size, j % size]

        if (abs((i[0]%size)-(j[0]%size)) % (size-1) < width)  and (abs((i[1]%size)-(j[1]%size)) % (size-1) < width):
            return 1
        else:
            return 0

    def set_blocks(self, width=5):
        """Change the topography by making each second layer unit respond to
        a square block of layer one with given width. g becomes binary."""
        # FIXME: doesn't work because block_membership doesn't work
        self.g = np.zeros_like(self.g)
        nunits = np.prod(self.dict_shape)
        for i in range(nunits):
            for j in range(nunits):
                self.g[i, j] = self.block_membership(i, j, width)

    def binarize(self, g, thresh=1/2, width=None):
        if width is not None:
            thresh = np.exp(-width**2/(2*self.sigma**2))
        return np.array(g >= thresh, dtype=int)<|endoftext|>


<commit_before>from google.appengine.ext import ndb
import datetime
import json
import pytz
import re

from consts.district_type import DistrictType
from consts.event_type import EventType

class Event(ndb.Model):
    """
    Events represent FIRST Robotics Competition events, both official and unofficial.
    key_name is like '2010ct'
    """
    name = ndb.StringProperty()
    event_type_enum = ndb.IntegerProperty(required=True)
    short_name = ndb.StringProperty(indexed=False)  # Should not contain "Regional" or "Division", like "Hartford"
    event_short = ndb.StringProperty(required=True, indexed=False)  # Smaller abbreviation like "CT"
    year = ndb.IntegerProperty(required=True)
    event_district_enum = ndb.IntegerProperty()
    start_date = ndb.DateTimeProperty()
    end_date = ndb.DateTimeProperty()
    venue = ndb.StringProperty(indexed=False)
    venue_address = ndb.StringProperty(indexed=False)  # We can scrape this.
    location = ndb.StringProperty(indexed=False)  # in the format "locality, region, country". similar to Team.address
    timezone_id = ndb.StringProperty()  # such as 'America/Los_Angeles' or 'Asia/Jerusalem'
    official = ndb.BooleanProperty(default=False)  # Is the event FIRST-official?
    first_eid = ndb.StringProperty()  # from USFIRST
    facebook_eid = ndb.StringProperty(indexed=False)  # from Facebook
    custom_hashtag = ndb.StringProperty(indexed=False) #Custom HashTag
    website = ndb.StringProperty(indexed=False)
    webcast_json = ndb.TextProperty(indexed=False)  # list of dicts, valid keys include 'type' and 'channel'
    matchstats_json = ndb.TextProperty(indexed=False)  # for OPR, DPR, CCWM, etc.
    rankings_json = ndb.TextProperty(indexed=False)
    alliance_selections_json = ndb.TextProperty(indexed=False)  # Formatted as: [{'picks': [captain, pick1, pick2, 'frc123', ...], 'declines':[decline1, decline2, ...] }, {'picks': [], 'declines': []}, ... ]
    district_points_json = ndb.TextProperty(indexed=False)

    created = ndb.DateTimeProperty(auto_now_add=True, indexed=False)
    updated = ndb.DateTimeProperty(auto_now=True, indexed=False)

    def __init__(self, *args, **kw):
        # store set of affected references referenced keys for cache clearing
        # keys must be model properties
        self._affected_references = {
            'key': set(),
            'year': set(),
            'event_district_abbrev': set(),
        }
        self._alliance_selections = None
        self._awards = None
        self._district_points = None
        self._matches = None
        self._matchstats = None
        self._rankings = None
        self._teams = None
        self._webcast = None
        self._updated_attrs = []  # Used in EventManipulator to track what changed
        super(Event, self).__init__(*args, **kw)

    @ndb.tasklet
    def get_awards_async(self):
        from models.award import Award
        award_keys = yield Award.query(Award.event == self.key).fetch_async(500, keys_only=True)
        self._awards = yield ndb.get_multi_async(award_keys)

    @property
    def alliance_selections(self):
        """
        Lazy load alliance_selections JSON
        """
        if self._alliance_selections is None:
            try:
                self._alliance_selections = json.loads(self.alliance_selections_json)
            except Exception, e:
                self._alliance_selections = None
        return self._alliance_selections

    @property
    def alliance_teams(self):
        """
        Load a list of team keys playing in elims
        """
        alliances = self.alliance_selections
        teams = []
        for alliance in alliances:
            for pick in alliance['picks']:
                teams.append(pick)
        return teams

    @property
    def awards(self):
        # This import is ugly, and maybe all the models should be in one file again -gregmarra 20121006
        from models.award import Award
        if self._awards is None:
            self.get_awards_async().wait()
        return self._awards

    @property
    def district_points(self):
        """
        Lazy load district_points JSON
        """
        if self._district_points is None:
            try:
                self._district_points = json.loads(self.district_points_json)
            except Exception, e:
                self._district_points = None
        return self._district_points

    @ndb.tasklet
    def get_matches_async(self):
        from models.match import Match
        match_keys = yield Match.query(Match.event == self.key).fetch_async(500, keys_only=True)
        self._matches = yield ndb.get_multi_async(match_keys)

    @property
    def matches(self):
        # This import is ugly, and maybe all the models should be in one file again -gregmarra 20121006
        from models.match import Match
        if self._matches is None:
            if self._matches is None:
                self.get_matches_async().wait()
        return self._matches

    def withinDays(self, negative_days_before, days_after):
        if not self.start_date or not self.end_date:
            return False
        now = datetime.datetime.now()
        if self.timezone_id is not None:
            tz = pytz.timezone(self.timezone_id)
            try:
                now = now + tz.utcoffset(now)
            except pytz.NonExistentTimeError:  # may happen during DST
                now = now + tz.utcoffset(now + datetime.timedelta(hours=1))  # add offset to get out of non-existant time
        after_start = self.start_date.date() + datetime.timedelta(days=negative_days_before) <= now.date()
        before_end = self.end_date.date() + datetime.timedelta(days=days_after) >= now.date()

        return (after_start and before_end)

    @property
    def now(self):
        if self.timezone_id is not None:
            return self.withinDays(0, 0)
        else:
            return self.within_a_day  # overestimate what is "now" if no timezone

    @property
    def within_a_day(self):
        return self.withinDays(-1, 1)

    @property
    def past(self):
        return self.end_date.date() < datetime.date.today() and not self.within_a_day

    @property
    def future(self):
        return self.start_date.date() > datetime.date.today() and not self.within_a_day

    @ndb.tasklet
    def get_teams_async(self):
        from models.event_team import EventTeam
        event_team_keys = yield EventTeam.query(EventTeam.event == self.key).fetch_async(500, keys_only=True)
        event_teams = yield ndb.get_multi_async(event_team_keys)
        team_keys = map(lambda event_team: event_team.team, event_teams)
        self._teams = yield ndb.get_multi_async(team_keys)

    @property
    def teams(self):
        if self._teams is None:
            self.get_teams_async().wait()
        return self._teams

    @ndb.toplevel
    def prepAwardsMatchesTeams(self):
        yield self.get_awards_async(), self.get_matches_async(), self.get_teams_async()

    @ndb.toplevel
    def prepTeams(self):
        yield self.get_teams_async()

    @ndb.toplevel
    def prepTeamsMatches(self):
        yield self.get_matches_async(), self.get_teams_async()

    @property
    def matchstats(self):
        """
        Lazy load parsing matchstats JSON
        """
        if self._matchstats is None:
            try:
                self._matchstats = json.loads(self.matchstats_json)
            except Exception, e:
                self._matchstats = None
        return self._matchstats

    @property
    def rankings(self):
        """
        Lazy load parsing rankings JSON
        """
        if self._rankings is None:
            try:
                self._rankings = json.loads(self.rankings_json)
            except Exception, e:
                self._rankings = None
        return self._rankings

    @property
    def venue_or_venue_from_address(self):
        if self.venue:
            return self.venue
        else:
            try:
                return self.venue_address.split('\r\n')[0]
            except:
                return None

    @property
    def webcast(self):
        """
        Lazy load parsing webcast JSON
        """
        if self._webcast is None:
            try:
                self._webcast = json.loads(self.webcast_json)
            except Exception, e:
                self._webcast = None
        return self._webcast

    @property
    def key_name(self):
        """
        Returns the string of the key_name of the Event object before writing it.
        """
        return str(self.year) + self.event_short

    @property
    def facebook_event_url(self):
        """
        Return a string of the Facebook Event URL.
        """
        return "http://www.facebook.com/event.php?eid=%s" % self.facebook_eid

    @property
    def details_url(self):
        """
        Returns the URL pattern for the link to this Event on TBA
        """
        return "/event/%s" % self.key_name

    @property
    def gameday_url(self):
        """
        Returns the URL pattern for the link to watch webcasts in Gameday
        """
        if self.webcast:
            gameday_link = '/gameday'
            view_num = 0
            for webcast in self.webcast:
                if view_num == 0:
                    gameday_link += '#'
                else:
                    gameday_link += '&'
                if 'type' in webcast and 'channel' in webcast:
                    gameday_link += 'view_' + str(view_num) + '=' + self.key_name + '-' + str(view_num + 1)
                view_num += 1
            return gameday_link
        else:
            return None

    @property
    def hashtag(self):
        """
        Return the hashtag used for the event.
        """
        if self.custom_hashtag:
            return self.custom_hashtag
        else:
            return "frc" + self.event_short

    # Depreciated, still here to keep GAE clean.
    webcast_url = ndb.StringProperty(indexed=False)

    @classmethod
    def validate_key_name(self, event_key):
        key_name_regex = re.compile(r'^[1-9]\d{3}[a-z]+[1-9]?$')
        match = re.match(key_name_regex, event_key)
        return True if match else False

    @property
    def event_district_str(self):
        return DistrictType.type_names.get(self.event_district_enum, None)

    @property
    def event_district_abbrev(self):
        return DistrictType.type_abbrevs.get(self.event_district_enum, None)

    @property
    def event_type_str(self):
        return EventType.type_names[self.event_type_enum]
<commit_msg>fix None error<commit_after>from google.appengine.ext import ndb
import datetime
import json
import pytz
import re

from consts.district_type import DistrictType
from consts.event_type import EventType

class Event(ndb.Model):
    """
    Events represent FIRST Robotics Competition events, both official and unofficial.
    key_name is like '2010ct'
    """
    name = ndb.StringProperty()
    event_type_enum = ndb.IntegerProperty(required=True)
    short_name = ndb.StringProperty(indexed=False)  # Should not contain "Regional" or "Division", like "Hartford"
    event_short = ndb.StringProperty(required=True, indexed=False)  # Smaller abbreviation like "CT"
    year = ndb.IntegerProperty(required=True)
    event_district_enum = ndb.IntegerProperty()
    start_date = ndb.DateTimeProperty()
    end_date = ndb.DateTimeProperty()
    venue = ndb.StringProperty(indexed=False)
    venue_address = ndb.StringProperty(indexed=False)  # We can scrape this.
    location = ndb.StringProperty(indexed=False)  # in the format "locality, region, country". similar to Team.address
    timezone_id = ndb.StringProperty()  # such as 'America/Los_Angeles' or 'Asia/Jerusalem'
    official = ndb.BooleanProperty(default=False)  # Is the event FIRST-official?
    first_eid = ndb.StringProperty()  # from USFIRST
    facebook_eid = ndb.StringProperty(indexed=False)  # from Facebook
    custom_hashtag = ndb.StringProperty(indexed=False) #Custom HashTag
    website = ndb.StringProperty(indexed=False)
    webcast_json = ndb.TextProperty(indexed=False)  # list of dicts, valid keys include 'type' and 'channel'
    matchstats_json = ndb.TextProperty(indexed=False)  # for OPR, DPR, CCWM, etc.
    rankings_json = ndb.TextProperty(indexed=False)
    alliance_selections_json = ndb.TextProperty(indexed=False)  # Formatted as: [{'picks': [captain, pick1, pick2, 'frc123', ...], 'declines':[decline1, decline2, ...] }, {'picks': [], 'declines': []}, ... ]
    district_points_json = ndb.TextProperty(indexed=False)

    created = ndb.DateTimeProperty(auto_now_add=True, indexed=False)
    updated = ndb.DateTimeProperty(auto_now=True, indexed=False)

    def __init__(self, *args, **kw):
        # store set of affected references referenced keys for cache clearing
        # keys must be model properties
        self._affected_references = {
            'key': set(),
            'year': set(),
            'event_district_abbrev': set(),
        }
        self._alliance_selections = None
        self._awards = None
        self._district_points = None
        self._matches = None
        self._matchstats = None
        self._rankings = None
        self._teams = None
        self._webcast = None
        self._updated_attrs = []  # Used in EventManipulator to track what changed
        super(Event, self).__init__(*args, **kw)

    @ndb.tasklet
    def get_awards_async(self):
        from models.award import Award
        award_keys = yield Award.query(Award.event == self.key).fetch_async(500, keys_only=True)
        self._awards = yield ndb.get_multi_async(award_keys)

    @property
    def alliance_selections(self):
        """
        Lazy load alliance_selections JSON
        """
        if self._alliance_selections is None:
            try:
                self._alliance_selections = json.loads(self.alliance_selections_json)
            except Exception, e:
                self._alliance_selections = None
        return self._alliance_selections

    @property
    def alliance_teams(self):
        """
        Load a list of team keys playing in elims
        """
        alliances = self.alliance_selections
        if alliances is None:
            return []
        teams = []
        for alliance in alliances:
            for pick in alliance['picks']:
                teams.append(pick)
        return teams

    @property
    def awards(self):
        # This import is ugly, and maybe all the models should be in one file again -gregmarra 20121006
        from models.award import Award
        if self._awards is None:
            self.get_awards_async().wait()
        return self._awards

    @property
    def district_points(self):
        """
        Lazy load district_points JSON
        """
        if self._district_points is None:
            try:
                self._district_points = json.loads(self.district_points_json)
            except Exception, e:
                self._district_points = None
        return self._district_points

    @ndb.tasklet
    def get_matches_async(self):
        from models.match import Match
        match_keys = yield Match.query(Match.event == self.key).fetch_async(500, keys_only=True)
        self._matches = yield ndb.get_multi_async(match_keys)

    @property
    def matches(self):
        # This import is ugly, and maybe all the models should be in one file again -gregmarra 20121006
        from models.match import Match
        if self._matches is None:
            if self._matches is None:
                self.get_matches_async().wait()
        return self._matches

    def withinDays(self, negative_days_before, days_after):
        if not self.start_date or not self.end_date:
            return False
        now = datetime.datetime.now()
        if self.timezone_id is not None:
            tz = pytz.timezone(self.timezone_id)
            try:
                now = now + tz.utcoffset(now)
            except pytz.NonExistentTimeError:  # may happen during DST
                now = now + tz.utcoffset(now + datetime.timedelta(hours=1))  # add offset to get out of non-existant time
        after_start = self.start_date.date() + datetime.timedelta(days=negative_days_before) <= now.date()
        before_end = self.end_date.date() + datetime.timedelta(days=days_after) >= now.date()

        return (after_start and before_end)

    @property
    def now(self):
        if self.timezone_id is not None:
            return self.withinDays(0, 0)
        else:
            return self.within_a_day  # overestimate what is "now" if no timezone

    @property
    def within_a_day(self):
        return self.withinDays(-1, 1)

    @property
    def past(self):
        return self.end_date.date() < datetime.date.today() and not self.within_a_day

    @property
    def future(self):
        return self.start_date.date() > datetime.date.today() and not self.within_a_day

    @ndb.tasklet
    def get_teams_async(self):
        from models.event_team import EventTeam
        event_team_keys = yield EventTeam.query(EventTeam.event == self.key).fetch_async(500, keys_only=True)
        event_teams = yield ndb.get_multi_async(event_team_keys)
        team_keys = map(lambda event_team: event_team.team, event_teams)
        self._teams = yield ndb.get_multi_async(team_keys)

    @property
    def teams(self):
        if self._teams is None:
            self.get_teams_async().wait()
        return self._teams

    @ndb.toplevel
    def prepAwardsMatchesTeams(self):
        yield self.get_awards_async(), self.get_matches_async(), self.get_teams_async()

    @ndb.toplevel
    def prepTeams(self):
        yield self.get_teams_async()

    @ndb.toplevel
    def prepTeamsMatches(self):
        yield self.get_matches_async(), self.get_teams_async()

    @property
    def matchstats(self):
        """
        Lazy load parsing matchstats JSON
        """
        if self._matchstats is None:
            try:
                self._matchstats = json.loads(self.matchstats_json)
            except Exception, e:
                self._matchstats = None
        return self._matchstats

    @property
    def rankings(self):
        """
        Lazy load parsing rankings JSON
        """
        if self._rankings is None:
            try:
                self._rankings = json.loads(self.rankings_json)
            except Exception, e:
                self._rankings = None
        return self._rankings

    @property
    def venue_or_venue_from_address(self):
        if self.venue:
            return self.venue
        else:
            try:
                return self.venue_address.split('\r\n')[0]
            except:
                return None

    @property
    def webcast(self):
        """
        Lazy load parsing webcast JSON
        """
        if self._webcast is None:
            try:
                self._webcast = json.loads(self.webcast_json)
            except Exception, e:
                self._webcast = None
        return self._webcast

    @property
    def key_name(self):
        """
        Returns the string of the key_name of the Event object before writing it.
        """
        return str(self.year) + self.event_short

    @property
    def facebook_event_url(self):
        """
        Return a string of the Facebook Event URL.
        """
        return "http://www.facebook.com/event.php?eid=%s" % self.facebook_eid

    @property
    def details_url(self):
        """
        Returns the URL pattern for the link to this Event on TBA
        """
        return "/event/%s" % self.key_name

    @property
    def gameday_url(self):
        """
        Returns the URL pattern for the link to watch webcasts in Gameday
        """
        if self.webcast:
            gameday_link = '/gameday'
            view_num = 0
            for webcast in self.webcast:
                if view_num == 0:
                    gameday_link += '#'
                else:
                    gameday_link += '&'
                if 'type' in webcast and 'channel' in webcast:
                    gameday_link += 'view_' + str(view_num) + '=' + self.key_name + '-' + str(view_num + 1)
                view_num += 1
            return gameday_link
        else:
            return None

    @property
    def hashtag(self):
        """
        Return the hashtag used for the event.
        """
        if self.custom_hashtag:
            return self.custom_hashtag
        else:
            return "frc" + self.event_short

    # Depreciated, still here to keep GAE clean.
    webcast_url = ndb.StringProperty(indexed=False)

    @classmethod
    def validate_key_name(self, event_key):
        key_name_regex = re.compile(r'^[1-9]\d{3}[a-z]+[1-9]?$')
        match = re.match(key_name_regex, event_key)
        return True if match else False

    @property
    def event_district_str(self):
        return DistrictType.type_names.get(self.event_district_enum, None)

    @property
    def event_district_abbrev(self):
        return DistrictType.type_abbrevs.get(self.event_district_enum, None)

    @property
    def event_type_str(self):
        return EventType.type_names[self.event_type_enum]
<|endoftext|>


<commit_before>import datetime
import multiprocessing

from django.conf import settings
from django.core.management.base import BaseCommand, CommandError

from core.models import WebPage, PageScrapeResult
from core.views import scrape_url

class Command(BaseCommand):
    help = "Poll all the urls and scrape the results"
    can_import_settings = True

    def handle(self, *args, **options):
        pool = multiprocessing.Pool(multiprocessing.cpu_count())
        today = int(datetime.date.today().strftime("%s"))
        now = datetime.datetime.now()
        curr_time = int(now.strftime("%s")) - now.second
        mins_passed = int((curr_time - today) / 60.0)
        for page in WebPage.objects.all():
            if mins_passed % page.interval == 0 or settings.DEBUG:
                pool.apply_async(scrape_url, (page, ))
        pool.close()
        pool.join()
<commit_msg>Don't loop over all the pages. Only those whose interval matches.<commit_after>import datetime
import multiprocessing

from django.conf import settings
from django.core.management.base import BaseCommand, CommandError

from core.models import WebPage, PageScrapeResult
from core.views import scrape_url

class Command(BaseCommand):
    help = "Poll all the urls and scrape the results"
    can_import_settings = True

    def handle(self, *args, **options):
        pool = multiprocessing.Pool(multiprocessing.cpu_count())
        today = int(datetime.date.today().strftime("%s"))
        now = datetime.datetime.now()
        curr_time = int(now.strftime("%s")) - now.second
        mins_passed = int((curr_time - today) / 60.0)
        intervals = WebPage.objects.filter(interval__lte=mins_passed)\
                                   .values('interval').\
                                   order_by('interval').distinct()
        for interval in intervals:
            if mins_passed % interval == 0 or settings.DEBUG:
                for page in WebPage.objects.filter(interval=interval):
                    pool.apply_async(scrape_url, (page, ))
        pool.close()
        pool.join()
<|endoftext|>


<commit_before>import os
import functools

from .warehouse import Vlermv

class CachedFunction(Vlermv):
    def __init__(self, warehouse, func):
        self.warehouse = warehouse
        raise NotImplementedError

    def __call__(self, *args, **kwargs):
        if args in self.warehouse:
            output = self.warehouse[args]
        else:
            try:
                result = func(*args, **kwargs)
            except Exception as error:
                output = error, None
            else:
                output = None, result
            self.warehouse[args] = output
        error, result = output
        if error == None:
            return result
        else:
            raise error

def cache(*args, **kwargs):
    '''
    Cache a function with a vlermv.Vlermv.

    Decorate the function with @cache(*args, **kwargs).
    The args and kwargs get passed to the Vlermv.
    For example::

        @cache('~/.http')
        def get(url):
            return requests.get(url, auth = ('username', 'password'))

    vlermv.Vlermv would ordinarily fail if
    no arguments were passed to it. If you pass no arguments
    to cache, the Vlermv directory argument (the one
    required argument) will be set to the name of the function.
    '''
    def _decorator(args, kwargs, func):
        if len(args) == 0:
            cachedir = func.__name__
        else:
            cachedir = os.path.expanduser(args[0])
            args = args[1:]
        warehouse = Vlermv(cachedir, *args, **kwargs)
    return functools.partial(_decorator, args, kwargs)
<commit_msg>cache<commit_after>import os

from .warehouse import Vlermv

class CachedFunction(Vlermv):
    def __init__(self, warehouse, func):
        self.warehouse = warehouse
        self.func = func

    def __call__(self, *args, **kwargs):
        if args in self.warehouse:
            output = self.warehouse[args]
        else:
            try:
                result = self.func(*args, **kwargs)
            except Exception as error:
                output = error, None
            else:
                output = None, result
            self.warehouse[args] = output
        error, result = output
        if error == None:
            return result
        else:
            raise error

def cache(*args, **kwargs):
    '''
    Cache a function with a vlermv.Vlermv.

    Decorate the function with @cache(*args, **kwargs).
    The args and kwargs get passed to the Vlermv.
    For example::

        @cache('~/.http')
        def get(url):
            return requests.get(url, auth = ('username', 'password'))

    vlermv.Vlermv would ordinarily fail if
    no arguments were passed to it. If you pass no arguments
    to cache, the Vlermv directory argument (the one
    required argument) will be set to the name of the function.
    '''
    def _decorator(func):
        if len(args) == 0:
            cachedir = func.__name__
        else:
            cachedir = os.path.expanduser(args[0])
            args = args[1:]
        warehouse = Vlermv(cachedir, *args, **kwargs)
        return CachedFunction(warehouse, func)
    return _decorator
<|endoftext|>


<commit_before>"""Intervals in Theano including special functions for sparsifying."""

from athenet.sparsifying.utils.numlike import Numlike
from theano import function
from theano import tensor as T
from theano.ifelse import ifelse

class Interval(Numlike):
    """Theano interval matrix class

    represents matrix of interval. Behaves like limited numpy.ndarray.

    .. note:: Should be treated as interval type with bounds as Theano nodes.
              Opetations on Interval create nodes in Theano graph. In order to
              read result of given operations, use eval method.
    """

    def __init__(self, lower, upper):
        """Creates interval.

        :lower: lower bound of Interval to be set
        :upper: upper bound of Interval to be set

        .. note:: lower must be lower than upper. It is not being checked.
        """
        self.lower = lower
        self.upper = upper

    def __getitem__(self, at):
        """Returns specified slice of interval as a interval.

        :at: Coordinates / slice to be taken.

        Does not copy data.
        """
        return Interval(self.lower[at], self.upper[at])

    def __setitem__(self, at, other):
        """Just like Theano set_subtensor function, but as a operator.

        :at: Coordinates / slice to be set.
        :other: Data to be put at 'at'

        """
        self.lower = T.set_subtensor(self.lower[at], other.lower)
        self.upper = T.set_subtensor(self.upper[at], other.upper)

    def __repr__(self):
        """Standard repr method."""
        return '[' + repr(self.lower) + ', ' + repr(self.upper) + ']'

    def __str__(self):
        """"Standard str method."""
        return '[' + str(self.lower) + ', ' + str(self.upper) + ']'

    def shape(self):
        """Returns shape of interval. Checks only 'lower' matrix.

        Note: does not require self.upper for computations. Therefore it is not
        safe, but faster."""
        return self.lower.shape

    def __add__(self, other):
        """Returns sum of two intervals.

        :other: Interval or numpy.ndarray to be added."""
        if isinstance(other, Interval):
            res_lower = self.lower + other.lower
            res_upper = self.upper + other.upper
        else:
            res_lower = self.lower + other
            res_upper = self.upper + other
        return Interval(res_lower, res_upper)

    __radd__ = __add__

    def __sub__(self, other):
        """Returns difference between two intervals.

        :other: Interval or numpy.ndarray to be subtracted."""
        if isinstance(other, Interval):
            res_lower = self.lower - other.upper
            res_upper = self.upper - other.lower
        else:
            res_lower = self.lower - other
            res_upper = self.upper - other
        return Interval(res_lower, res_upper)

    def __rsub__(self, other):
        """Returns diffeerence between number and interval.

        :other: A number that self will be subtracted from."""
        res_lower = other - self.upper
        res_upper = other - self.lower
        return Interval(res_lower, res_upper)

    def __mul__(self, other):
        """Returns product of two intervals.

        :other: Interval or numpy.ndarray to be multiplied."""
        if isinstance(other, Interval):
            ll = self.lower * other.lower
            lu = self.lower * other.upper
            ul = self.upper * other.lower
            uu = self.upper * other.upper
            l = T.minimum(ll, lu)
            l = T.minimum(l, ul)
            l = T.minimum(l, uu)
            u = T.maximum(ll, lu)
            u = T.maximum(u, ul)
            u = T.maximum(u, uu)
            return Interval(l, u)
        else:
            ll = self.lower * other
            uu = self.upper * other
            l = T.minimum(ll, uu)
            u = T.maximum(ll, uu)
            return Interval(l, u)

    __rmul__ = __mul__

    def __div__(self, other):
        """Returns quotient of self and other.

        :param other: Divisor.
        :type other: Interval or numpy.ndarray

        .. warning:: Divisor should not contain zero."""
        lower = self.lower
        upper = self.upper
        if isinstance(other, Interval):
            o_lower = other.lower
            o_upper = other.upper
            A = T.switch(T.gt(o_lower, 0.0), 1, 0)  # not(b_la), b_ua
            B = T.switch(T.gt(lower, 0.0), 1, 0)
            C = T.switch(T.gt(upper, 0.0), 1, 0)
            b_lb = T.or_(T.and_(A, B),
                         T.and_(1 - A, C))
            b_ub = T.or_(1 - T.or_(A, B),
                         T.and_(A, 1 - C))
            la = T.switch(A, lower, upper)
            ua = T.switch(A, upper, lower)
            lb = T.switch(b_lb, o_upper, o_lower)
            ub = T.switch(b_ub, o_upper, o_lower)
            return Interval(la / lb, ua / ub)
        else:
            if other > 0:
                return Interval(lower / other, upper / other)
            else:
                return Interval(upper / other, lower / other)

    def __rdiv__(self, other):
        """Returns quotient of other and self.

        :param other: Dividend.
        :type other: Interval or numpy.ndarray

        .. warning:: Divisor (self) should not contain zero."""
        if isinstance(other, Interval):
            # Should never happen. __div__ should be used instead.
            raise NotImplementedError
        else:
            lower = self.lower
            upper = self.upper
            l_o = (other > 0)
            if l_o:
                return Interval(other / upper, other / lower)
            else:
                return Interval(other / lower, other / upper)

    def _has_zero(self):
        """For any interval in Interval, returns whether is contains zero."""
        return T.and_(T.lt(self.lower, 0), T.gt(self.upper, 0))

    def reciprocal(self):
        """Returns reciprocal of the interval.

        It is a partial reciprocal function. Does not allow 0 to be within
        interval. Should not be treated as general reciprocal function."""
        # Note: Could be concidered whether not to use input check.
        # If 0 is within interval, returns 1/0 that, we hope, will throw
        # some exception on the device. Be careful with this.

        # Input check below causes program interrupt if any _has_zero happened.
        # return Interval(switch(self._has_zero(),
        #                       T.constant(1)/T.constant(0),
        #                       T.inv(self.upper)),
        #                T.inv(self.lower))
        return Interval(T.inv(self.upper), T.inv(self.lower))

    def neg(self):
        """For interval [a, b], returns interval [-b, -a]."""
        return Interval(T.neg(self.upper), T.neg(self.lower))

    def exp(self):
        """Returns interval representing the exponential of the interval."""
        return Interval(T.exp(self.lower), T.exp(self.upper))

    def square(self):
        """For interval I, returns I' such that for any x in I, I' contains
        x*x and no other.

        e.g [-1, 1].square() == [0, 1]"""
        lsq = self.lower * self.lower
        usq = self.upper * self.upper
        u = T.maximum(lsq, usq)
        l = T.switch(self._has_zero(), 0, T.minimum(lsq, usq))
        return Interval(l, u)

    def power(self, exponent):
        """For interval i, returns i^exponent.

        :exponent: Number to be passed as exponent to i^exponent.

        Note: If interval contains some elements lower/equal to 0, exponent
        should be integer."""
        # If You want to understand what is happening here, make plot of
        # f(x, y) = x^y domain. 'if's divide this domain with respect to
        # monocity.
        le = T.pow(self.lower, exponent)
        ue = T.pow(self.upper, exponent)
        l, u = None, None
        if isinstance(exponent, (int, long)):
            if exponent > 0:
                if exponent % 2 == 0:
                    l = T.switch(self._has_zero(), 0, T.minimum(le, ue))
                    u = T.maximum(le, ue)
                else:
                    l = le
                    u = ue
            else:
                if exponent % 2 == 0:
                    l = T.minimum(le, ue)
                    u = T.maximum(le, ue)
                else:
                    l = ue
                    u = le
        else:
            # Assumes self.lower >= 0. Otherwise it is incorrectly defined.
            # There is no check.
            if exponent > 0:
                l = le
                u = ue
            else:
                l = ue
                u = le
        return Interval(l, u)

    def dot(self, other):
        """Dot product of Interval(self) vector and a number array (other).

        :param other: Number array to be multiplied.
        :type other: numpy.ndarray.
        """
        # Requires project decision that could be better made after checking
        # number of weights, edges and neurons in considered networks.
        # TODO: Decide how to implement this. Decide whether not to consider
        # batch in this implementation as it will be probably only used for
        # batches.
        raise NotImplementedError

    def max(self, other):
        """Returns interval such that for any numbers (x, y) in a pair of
        corresponding intervals in (self, other) arrays, max(x, y) is in result
        and no other.

        :param other: Interval to be compared.
        :type other: Interval.
        """
        return Interval(T.maximum(self.lower, other.lower),
                        T.maximum(self.upper, other.upper))

    def reshape(self, shape, ndim=None):
        """Reshapes interval tensor like theano Tensor.

        :param shape: Something that can be converted to a symbolic vector of
                      integers.
        :param ndim: The length of the shape. Passing None here means for
                     Theano to try and guess the length of shape.
        """
        return Interval(self.lower.reshape(shape, ndim),
                        self.upper.reshape(shape, ndim))

    def flatten(self, ndim=1):
        """Flattens interval tensor like theano Tensor.

        :param ndim: The number of dimensions in the returned variable.
        :return: Variable with same dtype as x and outdim dimensions.
        :rtype: Variable with the same shape as x in the leading outdim-1
                dimensions, but with all remaining dimensions of x collapsed
                into the last dimension.
        """
        return Interval(self.lower.flatten(ndim),
                        self.upper.flatten(ndim))

    def eval(self, *eval_map):
        """Evaluates interval in terms of theano TensorType eval method.

        :*eval_map: map of Theano variables to be set, just like in
                    theano.tensor.dtensorX.eval method.
        Returns pair (lower, upper) of """
        has_args = (len(eval_map) != 0)
        if has_args:
            eval_map = eval_map[0]
            has_args = (len(eval_map) != 0)
        if not has_args:
            try:
                f = function([], [self.lower, self.upper])
                rlower, rupper = f()
                return (rlower, rupper)
            except:
                return (self.lower, self.upper)
        keys = eval_map.keys()
        values = eval_map.values()
        f = function(keys, [self.lower, self.upper])
        rlower, rupper = f(*values)
        return (rlower, rupper)
<commit_msg>sphinx style comments<commit_after>"""Intervals in Theano including special functions for sparsifying."""

from athenet.sparsifying.utils.numlike import Numlike
from theano import function
from theano import tensor as T
from theano.ifelse import ifelse

class Interval(Numlike):
    """Theano interval matrix class

    represents matrix of interval. Behaves like limited numpy.ndarray.

    .. note:: Should be treated as interval type with bounds as Theano nodes.
              Opetations on Interval create nodes in Theano graph. In order to
              read result of given operations, use eval method.
    """

    def __init__(self, lower, upper):
        """Creates interval.

        :param lower: lower bound of Interval to be set
        :param upper: upper bound of Interval to be set
        :type lower: numpy.ndarray or theano variable
        :type upper: numpy.ndarray or theano variable

        .. note:: lower must be lower than upper. It is not being checked.
        """
        self.lower = lower
        self.upper = upper

    def __getitem__(self, at):
        """Returns specified slice of interval as a interval.

        :param at: coordinates / slice to be taken
        :rtype: Interval

        .. note:: Does not copy data.
        """
        return Interval(self.lower[at], self.upper[at])

    def __setitem__(self, at, other):
        """Just like Theano set_subtensor function, but as a operator.

        :param at: coordinates / slice to be set
        :param other: data to be put at 'at'
        :type other: Interval
        """
        self.lower = T.set_subtensor(self.lower[at], other.lower)
        self.upper = T.set_subtensor(self.upper[at], other.upper)

    def __repr__(self):
        """Standard repr method."""
        return '[' + repr(self.lower) + ', ' + repr(self.upper) + ']'

    def __str__(self):
        """"Standard str method."""
        return '[' + str(self.lower) + ', ' + str(self.upper) + ']'

    def shape(self):
        """Returns shape of interval. Checks only 'lower' matrix.

        :rtype: tuple of integers

        .. note:: does not require self.upper for computations. Therefore it is
        not safe, but faster.
        """
        return self.lower.shape

    def __add__(self, other):
        """Returns sum of two intervals.

        :param other: matrix to be added
        :type other: Interval or numpy.ndarray
        :rtype: Interval
        """
        if isinstance(other, Interval):
            res_lower = self.lower + other.lower
            res_upper = self.upper + other.upper
        else:
            res_lower = self.lower + other
            res_upper = self.upper + other
        return Interval(res_lower, res_upper)

    __radd__ = __add__

    def __sub__(self, other):
        """Returns difference between two intervals.

        :param other: matrix to be subtracted
        :type other: Interval or numpy.ndarray
        :rtype: Interval
        """
        if isinstance(other, Interval):
            res_lower = self.lower - other.upper
            res_upper = self.upper - other.lower
        else:
            res_lower = self.lower - other
            res_upper = self.upper - other
        return Interval(res_lower, res_upper)

    def __rsub__(self, other):
        """Returns diffeerence between number and interval.

        :param other: A number matrix that self will be subtracted from.
        :type other: numpy.ndarray
        :rtype: Interval
        """
        res_lower = other - self.upper
        res_upper = other - self.lower
        return Interval(res_lower, res_upper)

    def __mul__(self, other):
        """Returns product of two intervals.

        :param other: matrix to be multiplied
        :type other: Interval or numpy.ndarray
        :rtype: Interval
        """
        if isinstance(other, Interval):
            ll = self.lower * other.lower
            lu = self.lower * other.upper
            ul = self.upper * other.lower
            uu = self.upper * other.upper
            l = T.minimum(ll, lu)
            l = T.minimum(l, ul)
            l = T.minimum(l, uu)
            u = T.maximum(ll, lu)
            u = T.maximum(u, ul)
            u = T.maximum(u, uu)
            return Interval(l, u)
        else:
            ll = self.lower * other
            uu = self.upper * other
            l = T.minimum(ll, uu)
            u = T.maximum(ll, uu)
            return Interval(l, u)

    __rmul__ = __mul__

    def __div__(self, other):
        """Returns quotient of self and other.

        :param other: divisor
        :type other: Interval or numpy.ndarray
        :rtype: Interval

        .. warning:: Divisor should not contain zero.
        """
        lower = self.lower
        upper = self.upper
        if isinstance(other, Interval):
            o_lower = other.lower
            o_upper = other.upper
            A = T.switch(T.gt(o_lower, 0.0), 1, 0)  # not(b_la), b_ua
            B = T.switch(T.gt(lower, 0.0), 1, 0)
            C = T.switch(T.gt(upper, 0.0), 1, 0)
            b_lb = T.or_(T.and_(A, B),
                         T.and_(1 - A, C))
            b_ub = T.or_(1 - T.or_(A, B),
                         T.and_(A, 1 - C))
            la = T.switch(A, lower, upper)
            ua = T.switch(A, upper, lower)
            lb = T.switch(b_lb, o_upper, o_lower)
            ub = T.switch(b_ub, o_upper, o_lower)
            return Interval(la / lb, ua / ub)
        else:
            if other > 0:
                return Interval(lower / other, upper / other)
            else:
                return Interval(upper / other, lower / other)

    def __rdiv__(self, other):
        """Returns quotient of other and self.

        :param other: dividend
        :type other: Interval or numpy.ndarray
        :rtype: Interval

        .. warning:: Divisor (self) should not contain zero."""
        if isinstance(other, Interval):
            # Should never happen. __div__ should be used instead.
            raise NotImplementedError
        else:
            lower = self.lower
            upper = self.upper
            l_o = (other > 0)
            if l_o:
                return Interval(other / upper, other / lower)
            else:
                return Interval(other / lower, other / upper)

    def _has_zero(self):
        """For any interval in Interval, returns whether is contains zero.

        :rtype: Boolean
        """
        return T.and_(T.lt(self.lower, 0), T.gt(self.upper, 0))

    def reciprocal(self):
        """Returns reciprocal of the interval.

        It is a partial reciprocal function. Does not allow 0 to be within
        interval. Should not be treated as general reciprocal function.

        :rtype: Interval
        """
        # Note: Could be concidered whether not to use input check.
        # If 0 is within interval, returns 1/0 that, we hope, will throw
        # some exception on the device. Be careful with this.

        # Input check below causes program interrupt if any _has_zero happened.
        # return Interval(switch(self._has_zero(),
        #                       T.constant(1)/T.constant(0),
        #                       T.inv(self.upper)),
        #                T.inv(self.lower))
        return Interval(T.inv(self.upper), T.inv(self.lower))

    def neg(self):
        """For interval [a, b], returns interval [-b, -a]."""
        return Interval(T.neg(self.upper), T.neg(self.lower))

    def exp(self):
        """Returns interval representing the exponential of the interval."""
        return Interval(T.exp(self.lower), T.exp(self.upper))

    def square(self):
        """For interval I, returns I' such that for any x in I, I' contains
        x*x and no other.
        :rtype: Interval

        :Example:

        >>> from athenet.sparsifying.utils import Interval
        >>> import numpy
        >>> a = numpy.array([-1])
        >>> b = numpy.array([1])
        >>> i = Interval(-1, 1)
        >>> s = i.square()
        >>> s.eval()
        (array([0]), array([1]))

        """
        lsq = self.lower * self.lower
        usq = self.upper * self.upper
        u = T.maximum(lsq, usq)
        l = T.switch(self._has_zero(), 0, T.minimum(lsq, usq))
        return Interval(l, u)

    def power(self, exponent):
        """For interval i, returns i^exponent.

        :param exponent: Number to be passed as exponent to i^exponent.
        :type exponent: integer or float
        :rtype: Interval

        .. note:: If interval contains some elements lower/equal to 0, exponent
                  should be integer."""
        # If You want to understand what is happening here, make plot of
        # f(x, y) = x^y domain. 'if's divide this domain with respect to
        # monocity.
        le = T.pow(self.lower, exponent)
        ue = T.pow(self.upper, exponent)
        l, u = None, None
        if isinstance(exponent, (int, long)):
            if exponent > 0:
                if exponent % 2 == 0:
                    l = T.switch(self._has_zero(), 0, T.minimum(le, ue))
                    u = T.maximum(le, ue)
                else:
                    l = le
                    u = ue
            else:
                if exponent % 2 == 0:
                    l = T.minimum(le, ue)
                    u = T.maximum(le, ue)
                else:
                    l = ue
                    u = le
        else:
            # Assumes self.lower >= 0. Otherwise it is incorrectly defined.
            # There is no check.
            if exponent > 0:
                l = le
                u = ue
            else:
                l = ue
                u = le
        return Interval(l, u)

    def dot(self, other):
        """Returns dot product of Interval(self) vector and a number array
        (other).

        :param other: number array to be multiplied
        :type other: numpy.ndarray
        :rtype: Interval
        """
        # Requires project decision that could be better made after checking
        # number of weights, edges and neurons in considered networks.
        # TODO: Decide how to implement this. Decide whether not to consider
        # batch in this implementation as it will be probably only used for
        # batches.
        raise NotImplementedError

    def max(self, other):
        """Returns interval such that for any numbers (x, y) in a pair of
        corresponding intervals in (self, other) arrays, max(x, y) is in result
        and no other.

        :param other: Interval to be compared
        :type other: Interval
        :rtype: Interval
        """
        return Interval(T.maximum(self.lower, other.lower),
                        T.maximum(self.upper, other.upper))

    def reshape(self, shape, ndim=None):
        """Reshapes interval tensor like theano Tensor.

        :param shape: Something that can be converted to a symbolic vector of
                      integers.
        :param ndim: The length of the shape. Passing None here means for
                     Theano to try and guess the length of shape.
        """
        return Interval(self.lower.reshape(shape, ndim),
                        self.upper.reshape(shape, ndim))

    def flatten(self, ndim=1):
        """Flattens interval tensor like theano Tensor.

        :param ndim: The number of dimensions in the returned variable.
        :return: Variable with same dtype as x and outdim dimensions.
        :rtype: Variable with the same shape as x in the leading outdim-1
                dimensions, but with all remaining dimensions of x collapsed
                into the last dimension.
        """
        return Interval(self.lower.flatten(ndim),
                        self.upper.flatten(ndim))

    def eval(self, *eval_map):
        """Evaluates interval in terms of theano TensorType eval method.

        :*eval_map: map of Theano variables to be set, just like in
                    theano.tensor.dtensorX.eval method

        :rtype: (lower, upper) pair of numpy.ndarrays
        """
        has_args = (len(eval_map) != 0)
        if has_args:
            eval_map = eval_map[0]
            has_args = (len(eval_map) != 0)
        if not has_args:
            try:
                f = function([], [self.lower, self.upper])
                rlower, rupper = f()
                return (rlower, rupper)
            except:
                return (self.lower, self.upper)
        keys = eval_map.keys()
        values = eval_map.values()
        f = function(keys, [self.lower, self.upper])
        rlower, rupper = f(*values)
        return (rlower, rupper)
<|endoftext|>


<commit_before>import hashlib
import random
import string
import sys
import uuid
from flask_babel import gettext

# required answerer attribute
# specifies which search query keywords triggers this answerer
keywords = ('random',)

random_int_max = 2**31

if sys.version_info[0] == 2:
    random_string_letters = string.lowercase + string.digits + string.uppercase
else:
    unicode = str
    random_string_letters = string.ascii_lowercase + string.digits + string.ascii_uppercase

def random_characters():
    return [random.choice(random_string_letters)
            for _ in range(random.randint(8, 32))]

def random_string():
    return u''.join(random_characters())

def random_float():
    return unicode(random.random())

def random_int():
    return unicode(random.randint(-random_int_max, random_int_max))

def random_sha256():
    m = hashlib.sha256()
    m.update(b''.join(random_characters()))
    return unicode(m.hexdigest())

def random_uuid():
    return unicode(uuid.uuid4())

random_types = {b'string': random_string,
                b'int': random_int,
                b'float': random_float,
                b'sha256': random_sha256,
                b'uuid': random_uuid}

# required answerer function
# can return a list of results (any result type) for a given query
def answer(query):
    parts = query.query.split()
    if len(parts) != 2:
        return []

    if parts[1] not in random_types:
        return []

    return [{'answer': random_types[parts[1]]()}]

# required answerer function
# returns information about the answerer
def self_info():
    return {'name': gettext('Random value generator'),
            'description': gettext('Generate different random values'),
            'examples': [u'random {}'.format(x.decode('utf-8')) for x in random_types]}
<commit_msg>Fix search error with query 'random sha256'<commit_after>import hashlib
import random
import string
import sys
import uuid
from flask_babel import gettext

# required answerer attribute
# specifies which search query keywords triggers this answerer
keywords = ('random',)

random_int_max = 2**31

if sys.version_info[0] == 2:
    random_string_letters = string.lowercase + string.digits + string.uppercase
else:
    unicode = str
    random_string_letters = string.ascii_lowercase + string.digits + string.ascii_uppercase

def random_characters():
    return [random.choice(random_string_letters)
            for _ in range(random.randint(8, 32))]

def random_string():
    return u''.join(random_characters())

def random_float():
    return unicode(random.random())

def random_int():
    return unicode(random.randint(-random_int_max, random_int_max))

def random_sha256():
    m = hashlib.sha256()
    m.update(''.join(random_characters()).encode())
    return unicode(m.hexdigest())

def random_uuid():
    return unicode(uuid.uuid4())

random_types = {b'string': random_string,
                b'int': random_int,
                b'float': random_float,
                b'sha256': random_sha256,
                b'uuid': random_uuid}

# required answerer function
# can return a list of results (any result type) for a given query
def answer(query):
    parts = query.query.split()
    if len(parts) != 2:
        return []

    if parts[1] not in random_types:
        return []

    return [{'answer': random_types[parts[1]]()}]

# required answerer function
# returns information about the answerer
def self_info():
    return {'name': gettext('Random value generator'),
            'description': gettext('Generate different random values'),
            'examples': [u'random {}'.format(x.decode('utf-8')) for x in random_types]}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from openerp import pooler
from openerp.report import report_sxw
from openerp.tools.translate import _
from osv import osv

from report_webkit.webkit_report import WebKitParser

class remittance_letter_parser(report_sxw.rml_parse):

    def __init__(self, cr, uid, name, context):
        super(remittance_letter_parser, self).__init__(cr, uid, name, context=context)
        self.__check_vouchers(cr, uid, context)
        self.localcontext.update({
            'debit_credit': self.get_debit_credit,
            'format_amount': self.format_amount,
            'message': self.get_message,
        })

    def __check_vouchers(self, cr, uid, context=None):
        """ This function check if the message for payment
        is set in the company settings and raise in the other case.
        """
        company_osv = self.pool.get('res.company')
        company_id = company_osv._company_default_get(cr, uid, 'account.voucher', context=context)
        company_br = company_osv.browse(cr, uid, company_id, context=context)
        if not company_br.message_voucher_validate:
            raise osv.except_osv(
                _('Error'),
                _('Please set the message for payments'
                  ' in your company settings.')
            )

    def get_debit_credit(self, br):
        return _('Debit') if br.type == 'debit' else _('Credit')

    def format_amount(self, amount, br):
        # little check
        if not amount:
            return '0.00'
        # shortcut
        position = br.currency_id.position
        symbol = br.currency_id.symbol
        # currency after
        if position == 'after':
            return '%s %s' % (amount, symbol)
        # currency before
        if position == 'before':
            return '%s %s' % (symbol, amount)
        return amount.strip()

    def get_message(self, this_br):
        company = this_br.company_id
        if not company:
            return ''
        iban = this_br.partner_bank_id.acc_number
        date = this_br.date
        if this_br.state == 'posted':
            return company.message_voucher_validate.replace('$iban', iban).replace('$date', str(date))
        return ''

class remittance_letter_report(WebKitParser):
    def remove_previous(self, cr, uid, ids, context=None):
        # get attachement model
        ir_att_osv = pooler.get_pool(cr.dbname).get('ir.attachment')
        # previous ids
        data_ids = ir_att_osv.search(
            cr,
            uid,
            [('res_model', '=', 'account.voucher'),
             ('res_id', 'in', ids)],
            context=context)
        # remove previous items
        ir_att_osv.unlink(cr, uid, data_ids)

    def create(self, cr, uid, ids, datas, context=None):
        # remove previous items
        self.remove_previous(cr, uid, ids, context=context)
        # call parent
        return super(remittance_letter_report, self).create(
            cr, uid, ids, datas, context)

remittance_letter_report('report.account_streamline.remittance_letter',
                         'account.voucher',
                         'addons/account_streamline/report/remittance_letter.mako',
                         parser=remittance_letter_parser)
<commit_msg>Don't fail when the IBAN or date can't be found - fix #29<commit_after># -*- coding: utf-8 -*-

from openerp import pooler
from openerp.report import report_sxw
from openerp.tools.translate import _
from osv import osv

from report_webkit.webkit_report import WebKitParser

class remittance_letter_parser(report_sxw.rml_parse):

    def __init__(self, cr, uid, name, context):
        super(remittance_letter_parser, self).__init__(cr, uid, name, context=context)
        self.__check_vouchers(cr, uid, context)
        self.localcontext.update({
            'debit_credit': self.get_debit_credit,
            'format_amount': self.format_amount,
            'message': self.get_message,
        })

    def __check_vouchers(self, cr, uid, context=None):
        """ This function check if the message for payment
        is set in the company settings and raise in the other case.
        """
        company_osv = self.pool.get('res.company')
        company_id = company_osv._company_default_get(cr, uid, 'account.voucher', context=context)
        company_br = company_osv.browse(cr, uid, company_id, context=context)
        if not company_br.message_voucher_validate:
            raise osv.except_osv(
                _('Error'),
                _('Please set the message for payments'
                  ' in your company settings.')
            )

    def get_debit_credit(self, br):
        return _('Debit') if br.type == 'debit' else _('Credit')

    def format_amount(self, amount, br):
        # little check
        if not amount:
            return '0.00'
        # shortcut
        position = br.currency_id.position
        symbol = br.currency_id.symbol
        # currency after
        if position == 'after':
            return '%s %s' % (amount, symbol)
        # currency before
        if position == 'before':
            return '%s %s' % (symbol, amount)
        return amount.strip()

    def get_message(self, this_br):
        company = this_br.company_id
        if not company:
            return ''

        bank = this_br.partner_bank_id
        if not bank:
            return ''

        iban = bank.acc_number or ''
        date = this_br.date or ''
        if this_br.state == 'posted':
            return company.message_voucher_validate.replace('$iban', iban).replace('$date', str(date))
        return ''

class remittance_letter_report(WebKitParser):
    def remove_previous(self, cr, uid, ids, context=None):
        # get attachement model
        ir_att_osv = pooler.get_pool(cr.dbname).get('ir.attachment')
        # previous ids
        data_ids = ir_att_osv.search(
            cr,
            uid,
            [('res_model', '=', 'account.voucher'),
             ('res_id', 'in', ids)],
            context=context)
        # remove previous items
        ir_att_osv.unlink(cr, uid, data_ids)

    def create(self, cr, uid, ids, datas, context=None):
        # remove previous items
        self.remove_previous(cr, uid, ids, context=context)
        # call parent
        return super(remittance_letter_report, self).create(
            cr, uid, ids, datas, context)

remittance_letter_report('report.account_streamline.remittance_letter',
                         'account.voucher',
                         'addons/account_streamline/report/remittance_letter.mako',
                         parser=remittance_letter_parser)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Default data records and associations, used to setup the database.
"""

# Place data

# Mapping of continent names to WOEID values. The WOEIDs were looked up from
# Yahoo index since they did not appear in Twitter API.
continentBase = {
    24865670: 'Africa',
    24865671: 'Asia',
    24865675: 'Europe',
    24865672: 'North America',
    55949069: 'Oceana',
    24865673: 'South America',
}

# A manual mapping of continents to countries, based on country data available
# in the Twitter API.
continentMapping = {
    'Africa': [
        'Nigeria',
        'Ecuador',
        'Ghana',
        'Algeria',
        'Dominican Republic',
        'Egypt',
        'South Africa',
        'Kenya'],
    'Asia': [
        'Qatar',
        'Kuwait',
        'Israel',
        'Korea',
        'Singapore',
        'Thailand',
        'Jordan',
        'Oman',
        'Bahrain',
        'Philippines',
        'Indonesia',
        'Saudi Arabia',
        'Turkey',
        'Japan',
        'Russia',
        'Pakistan',
        'United Arab Emirates',
        'Lebanon',
        'India',
        'Malaysia',
        'Vietnam'],
    'Europe': [
        'Italy',
        'France',
        'Ireland',
        'Norway',
        'Belarus',
        'Belgium',
        'Germany',
        'Poland',
        'Spain',
        'Ukraine',
        'Netherlands',
        'Denmark',
        'Sweden',
        'Switzerland',
        'Portugal',
        'United Kingdom',
        'Austria',
        'Latvia',
        'Greece'],
    'North America': [
        'Canada',
        'Panama',
        'Puerto Rico',
        'Guatemala',
        'United States'],
    'Oceana': [
        'Australia',
        'New Zealand'],
    'South America': [
        'Brazil',
        'Per',
        'Argentina',
        'Venezuela',
        'Chile',
        'Mexico',
        'Colombia'],
}
<commit_msg>fix: Restore Peru after removing unicode prefixes<commit_after># -*- coding: utf-8 -*-
"""
Default data records and associations, used to setup the database.
"""

# Place data

# Mapping of continent names to WOEID values. The WOEIDs were looked up from
# Yahoo index since they did not appear in Twitter API.
continentBase = {
    24865670: 'Africa',
    24865671: 'Asia',
    24865675: 'Europe',
    24865672: 'North America',
    55949069: 'Oceana',
    24865673: 'South America',
}

# A manual mapping of continents to countries, based on country data available
# in the Twitter API.
continentMapping = {
    'Africa': [
        'Nigeria',
        'Ecuador',
        'Ghana',
        'Algeria',
        'Dominican Republic',
        'Egypt',
        'South Africa',
        'Kenya'],
    'Asia': [
        'Qatar',
        'Kuwait',
        'Israel',
        'Korea',
        'Singapore',
        'Thailand',
        'Jordan',
        'Oman',
        'Bahrain',
        'Philippines',
        'Indonesia',
        'Saudi Arabia',
        'Turkey',
        'Japan',
        'Russia',
        'Pakistan',
        'United Arab Emirates',
        'Lebanon',
        'India',
        'Malaysia',
        'Vietnam'],
    'Europe': [
        'Italy',
        'France',
        'Ireland',
        'Norway',
        'Belarus',
        'Belgium',
        'Germany',
        'Poland',
        'Spain',
        'Ukraine',
        'Netherlands',
        'Denmark',
        'Sweden',
        'Switzerland',
        'Portugal',
        'United Kingdom',
        'Austria',
        'Latvia',
        'Greece'],
    'North America': [
        'Canada',
        'Panama',
        'Puerto Rico',
        'Guatemala',
        'United States'],
    'Oceana': [
        'Australia',
        'New Zealand'],
    'South America': [
        'Brazil',
        'Peru',
        'Argentina',
        'Venezuela',
        'Chile',
        'Mexico',
        'Colombia'],
}
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# rotate-to-s3.py
#
# ATonns Wed Mar 14 19:40:57 UTC 2012
import boto
import gzip
import httplib2
import os
import re
import signal
import time

logdir = "/var/www/logs"
filename = "healthguru.access.log"
pidfile = "/var/run/nginx.pid"
destination_bucket = "live-test-logs"

def getinstanceid():
  h = httplib2.Http()
  # get the instance id from the aws meta data
  resp, content = h.request("http://169.254.169.254/latest/meta-data/instance-id", "GET")
  # strip out the leading 'i-'
  match = re.search('i-(.+)',content)
  id = match.group(1)
  return id

def getpid(pidfile):
  p = open(pidfile,"r")
  # remove all trailing whitespace, especially the newline
  pid = p.readline().rstrip()
  return int(pid)

def compressfile(sourcefile,destinationfile):
  # source
  f_in = open(sourcefile, 'rb')
  # destination
  f_out = gzip.open(destinationfile, 'wb')
  # go
  f_out.writelines(f_in)
  f_out.close()
  f_in.close()

def uploadtos3(sourcefile,destinationfile):
  # connect to s3
  #c = boto.connect_s3(AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY)
  c = boto.connect_s3()
  # create/open bucket
  b = c.create_bucket(destination_bucket)
  # set the key
  from boto.s3.key import Key
  k = Key(b)
  k.key = destinationfile
  # upload the file
  k.set_contents_from_filename(sourcefile)

if __name__ == '__main__':
	# prep for rotate
	pid = getpid(pidfile)
	now = time.strftime('%Y%m%d-%H%M%S')
	oldname = logdir +"/"+                   filename
	newname = logdir +"/"+         now +'-'+ filename
	zipname = logdir +"/"+         now +'-'+ filename +'.gz'
	s3name = getinstanceid() +'-'+ now +'-'+ filename +'.gz'
	# rotate the logs
	os.rename(oldname,newname)
	# force nginx to logswitch
	os.kill(pid,signal.SIGUSR1)
	# wait for it to complete
	time.sleep(1)
	# gzip the file
	compressfile(newname,zipname)
	os.remove(newname)
	# push to s3
	uploadtos3(zipname,s3name)

<commit_msg>fixed using builtin id<commit_after>#!/usr/bin/python
#
# rotate-to-s3.py
#
# ATonns Wed Mar 14 19:40:57 UTC 2012
import boto
import gzip
import httplib2
import os
import re
import signal
import time

logdir = "/var/www/logs"
filename = "healthguru.access.log"
pidfile = "/var/run/nginx.pid"
destination_bucket = "live-test-logs"

def getinstanceid():
  h = httplib2.Http()
  # get the instance id from the aws meta data
  resp, content = h.request("http://169.254.169.254/latest/meta-data/instance-id", "GET")
  # strip out the leading 'i-'
  match = re.search('i-(.+)',content)
  instanceid = match.group(1)
  return instanceid

def getpid(pidfile):
  p = open(pidfile,"r")
  # remove all trailing whitespace, especially the newline
  pid = p.readline().rstrip()
  return int(pid)

def compressfile(sourcefile,destinationfile):
  # source
  f_in = open(sourcefile, 'rb')
  # destination
  f_out = gzip.open(destinationfile, 'wb')
  # go
  f_out.writelines(f_in)
  f_out.close()
  f_in.close()

def uploadtos3(sourcefile,destinationfile):
  # connect to s3
  #c = boto.connect_s3(AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY)
  c = boto.connect_s3()
  # create/open bucket
  b = c.create_bucket(destination_bucket)
  # set the key
  from boto.s3.key import Key
  k = Key(b)
  k.key = destinationfile
  # upload the file
  k.set_contents_from_filename(sourcefile)

if __name__ == '__main__':
	# prep for rotate
	pid = getpid(pidfile)
	now = time.strftime('%Y%m%d-%H%M%S')
	oldname = logdir +"/"+                   filename
	newname = logdir +"/"+         now +'-'+ filename
	zipname = logdir +"/"+         now +'-'+ filename +'.gz'
	s3name = getinstanceid() +'-'+ now +'-'+ filename +'.gz'
	# rotate the logs
	os.rename(oldname,newname)
	# force nginx to logswitch
	os.kill(pid,signal.SIGUSR1)
	# wait for it to complete
	time.sleep(1)
	# gzip the file
	compressfile(newname,zipname)
	os.remove(newname)
	# push to s3
	uploadtos3(zipname,s3name)

<|endoftext|>


<commit_before># Miriam Baumann, 2015-11-10
# Reads in a dataframe in the form of a simple text file and
# plots the mean of each column in the y- axis, and then saves the plot

# import libraries
import pandas as pd
import sys
import matplotlib as plt

def main ():
# assign variables
	filename = sys.argv[1]
	savename = sys.argv[2]

	# load the data
	data = pd.read_csv(filename, delimiter=',')

	# call process to get values from data
	values = process(data)

def process(data):
	#Defining what to do with the dataframe
	data.mean(axis = 1)
	#return(out)

def plot(plot, data):

	plt.plot(plot)
	#Saving the figure
	plt.savefig(savename)

	plt.show()

main()<commit_msg>Added a command to the process function<commit_after># Miriam Baumann, 2015-11-10
# Reads in a dataframe in the form of a simple text file and
# plots the mean of each column in the y- axis, and then saves the plot

# import libraries
import pandas as pd
import sys
import matplotlib as plt

def main ():
# assign variables
	filename = sys.argv[1]
	savename = sys.argv[2]

	# load the data
	data = pd.read_csv(filename, delimiter=',')

	# call process to get values from data
	values = process(command, data)

def process(command, data):
	#Defining what to do with the dataframe
	command = data.mean(axis = 1)

def plot(plot, data):

	plot = plt.plot(data)
	#Saving the figure
	plot = plt.savefig(savename)

	plt.show()

main()<|endoftext|>