<commit_before># Copyright (c) 2009-2010 Six Apart Ltd.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
#   this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of Six Apart Ltd. nor the names of its contributors may
#   be used to endorse or promote products derived from this software without
#   specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

"""

`DataObject` is a class of object that provides coding between object
attributes and dictionaries, suitable for

In `DataObject` is the mechanism for converting between dictionaries and
objects. These conversions are performed with aid of `Field` instances
declared on `DataObject` subclasses. `Field` classes reside in the
`remoteobjects.field` module.

"""

from copy import deepcopy
import logging

import remoteobjects.fields

classes_by_name = {}
classes_by_constant_field = {}

def find_by_name(name):
    """Finds and returns the DataObject subclass with the given name.

    Parameter `name` should be a bare class name with no module. If there is
    no class by that name, raises `KeyError`.

    """
    return classes_by_name[name]

class DataObjectMetaclass(type):
    """Metaclass for `DataObject` classes.

    This metaclass installs all `remoteobjects.fields.Property` instances
    declared as attributes of the new class, including all `Field` and `Link`
    instances.

    This metaclass also makes the new class findable through the
    `dataobject.find_by_name()` function.

    """

    def __new__(cls, name, bases, attrs):
        """Creates and returns a new `DataObject` class with its declared
        fields and name."""
        fields = {}
        new_fields = {}
        new_properties = {}

        # Inherit all the parent DataObject classes' fields.
        for base in bases:
            if isinstance(base, DataObjectMetaclass):
                fields.update(base.fields)

        # Move all the class's attributes that are Fields to the fields set.
        for attrname, field in attrs.items():
            if isinstance(field, remoteobjects.fields.Property):
                new_properties[attrname] = field
                if isinstance(field, remoteobjects.fields.Field):
                    new_fields[attrname] = field
            elif attrname in fields:
                # Throw out any parent fields that the subclass defined as
                # something other than a Field.
                del fields[attrname]

        fields.update(new_fields)
        attrs['fields'] = fields
        obj_cls = super(DataObjectMetaclass, cls).__new__(cls, name, bases, attrs)

        for field, value in new_properties.items():
            obj_cls.add_to_class(field, value)

        # Register the new class so Object fields can have forward-referenced it.
        classes_by_name[name] = obj_cls

        # Tell this class's fields what this class is, so they can find their
        # forward references later.
        for field in new_properties.values():
            field.of_cls = obj_cls

        return obj_cls

    def add_to_class(cls, name, value):
        try:
            value.install(name, cls)
        except (NotImplementedError, AttributeError):
            setattr(cls, name, value)

class DataObject(object):

    """An object that can be decoded from or encoded as a dictionary.

    DataObject subclasses should be declared with their different data
    attributes defined as instances of fields from the `remoteobjects.fields`
    module. For example:

    >>> from remoteobjects import DataObject, fields
    >>> class Asset(DataObject):
    ...     name    = fields.Field()
    ...     updated = fields.Datetime()
    ...     author  = fields.Object('Author')
    ...

    A DataObject's fields then provide the coding between live DataObject
    instances and dictionaries.

    """

    __metaclass__ = DataObjectMetaclass

    def __init__(self, **kwargs):
        """Initializes a new `DataObject` with the given field values."""
        self.api_data = {}
        self.__dict__.update(kwargs)

    def __eq__(self, other):
        """Returns whether two `DataObject` instances are equivalent.

        If the `DataObject` instances are of the same type and contain the
        same data in all their fields, the objects are equivalent.

        """
        if type(self) != type(other):
            return False
        for k, v in self.fields.iteritems():
            if isinstance(v, remoteobjects.fields.Field):
                if getattr(self, k) != getattr(other, k):
                    return False
        return True

    def __ne__(self, other):
        """Returns whether two `DataObject` instances are different.

        `DataObject` instances are different if they are not equivalent as
        determined through `__eq__()`.

        """
        return not self == other

    @classmethod
    def statefields(cls):
        return cls.fields.keys() + ['api_data']

    def __getstate__(self):
        return dict((k, self.__dict__[k]) for k in self.statefields()
            if k in self.__dict__)

    def get(self, attr, *args):
        return getattr(self, attr, *args)

    def __iter__(self):
        for key in self.fields.keys():
            yield key

    def to_dict(self):
        """Encodes the DataObject to a dictionary."""
        data = deepcopy(self.api_data)
        for field_name, field in self.fields.iteritems():
            value = getattr(self, field.attrname, None)
            if value is not None:
                data[field.api_name] = field.encode(value)
        return data

    @classmethod
    def from_dict(cls, data):
        """Decodes a dictionary into a new `DataObject` instance."""
        self = cls()
        self.update_from_dict(data)
        return self

    def update_from_dict(self, data):
        """Adds the content of a dictionary to this DataObject.

        Parameter `data` is the dictionary from which to update the object.

        Use this only when receiving newly updated or partial content for a
        DataObject; that is, when the data is from the outside data source and
        needs decoded through the object's fields. Data from "inside" your
        application should be added to an object manually by setting the
        object's attributes. Data that constitutes a new object should be
        turned into another object with `from_dict()`.

        """
        if not isinstance(data, dict):
            raise TypeError
        # Clear any local instance field data
        for k in self.fields.iterkeys():
            if k in self.__dict__:
                del self.__dict__[k]
        self.api_data = data

    @classmethod
    def subclass_with_constant_field(cls, fieldname, value):
        """Returns the closest subclass of this class that has a `Constant`
        field with the given value.

        Use this method in combination with the `fields.Constant` field class
        to find the most appropriate subclass of `cls` based on a content
        field. For example, if you have an ``Asset`` class, but want to
        declare subclasses with special behavior based on the ``kind`` field
        of the ``Asset`` instances, declare ``kind`` as a `Constant` field on
        each subclass. Then when you want to create a new ``Asset`` instance
        (as in ``Asset.from_dict()``), you can use this method to select a
        more appropriate class to instantiate.

        Parameters `fieldname` and `value` are the name and value of the
        `Constant` field for which to search respectively.

        If a subclass of `cls` has been declared with a `Constant` field of
        the given name and value, it will be returned. If multiple subclasses
        of `cls` declare a matching `Constant` field, one of the matching
        subclasses will be returned, but which subclass is not defined.

        """
        try:
            clsname = classes_by_constant_field[fieldname][tuple(value)]
        except KeyError:
            # No matching classes, then.
            pass
        else:
            return find_by_name(clsname)

        raise ValueError('No such subclass of %s with field %r equivalent to %r'
            % (cls.__name__, fieldname, value))
<commit_msg>dataobject: fixed broken import in docstring<commit_after># Copyright (c) 2009-2010 Six Apart Ltd.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice,
#   this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of Six Apart Ltd. nor the names of its contributors may
#   be used to endorse or promote products derived from this software without
#   specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

"""

`DataObject` is a class of object that provides coding between object
attributes and dictionaries, suitable for

In `DataObject` is the mechanism for converting between dictionaries and
objects. These conversions are performed with aid of `Field` instances
declared on `DataObject` subclasses. `Field` classes reside in the
`remoteobjects.field` module.

"""

from copy import deepcopy
import logging

import remoteobjects.fields

classes_by_name = {}
classes_by_constant_field = {}

def find_by_name(name):
    """Finds and returns the DataObject subclass with the given name.

    Parameter `name` should be a bare class name with no module. If there is
    no class by that name, raises `KeyError`.

    """
    return classes_by_name[name]

class DataObjectMetaclass(type):
    """Metaclass for `DataObject` classes.

    This metaclass installs all `remoteobjects.fields.Property` instances
    declared as attributes of the new class, including all `Field` and `Link`
    instances.

    This metaclass also makes the new class findable through the
    `dataobject.find_by_name()` function.

    """

    def __new__(cls, name, bases, attrs):
        """Creates and returns a new `DataObject` class with its declared
        fields and name."""
        fields = {}
        new_fields = {}
        new_properties = {}

        # Inherit all the parent DataObject classes' fields.
        for base in bases:
            if isinstance(base, DataObjectMetaclass):
                fields.update(base.fields)

        # Move all the class's attributes that are Fields to the fields set.
        for attrname, field in attrs.items():
            if isinstance(field, remoteobjects.fields.Property):
                new_properties[attrname] = field
                if isinstance(field, remoteobjects.fields.Field):
                    new_fields[attrname] = field
            elif attrname in fields:
                # Throw out any parent fields that the subclass defined as
                # something other than a Field.
                del fields[attrname]

        fields.update(new_fields)
        attrs['fields'] = fields
        obj_cls = super(DataObjectMetaclass, cls).__new__(cls, name, bases, attrs)

        for field, value in new_properties.items():
            obj_cls.add_to_class(field, value)

        # Register the new class so Object fields can have forward-referenced it.
        classes_by_name[name] = obj_cls

        # Tell this class's fields what this class is, so they can find their
        # forward references later.
        for field in new_properties.values():
            field.of_cls = obj_cls

        return obj_cls

    def add_to_class(cls, name, value):
        try:
            value.install(name, cls)
        except (NotImplementedError, AttributeError):
            setattr(cls, name, value)

class DataObject(object):

    """An object that can be decoded from or encoded as a dictionary.

    DataObject subclasses should be declared with their different data
    attributes defined as instances of fields from the `remoteobjects.fields`
    module. For example:

    >>> from remoteobjects import dataobject, fields
    >>> class Asset(dataobject.DataObject):
    ...     name    = fields.Field()
    ...     updated = fields.Datetime()
    ...     author  = fields.Object('Author')
    ...

    A DataObject's fields then provide the coding between live DataObject
    instances and dictionaries.

    """

    __metaclass__ = DataObjectMetaclass

    def __init__(self, **kwargs):
        """Initializes a new `DataObject` with the given field values."""
        self.api_data = {}
        self.__dict__.update(kwargs)

    def __eq__(self, other):
        """Returns whether two `DataObject` instances are equivalent.

        If the `DataObject` instances are of the same type and contain the
        same data in all their fields, the objects are equivalent.

        """
        if type(self) != type(other):
            return False
        for k, v in self.fields.iteritems():
            if isinstance(v, remoteobjects.fields.Field):
                if getattr(self, k) != getattr(other, k):
                    return False
        return True

    def __ne__(self, other):
        """Returns whether two `DataObject` instances are different.

        `DataObject` instances are different if they are not equivalent as
        determined through `__eq__()`.

        """
        return not self == other

    @classmethod
    def statefields(cls):
        return cls.fields.keys() + ['api_data']

    def __getstate__(self):
        return dict((k, self.__dict__[k]) for k in self.statefields()
            if k in self.__dict__)

    def get(self, attr, *args):
        return getattr(self, attr, *args)

    def __iter__(self):
        for key in self.fields.keys():
            yield key

    def to_dict(self):
        """Encodes the DataObject to a dictionary."""
        data = deepcopy(self.api_data)
        for field_name, field in self.fields.iteritems():
            value = getattr(self, field.attrname, None)
            if value is not None:
                data[field.api_name] = field.encode(value)
        return data

    @classmethod
    def from_dict(cls, data):
        """Decodes a dictionary into a new `DataObject` instance."""
        self = cls()
        self.update_from_dict(data)
        return self

    def update_from_dict(self, data):
        """Adds the content of a dictionary to this DataObject.

        Parameter `data` is the dictionary from which to update the object.

        Use this only when receiving newly updated or partial content for a
        DataObject; that is, when the data is from the outside data source and
        needs decoded through the object's fields. Data from "inside" your
        application should be added to an object manually by setting the
        object's attributes. Data that constitutes a new object should be
        turned into another object with `from_dict()`.

        """
        if not isinstance(data, dict):
            raise TypeError
        # Clear any local instance field data
        for k in self.fields.iterkeys():
            if k in self.__dict__:
                del self.__dict__[k]
        self.api_data = data

    @classmethod
    def subclass_with_constant_field(cls, fieldname, value):
        """Returns the closest subclass of this class that has a `Constant`
        field with the given value.

        Use this method in combination with the `fields.Constant` field class
        to find the most appropriate subclass of `cls` based on a content
        field. For example, if you have an ``Asset`` class, but want to
        declare subclasses with special behavior based on the ``kind`` field
        of the ``Asset`` instances, declare ``kind`` as a `Constant` field on
        each subclass. Then when you want to create a new ``Asset`` instance
        (as in ``Asset.from_dict()``), you can use this method to select a
        more appropriate class to instantiate.

        Parameters `fieldname` and `value` are the name and value of the
        `Constant` field for which to search respectively.

        If a subclass of `cls` has been declared with a `Constant` field of
        the given name and value, it will be returned. If multiple subclasses
        of `cls` declare a matching `Constant` field, one of the matching
        subclasses will be returned, but which subclass is not defined.

        """
        try:
            clsname = classes_by_constant_field[fieldname][tuple(value)]
        except KeyError:
            # No matching classes, then.
            pass
        else:
            return find_by_name(clsname)

        raise ValueError('No such subclass of %s with field %r equivalent to %r'
            % (cls.__name__, fieldname, value))
<|endoftext|>


<commit_before>"""
Functions to calculate stats for each OSM contributor
"""
import MapGardening
import logging
from datetime import datetime, timedelta
import numpy

userstatstable = "usercounts"

class UserStats(object):

    conn = None
    cur = None
    utc_offset = 0
    nodetableobj = None
    blankspottableobj = None

    def __init__(self, connection, cursor, nodetableobj, blankspottableobj):
        """
        Initialize
        """
        self.conn = connection
        self.cur = cursor
        self.nodetableobj = nodetableobj
        self.blankspottableobj = blankspottableobj

    def drop_userstats_table(self):
        """
        Drop the userstats table.
        """

        querystring = "DROP TABLE " + userstatstable
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't select username and count")
            logging.error(inst)
        self.conn.commit()

    def create_userstats_table(self):
        """
        Create a table called userstats in the currently connected database
        TODO: check whether table exists

        Populates table with uid, username, and count of number of times username
        was mentioned in the node table. (This is the number of edits the user is
        responsible for)
        """

        querystring = "CREATE TABLE " + userstatstable + " AS SELECT uid, username, count(username) from " +\
            self.nodetableobj.getTableName() + " GROUP BY uid, username ORDER BY count DESC"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't select username and count")
            logging.error(inst)
        self.conn.commit()

    def add_userstats_countedits(self, queryparam = "", newcolumn = "newcount"):
        """
        Count number of edits per user, then add as new column to userstats table.

        By default, counts all the edits in the database (using an empty query parameter).
        To count v1 edits, use queryparam = "WHERE version=1".
        This function can also be called by a number of convenience functions that
        specify which type of edits to count (all, only v1, etc).
        """

        temptablename = userstatstable + "_temp" + newcolumn

        querystring = "CREATE TEMP TABLE " + temptablename + \
            " AS " + \
            "SELECT uid, username, count(uid) AS " + newcolumn + " FROM " + \
            self.nodetableobj.getTableName() + " " + queryparam + " " + \
            "GROUP BY uid, username ORDER BY " + newcolumn + " DESC"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't create temp table")
            logging.error(inst)
        self.conn.commit()

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " integer"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        querystring = "UPDATE " + userstatstable + " " + \
            "SET " + newcolumn + " = (" + \
            "SELECT " + newcolumn + " from " + \
            temptablename + " WHERE " + \
            temptablename + ".uid = " + userstatstable + ".uid)"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't update new column")
            logging.error(inst)
        self.conn.commit()

    def add_userstats_v1edits(self):
        """
        Count number of v1 edits (creation of nodes) by each user
        """
        return self.add_userstats_countedits("WHERE version=1", "v1count")

    def add_userstats_blankedits(self):
        """
        Count number of "blank spot" edits by each user
        """
        newcolumn = "blankcount"

        temptablename = userstatstable + "_temp" + newcolumn

        querystring = "CREATE TEMP TABLE " + temptablename + \
            " AS " + \
            "SELECT a.uid, a.username, count(a.uid) AS " + newcolumn + " FROM " + \
            self.nodetableobj.getTableName() + " a INNER JOIN " + \
            self.blankspottableobj.getTableName() + " b ON a.id = b.node_id " + \
            "WHERE a.version = 1 AND b.blank = true " + \
            "GROUP BY a.uid, a.username ORDER BY " + newcolumn + " DESC"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't create temp table")
            logging.error(inst)
        self.conn.commit()

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " integer"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        querystring = "UPDATE " + userstatstable + " " + \
            "SET " + newcolumn + " = (" + \
            "SELECT " + newcolumn + " from " + \
            temptablename + " WHERE " + \
            temptablename + ".uid = " + userstatstable + ".uid)"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't update new column")
            logging.error(inst)
        self.conn.commit()

    def add_userstats_firstedit(self, queryparam = "", newcolumn = "firstedit"):
        """
        Add date of each user's first edit (of any version)
        If their first edit was not v1
        """

        # Note: select "distinct on" is postgresql-specific.

        temptablename = userstatstable + "_temp" + newcolumn

        querystring = "CREATE TEMP TABLE " + temptablename + \
            " AS " + \
            "SELECT DISTINCT ON (username) username, valid_from AS " + newcolumn + " " + \
            "FROM " + self.nodetableobj.getTableName() + " " + queryparam + " " + \
            "ORDER BY username, valid_from ASC"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't create temp table")
            logging.error(inst)
        self.conn.commit()

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " date"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        querystring = "UPDATE " + userstatstable + " " \
            "SET " + newcolumn + " = (" + \
            "SELECT " + newcolumn + " from " + \
            temptablename + " WHERE " + \
            temptablename + ".username = " + userstatstable + ".username)"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't update new column")
            logging.error(inst)
        self.conn.commit()

    def add_userstats_firstedit_v1(self):
        """
        Add date of each user's first v1 edit... these are edits that
        create new features, not modify existing features.
        """

        return self.add_userstats_firstedit("WHERE version=1", "firsteditv1")

    def add_userstats_firstedit_blank(self):
        """
        Add date of each user's first blankspot edit... these are edits that
        create new features, in areas where no features previously existed
        """

        return self.add_userstats_firstedit("WHERE blank=true", "firsteditblank")

    def _days_since_epoch_obj(self, input_date_obj):
        """
        Given a date object, return number of days since epoch.
        This makes it easier to calculate the mean date of activity.
        """
        epoch = datetime.utcfromtimestamp(0)
        delta = input_date_obj - epoch
        return delta.days

    def _days_since_epoch(self, input_date_str):
        """
        Given a date string of format YYYY-MM-DD return number od days since epoch.
        """
        date_obj = datetime.strptime(input_date_str, "%Y-%m-%d")
        return self._days_since_epoch_obj(date_obj)

    def _date_from_days_since_epoch(self, days_since_epoch_int):
        """
        Reverse the calculation from _days_since_epoch(). Given an integer
        representing the number of days since Jan 1, 1970, return a date object
        for that date.
        """
        epoch = datetime.utcfromtimestamp(0)
        delta = timedelta(days=days_since_epoch_int)
        date_obj = epoch + delta
        return date_obj

    def get_dates_and_edit_counts(self):

        print "gathering dates and edit counts"
        user_date_dict = {}

        querystring = "SELECT username, valid_from, version, blank " + \
            "FROM " + self.nodetableobj.getTableName()
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't select from nodes table")
            logging.error(inst)

        rows = self.cur.fetchall()

        if rows == None:
            return ""

        for row in rows:
            (username, valid_from, version, blank) = row
            if not username in user_date_dict:
                user_date_dict[username] = {}

            # Shift utc datetimes to local time
            td = timedelta(hours=self.utc_offset)
            edit_dt = valid_from + td

            # Return a date object (drop time info) from the datetime object
            edit_date_str = str(edit_dt.date())

            if not edit_date_str in user_date_dict[username]:
                user_date_dict[username][edit_date_str] = {'all': 0, 'v1': 0, 'blank': 0}

            user_date_dict[username][edit_date_str]['all'] += 1
            if version == 1:
                user_date_dict[username][edit_date_str]['v1'] += 1
            if blank == True:
                user_date_dict[username][edit_date_str]['blank'] += 1
        return user_date_dict

    def add_userstats_mean_date(self, weighted=False, user_date_dict=None):
        """
        Of all the days the mapper is active, find the average (mean) date of activity
        """
        if user_date_dict == None:
            user_date_dict = self.get_dates_and_edit_counts()

        status = "calculating mean edit dates"

        newcolumn = "mean_date"
        if weighted == True:
            newcolumn = newcolumn + "_weighted"
            status += " (weighted)"

        print status

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " date"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        # Get keys for each user dict (list of dates)
        # convert keys to integers, find mean, convert back to date.
        for username in user_date_dict:
            date_keys = user_date_dict[username].keys()
            if weighted:
                weights = []
                for dk in date_keys:
                    weights.append(user_date_dict[username][dk]['all'])
            else:
                weights = None
            list_of_dates = map(self._days_since_epoch, date_keys)
            mean_date = round(numpy.average(list_of_dates, None, weights))
            mean_date_dt = self._date_from_days_since_epoch(mean_date)
            querystring = "UPDATE " + userstatstable + " " + \
                "SET " + newcolumn + " = %s " + \
                "WHERE username = %s"
            try:
                self.cur.execute(querystring, (mean_date_dt, username))
            except Exception, inst:
                logging.error("can't update mean_date column")
                logging.error(inst)
        self.conn.commit()

    def add_userstats_days_active(self, user_date_dict=None):
        """
        Count the number of days (calendar days in each region's timezone) the mapper edited the db
        """
        if user_date_dict == None:
            user_date_dict = self.get_dates_and_edit_counts()

        print "counting days active"

        newcolumn = "days_active"

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " integer"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        for username in user_date_dict:
            days_active = len(user_date_dict[username])
            querystring = "UPDATE " + userstatstable + " " + \
                "SET " + newcolumn + " = %s " + \
                "WHERE username = %s"
            try:
                self.cur.execute(querystring, (days_active, username))
            except Exception, inst:
                logging.error("can't update days_active column")
                logging.error(inst)
        self.conn.commit()

    def print_userstats(self, filename):
        querystring = "SELECT uid, username, count, blankcount, v1count, firstedit, firsteditv1, firsteditblank, days_active, mean_date, mean_date_weighted FROM " + userstatstable
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't select count and blankcount")
            logging.error(inst)

        localfile = open(filename, "w")

        print >> localfile, "uid\tusername\tcount\tblankcount\tv1count\tfirstedit\tfirsteditv1\tfirsteditblank\tdays_active\tmean_date\tmean_date_weighted"

        rows = self.cur.fetchall()

        for row in rows:
            # assign 0 instead of None
            uid = row[0] or 0
            username = row[1] or 0
            count = row[2] or 0
            blankcount = row[3] or 0
            v1count = row[4] or 0
            firstedit = row[5] or 0
            firsteditv1 = row[6] or 0
            firsteditblank = row[7] or 0
            days_active = row[8] or 0
            mean_date = row[9] or 0
            mean_date_weighted = row[10] or 0
            print >> localfile, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % (uid, username, count, blankcount, v1count, firstedit, firsteditv1, firsteditblank, days_active, mean_date, mean_date_weighted)

        localfile.close() <commit_msg>Now actually write mean_date and mean_date_weighted to file<commit_after>"""
Functions to calculate stats for each OSM contributor
"""
import MapGardening
import logging
from datetime import datetime, timedelta
import numpy

userstatstable = "usercounts"

class UserStats(object):

    conn = None
    cur = None
    utc_offset = 0
    nodetableobj = None
    blankspottableobj = None

    def __init__(self, connection, cursor, nodetableobj, blankspottableobj):
        """
        Initialize
        """
        self.conn = connection
        self.cur = cursor
        self.nodetableobj = nodetableobj
        self.blankspottableobj = blankspottableobj

    def drop_userstats_table(self):
        """
        Drop the userstats table.
        """

        querystring = "DROP TABLE " + userstatstable
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't select username and count")
            logging.error(inst)
        self.conn.commit()

    def create_userstats_table(self):
        """
        Create a table called userstats in the currently connected database
        TODO: check whether table exists

        Populates table with uid, username, and count of number of times username
        was mentioned in the node table. (This is the number of edits the user is
        responsible for)
        """

        querystring = "CREATE TABLE " + userstatstable + " AS SELECT uid, username, count(username) from " +\
            self.nodetableobj.getTableName() + " GROUP BY uid, username ORDER BY count DESC"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't select username and count")
            logging.error(inst)
        self.conn.commit()

    def add_userstats_countedits(self, queryparam = "", newcolumn = "newcount"):
        """
        Count number of edits per user, then add as new column to userstats table.

        By default, counts all the edits in the database (using an empty query parameter).
        To count v1 edits, use queryparam = "WHERE version=1".
        This function can also be called by a number of convenience functions that
        specify which type of edits to count (all, only v1, etc).
        """

        temptablename = userstatstable + "_temp" + newcolumn

        querystring = "CREATE TEMP TABLE " + temptablename + \
            " AS " + \
            "SELECT uid, username, count(uid) AS " + newcolumn + " FROM " + \
            self.nodetableobj.getTableName() + " " + queryparam + " " + \
            "GROUP BY uid, username ORDER BY " + newcolumn + " DESC"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't create temp table")
            logging.error(inst)
        self.conn.commit()

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " integer"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        querystring = "UPDATE " + userstatstable + " " + \
            "SET " + newcolumn + " = (" + \
            "SELECT " + newcolumn + " from " + \
            temptablename + " WHERE " + \
            temptablename + ".uid = " + userstatstable + ".uid)"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't update new column")
            logging.error(inst)
        self.conn.commit()

    def add_userstats_v1edits(self):
        """
        Count number of v1 edits (creation of nodes) by each user
        """
        return self.add_userstats_countedits("WHERE version=1", "v1count")

    def add_userstats_blankedits(self):
        """
        Count number of "blank spot" edits by each user
        """
        newcolumn = "blankcount"

        temptablename = userstatstable + "_temp" + newcolumn

        querystring = "CREATE TEMP TABLE " + temptablename + \
            " AS " + \
            "SELECT a.uid, a.username, count(a.uid) AS " + newcolumn + " FROM " + \
            self.nodetableobj.getTableName() + " a INNER JOIN " + \
            self.blankspottableobj.getTableName() + " b ON a.id = b.node_id " + \
            "WHERE a.version = 1 AND b.blank = true " + \
            "GROUP BY a.uid, a.username ORDER BY " + newcolumn + " DESC"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't create temp table")
            logging.error(inst)
        self.conn.commit()

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " integer"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        querystring = "UPDATE " + userstatstable + " " + \
            "SET " + newcolumn + " = (" + \
            "SELECT " + newcolumn + " from " + \
            temptablename + " WHERE " + \
            temptablename + ".uid = " + userstatstable + ".uid)"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't update new column")
            logging.error(inst)
        self.conn.commit()

    def add_userstats_firstedit(self, queryparam = "", newcolumn = "firstedit"):
        """
        Add date of each user's first edit (of any version)
        If their first edit was not v1
        """

        # Note: select "distinct on" is postgresql-specific.

        temptablename = userstatstable + "_temp" + newcolumn

        querystring = "CREATE TEMP TABLE " + temptablename + \
            " AS " + \
            "SELECT DISTINCT ON (username) username, valid_from AS " + newcolumn + " " + \
            "FROM " + self.nodetableobj.getTableName() + " " + queryparam + " " + \
            "ORDER BY username, valid_from ASC"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't create temp table")
            logging.error(inst)
        self.conn.commit()

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " date"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        querystring = "UPDATE " + userstatstable + " " \
            "SET " + newcolumn + " = (" + \
            "SELECT " + newcolumn + " from " + \
            temptablename + " WHERE " + \
            temptablename + ".username = " + userstatstable + ".username)"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't update new column")
            logging.error(inst)
        self.conn.commit()

    def add_userstats_firstedit_v1(self):
        """
        Add date of each user's first v1 edit... these are edits that
        create new features, not modify existing features.
        """

        return self.add_userstats_firstedit("WHERE version=1", "firsteditv1")

    def add_userstats_firstedit_blank(self):
        """
        Add date of each user's first blankspot edit... these are edits that
        create new features, in areas where no features previously existed
        """

        return self.add_userstats_firstedit("WHERE blank=true", "firsteditblank")

    def _days_since_epoch_obj(self, input_date_obj):
        """
        Given a date object, return number of days since epoch.
        This makes it easier to calculate the mean date of activity.
        """
        epoch = datetime.utcfromtimestamp(0)
        delta = input_date_obj - epoch
        return delta.days

    def _days_since_epoch(self, input_date_str):
        """
        Given a date string of format YYYY-MM-DD return number od days since epoch.
        """
        date_obj = datetime.strptime(input_date_str, "%Y-%m-%d")
        return self._days_since_epoch_obj(date_obj)

    def _date_from_days_since_epoch(self, days_since_epoch_int):
        """
        Reverse the calculation from _days_since_epoch(). Given an integer
        representing the number of days since Jan 1, 1970, return a date object
        for that date.
        """
        epoch = datetime.utcfromtimestamp(0)
        delta = timedelta(days=days_since_epoch_int)
        date_obj = epoch + delta
        return date_obj

    def get_dates_and_edit_counts(self):

        print "gathering dates and edit counts"
        user_date_dict = {}

        querystring = "SELECT username, valid_from, version, blank " + \
            "FROM " + self.nodetableobj.getTableName()
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't select from nodes table")
            logging.error(inst)

        rows = self.cur.fetchall()

        if rows == None:
            return ""

        for row in rows:
            (username, valid_from, version, blank) = row
            if not username in user_date_dict:
                user_date_dict[username] = {}

            # Shift utc datetimes to local time
            td = timedelta(hours=self.utc_offset)
            edit_dt = valid_from + td

            # Return a date object (drop time info) from the datetime object
            edit_date_str = str(edit_dt.date())

            if not edit_date_str in user_date_dict[username]:
                user_date_dict[username][edit_date_str] = {'all': 0, 'v1': 0, 'blank': 0}

            user_date_dict[username][edit_date_str]['all'] += 1
            if version == 1:
                user_date_dict[username][edit_date_str]['v1'] += 1
            if blank == True:
                user_date_dict[username][edit_date_str]['blank'] += 1
        return user_date_dict

    def add_userstats_mean_date(self, weighted=False, user_date_dict=None):
        """
        Of all the days the mapper is active, find the average (mean) date of activity
        """
        if user_date_dict == None:
            user_date_dict = self.get_dates_and_edit_counts()

        status = "calculating mean edit dates"

        newcolumn = "mean_date"
        if weighted == True:
            newcolumn = newcolumn + "_weighted"
            status += " (weighted)"

        print status

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " date"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        # Get keys for each user dict (list of dates)
        # convert keys to integers, find mean, convert back to date.
        for username in user_date_dict:
            date_keys = user_date_dict[username].keys()
            if weighted:
                weights = []
                for dk in date_keys:
                    weights.append(user_date_dict[username][dk]['all'])
            else:
                weights = None
            list_of_dates = map(self._days_since_epoch, date_keys)
            mean_date = round(numpy.average(list_of_dates, None, weights))
            mean_date_dt = self._date_from_days_since_epoch(mean_date)
            querystring = "UPDATE " + userstatstable + " " + \
                "SET " + newcolumn + " = %s " + \
                "WHERE username = %s"
            try:
                self.cur.execute(querystring, (mean_date_dt, username))
            except Exception, inst:
                logging.error("can't update mean_date column")
                logging.error(inst)
        self.conn.commit()

    def add_userstats_days_active(self, user_date_dict=None):
        """
        Count the number of days (calendar days in each region's timezone) the mapper edited the db
        """
        if user_date_dict == None:
            user_date_dict = self.get_dates_and_edit_counts()

        print "counting days active"

        newcolumn = "days_active"

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " integer"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        for username in user_date_dict:
            days_active = len(user_date_dict[username])
            querystring = "UPDATE " + userstatstable + " " + \
                "SET " + newcolumn + " = %s " + \
                "WHERE username = %s"
            try:
                self.cur.execute(querystring, (days_active, username))
            except Exception, inst:
                logging.error("can't update days_active column")
                logging.error(inst)
        self.conn.commit()

    def print_userstats(self, filename):
        querystring = "SELECT uid, username, count, blankcount, v1count, firstedit, firsteditv1, firsteditblank, days_active, mean_date, mean_date_weighted FROM " + userstatstable
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't select count and blankcount")
            logging.error(inst)

        localfile = open(filename, "w")

        print >> localfile, "uid\tusername\tcount\tblankcount\tv1count\tfirstedit\tfirsteditv1\tfirsteditblank\tdays_active\tmean_date\tmean_date_weighted"

        rows = self.cur.fetchall()

        for row in rows:
            # assign 0 instead of None
            uid = row[0] or 0
            username = row[1] or 0
            count = row[2] or 0
            blankcount = row[3] or 0
            v1count = row[4] or 0
            firstedit = row[5] or 0
            firsteditv1 = row[6] or 0
            firsteditblank = row[7] or 0
            days_active = row[8] or 0
            mean_date = row[9] or 0
            mean_date_weighted = row[10] or 0
            print >> localfile, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % (uid, username, count, blankcount, v1count, firstedit, firsteditv1, firsteditblank, days_active, mean_date, mean_date_weighted)

        localfile.close() <|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from __future__ import absolute_import

import os

from nailgun import extensions

import tuning_box

class Extension(extensions.BaseExtension):
    name = 'tuning_box'
    version = tuning_box.__version__
    description = 'Plug tuning_box endpoints into Nailgun itself'

    @classmethod
    def alembic_migrations_path(cls):
        return os.path.join(os.path.dirname(__file__), 'migrations')
<commit_msg>Plug our Flask app into Nailgun's web.py<commit_after># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from __future__ import absolute_import

import itertools
import os
import threading

from nailgun import db as nailgun_db
from nailgun import extensions
import web

import tuning_box
from tuning_box import app as tb_app
from tuning_box import db as tb_db

class App2WebPy(web.application):
    def __init__(self):
        web.application.__init__(self)
        self.__name__ = self
        self.app = None
        self.lock = threading.Lock()

    def create_app(self):
        raise NotImplementedError

    def get_app(self):
        with self.lock:
            if not self.app:
                self.app = self.create_app()
            return self.app

    def handle(self):
        written_data = []

        def write(data):
            assert start_response.called
            written_data.append(data)

        def start_response(status, headers, exc_info=None):
            assert not start_response.called
            assert not exc_info
            start_response.called = True
            web.ctx.status = status
            web.ctx.headers.extend(headers)
            return write

        start_response.called = False

        app = self.get_app()
        environ = dict(web.ctx.environ)
        environ["PATH_INFO"] = environ["REQUEST_URI"] = web.ctx.path
        result = app(environ, start_response)
        return itertools.chain(written_data, result)

class TB2WebPy(App2WebPy):
    def create_app(self):
        app = tb_app.build_app()
        tb_db.prefix_tables(Extension.table_prefix())
        tb_db.db.session = nailgun_db.db
        app.config["PROPAGATE_EXCEPTIONS"] = True
        return app

class Extension(extensions.BaseExtension):
    name = 'tuning_box'
    version = tuning_box.__version__
    description = 'Plug tuning_box endpoints into Nailgun itself'

    urls = [{'uri': '/config', 'handler': TB2WebPy()}]

    @classmethod
    def alembic_migrations_path(cls):
        return os.path.join(os.path.dirname(__file__), 'migrations')
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-
"""
Created on 2014/07/31
@author: Jimmy Liu
@group : waditu
@contact: jimmysoa@sina.cn
"""
VERSION = '0.1.3'
P_TYPE = {'http':'http://','ftp':'ftp://'}
DOMAINS = {'sina':'sina.com.cn','sinahq':'sinajs.cn','ifeng':'ifeng.com'}
DAY_PRICE_COLUMNS = ['date','open','high','close','low','volume','price_change','p_change',
                     'ma5','ma10','ma20','v_ma5','v_ma10','v_ma20','turnover']
TICK_COLUMNS = ['time','price','change','volume','amount','type']
TICK_PRICE_URL = '%smarket.finance.%s/downxls.php?date=%s&symbol=%s'
DAY_PRICE_URL = '%sapi.finance.%s/akdaily/?code=%s&type=last'
LIVE_DATA_URL = '%shq.%s/list=%s'
SINA_DAY_PRICE_URL = '%svip.stock.finance.%s/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?num=80&sort=changepercent&asc=0&node=hs_a&symbol=&_s_r_a=page&page=%s'
DAY_TRADING_COLUMNS = ['code','symbol','name','changepercent','trade','open','high','low','settlement','volume','turnoverratio']
DAY_PRICE_PAGES = 38
LIVE_DATA_COLS = ['name','open','pre_close','price','high','low','bid','ask','volume','amount',
               'b1_v','b1_p','b2_v','b2_p','b3_v','b3_p','b4_v','b4_p','b5_v','b5_p',
               'a1_v','a1_p','a2_v','a2_p','a3_v','a3_p','a4_v','a4_p','a5_v','a5_p','date','time','s']
REPORT_URL = '%svip.stock.finance.%s/q/go.php/vFinanceAnalyze/kind/mainindex/index.phtml?s_i=&s_a=&s_c=&reportdate=%s&quarter=%s&p=%s&num=60'
FORECAST_URL = '%svip.stock.finance.%s/q/go.php/vFinanceAnalyze/kind/performance/index.phtml?s_i=&s_a=&s_c=&s_type=&reportdate=%s&quarter=%s&p=%s&num=60'
PROFIT_URL = '%svip.stock.finance.%s/q/go.php/vFinanceAnalyze/kind/profit/index.phtml?s_i=&s_a=&s_c=&reportdate=%s&quarter=%s&p=%s&num=60'
REPORT_COLS = ['code','name','eps','eps_yoy','bvps','roe','epcf','net_profits','profits_yoy','distrib','report_date']
FORECAST_COLS = ['code','name','type','report_date','pre_eps','range']

def data_path():
    import os
    import inspect
    caller_file = inspect.stack()[1][1]  # caller's filename
    pardir = os.path.abspath(os.path.join(os.path.dirname(caller_file), os.path.pardir))
    return os.path.abspath(os.path.join(pardir, os.path.pardir))

ALL_STOCK_BASICS_FILE = '%s/tushare/data/all.csv'%data_path()<commit_msg>cons.py updated<commit_after># -*- coding:utf-8 -*-
"""
Created on 2014/07/31
@author: Jimmy Liu
@group : waditu
@contact: jimmysoa@sina.cn
"""

def data_path():
    import os
    import inspect
    caller_file = inspect.stack()[1][1]
    pardir = os.path.abspath(os.path.join(os.path.dirname(caller_file), os.path.pardir))
    return os.path.abspath(os.path.join(pardir, os.path.pardir))

VERSION = '0.1.3'
P_TYPE = {'http':'http://','ftp':'ftp://'}
DAY_PRICE_PAGES = 38
DOMAINS = {'sina':'sina.com.cn','sinahq':'sinajs.cn','ifeng':'ifeng.com'}
TICK_COLUMNS = ['time','price','change','volume','amount','type']
DAY_TRADING_COLUMNS = ['code','symbol','name','changepercent','trade','open','high','low','settlement','volume','turnoverratio']
REPORT_COLS = ['code','name','eps','eps_yoy','bvps','roe','epcf','net_profits','profits_yoy','distrib','report_date']
FORECAST_COLS = ['code','name','type','report_date','pre_eps','range']
DAY_PRICE_COLUMNS = ['date','open','high','close','low','volume','price_change','p_change',
                     'ma5','ma10','ma20','v_ma5','v_ma10','v_ma20','turnover']
LIVE_DATA_COLS = ['name','open','pre_close','price','high','low','bid','ask','volume','amount',
               'b1_v','b1_p','b2_v','b2_p','b3_v','b3_p','b4_v','b4_p','b5_v','b5_p',
               'a1_v','a1_p','a2_v','a2_p','a3_v','a3_p','a4_v','a4_p','a5_v','a5_p','date','time','s']
TICK_PRICE_URL = '%smarket.finance.%s/downxls.php?date=%s&symbol=%s'
DAY_PRICE_URL = '%sapi.finance.%s/akdaily/?code=%s&type=last'
LIVE_DATA_URL = '%shq.%s/list=%s'
SINA_DAY_PRICE_URL = '%svip.stock.finance.%s/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?num=80&sort=changepercent&asc=0&node=hs_a&symbol=&_s_r_a=page&page=%s'
REPORT_URL = '%svip.stock.finance.%s/q/go.php/vFinanceAnalyze/kind/mainindex/index.phtml?s_i=&s_a=&s_c=&reportdate=%s&quarter=%s&p=%s&num=60'
FORECAST_URL = '%svip.stock.finance.%s/q/go.php/vFinanceAnalyze/kind/performance/index.phtml?s_i=&s_a=&s_c=&s_type=&reportdate=%s&quarter=%s&p=%s&num=60'
PROFIT_URL = '%svip.stock.finance.%s/q/go.php/vFinanceAnalyze/kind/profit/index.phtml?s_i=&s_a=&s_c=&reportdate=%s&quarter=%s&p=%s&num=60'
ALL_STOCK_BASICS_FILE = '%s/tushare/data/all.csv'%data_path()<|endoftext|>


<commit_before>from __future__ import print_function
import auxA as intpol

#
# Function to generate test data
#
def testdata(N):
    """
    Generates N points of (x, y)-data to test the different routines
    Arguments:
    - `N`:
    """
    from math import cos, sin
    x = [i + 0.5 * sin(i) for i in range(N)]
    y = [i + cos(i*i) for i in range(N)]
    return x, y

#
# Main
#

# Generate test data
x, y = testdata(10)

# Print test data for plotting
for i in range(len(x)):
    print(x[i], end='\t')
    print(y[i])
print('\n')

# Initialize useful variables
n = 50
step = (x[-1] - x[0]) / n

# Linear interpolation
z_lin = x[0]
while z_lin < x[-1]:
    print(z_lin, end='\t')
    print(intpol.lspline(x, y, z_lin))
    z_lin += step
print('\n')

# Quadratic spline
z_quad = x[0]
quad_spline = intpol.qspline(x, y)
while z_quad < x[-1]:
    print(z_quad, end='\t')
    print(quad_spline(z_quad))
    z_quad += step
<commit_msg>Minor correction<commit_after>from __future__ import print_function
import auxA as intpol

#
# Function to generate test data
#
def testdata(N):
    """
    Generates N points of (x, y)-data to test the different routines
    """
    from math import cos, sin
    x = [i + 0.5 * sin(i) for i in range(N)]
    y = [i + cos(i*i) for i in range(N)]
    return x, y

#
# Main
#

# Generate test data
x, y = testdata(10)

# Print test data for plotting
for i in range(len(x)):
    print(x[i], end='\t')
    print(y[i])
print('\n')

# Initialize useful variables
n = 50
step = (x[-1] - x[0]) / n

# Linear interpolation
z_lin = x[0]
while z_lin < x[-1]:
    print(z_lin, end='\t')
    print(intpol.lspline(x, y, z_lin))
    z_lin += step
print('\n')

# Quadratic spline
z_quad = x[0]
quad_spline = intpol.qspline(x, y)
while z_quad < x[-1]:
    print(z_quad, end='\t')
    print(quad_spline(z_quad))
    z_quad += step
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-
"""
@brief      test log(time=30s)
"""

import sys
import os
import unittest
import ujson
import falcon
import falcon.testing as testing

try:
    import pyquickhelper as skip_
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..",
                "..",
                "pyquickhelper",
                "src")))
    if path not in sys.path:
        sys.path.append(path)
    import pyquickhelper as skip_

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

from pyquickhelper.loghelper import fLOG
from pyquickhelper.pycode import get_temp_folder, add_missing_development_version
from pyquickhelper.pycode import skipif_travis

class TestDummyAppSearchImg(testing.TestCase):

    def setUp(self):
        add_missing_development_version(["pymyinstall", "pyensae", "jyquickhelper",
                                         "pandas_streaming", "mlinsights", "lightmlrestapi"],
                                        __file__, hide=True)
        super().setUp()

        from src.ensae_projects.restapi import search_images_dogcat
        temp = get_temp_folder(__file__, 'temp_search_images_dogcat')
        search_images_dogcat(self.api, dest=temp)

    @skipif_travis('tensorflow/python/lib/core/bfloat16.cc:664] Check failed: PyBfloat16_Type.tp_base != nullptr')
    def test_dummy_search_app_search_img(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        from lightmlrestapi.args import image2base64

        # With a different image than the original.
        img2 = os.path.join(os.path.dirname(__file__),
                            "data", "wiki_modified.png")
        b64 = image2base64(img2)[1]
        bodyin = ujson.dumps({'X': b64})
        result = self.simulate_request(path='/', method="POST", body=bodyin)
        if result.status != falcon.HTTP_201:
            raise Exception("Failure\n{0}".format(result.status))
        self.assertEqual(result.status, falcon.HTTP_201)
        d = ujson.loads(result.content)
        self.assertTrue('Y' in d)
        self.assertIsInstance(d['Y'], list)
        self.assertEqual(len(d['Y']), 1)
        self.assertEqual(len(d['Y'][0]), 5)
        self.assertIsInstance(d['Y'][0][0][0], float)
        self.assertIsInstance(d['Y'][0][0][1], int)
        self.assertIn('name', d['Y'][0][0][2])
        self.assertEqual(d['Y'][0][0][2]['name'].replace(
            '\\', '/'), 'oneclass/cat-2922832__480.jpg')

    def test_dummy_error_img(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__ma²in__")

        from lightmlrestapi.args import image2base64, image2array, base642image

        img2 = os.path.join(os.path.dirname(__file__),
                            "data", "wiki_modified.png")
        ext_b64 = image2base64(img2)
        img2 = base642image(ext_b64[1]).convert('RGB')
        arr = image2array(img2)
        bodyin = ujson.dumps({'X': arr.tolist()})

        result = self.simulate_request(path='/', method="POST", body=bodyin)
        self.assertEqual(result.status, falcon.HTTP_400)
        d = ujson.loads(result.content)
        self.assertIn('Unable to predict', d['title'])
        self.assertIn("object has no attribute", d['description'])

if __name__ == "__main__":
    unittest.main()
<commit_msg>skip another unit test on travis<commit_after>#-*- coding: utf-8 -*-
"""
@brief      test log(time=30s)
"""

import sys
import os
import unittest
import ujson
import falcon
import falcon.testing as testing

try:
    import pyquickhelper as skip_
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..",
                "..",
                "pyquickhelper",
                "src")))
    if path not in sys.path:
        sys.path.append(path)
    import pyquickhelper as skip_

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

from pyquickhelper.loghelper import fLOG
from pyquickhelper.pycode import get_temp_folder, add_missing_development_version
from pyquickhelper.pycode import skipif_travis

class TestDummyAppSearchImg(testing.TestCase):

    def setUp(self):
        add_missing_development_version(["pymyinstall", "pyensae", "jyquickhelper",
                                         "pandas_streaming", "mlinsights", "lightmlrestapi"],
                                        __file__, hide=True)
        super().setUp()

        from src.ensae_projects.restapi import search_images_dogcat
        temp = get_temp_folder(__file__, 'temp_search_images_dogcat')
        search_images_dogcat(self.api, dest=temp)

    def test_dummy_search_app_search_img(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        from lightmlrestapi.args import image2base64

        # With a different image than the original.
        img2 = os.path.join(os.path.dirname(__file__),
                            "data", "wiki_modified.png")
        b64 = image2base64(img2)[1]
        bodyin = ujson.dumps({'X': b64})
        result = self.simulate_request(path='/', method="POST", body=bodyin)
        if result.status != falcon.HTTP_201:
            raise Exception("Failure\n{0}".format(result.status))
        self.assertEqual(result.status, falcon.HTTP_201)
        d = ujson.loads(result.content)
        self.assertTrue('Y' in d)
        self.assertIsInstance(d['Y'], list)
        self.assertEqual(len(d['Y']), 1)
        self.assertEqual(len(d['Y'][0]), 5)
        self.assertIsInstance(d['Y'][0][0][0], float)
        self.assertIsInstance(d['Y'][0][0][1], int)
        self.assertIn('name', d['Y'][0][0][2])
        self.assertEqual(d['Y'][0][0][2]['name'].replace(
            '\\', '/'), 'oneclass/cat-2922832__480.jpg')

    @skipif_travis('tensorflow/python/lib/core/bfloat16.cc:664] Check failed: PyBfloat16_Type.tp_base != nullptr')
    def test_dummy_error_img(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__ma²in__")

        from lightmlrestapi.args import image2base64, image2array, base642image

        img2 = os.path.join(os.path.dirname(__file__),
                            "data", "wiki_modified.png")
        ext_b64 = image2base64(img2)
        img2 = base642image(ext_b64[1]).convert('RGB')
        arr = image2array(img2)
        bodyin = ujson.dumps({'X': arr.tolist()})

        result = self.simulate_request(path='/', method="POST", body=bodyin)
        self.assertEqual(result.status, falcon.HTTP_400)
        d = ujson.loads(result.content)
        self.assertIn('Unable to predict', d['title'])
        self.assertIn("object has no attribute", d['description'])

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function, unicode_literals

from flask import url_for

from abilian.core.models.subjects import User
from abilian.sbe.testing import BaseTestCase

class TestSocialViews(BaseTestCase):

    no_login = True

    # Tests start here
    def test_home(self):
        response = self.client.get(url_for("social.home"))
        assert response.status_code == 200
        self.assert_valid(response)

    def test_users(self):
        response = self.client.get(url_for("social.users"))
        assert response.status_code == 200
        self.assert_valid(response)

    def test_groups(self):
        response = self.client.get(url_for("social.groups"))
        assert response.status_code == 200
        self.assert_valid(response)

    def test_user(self):
        user = User.query.all()[0]
        response = self.client.get(url_for("social.user", user_id=user.id))
        assert response.status_code == 200
        self.assert_valid(response)

# def test_home(client):
#     response = client.get(url_for("social.home"))
#     assert response.status_code == 200
#     # self.assert_valid(response)
#
#
# def test_users(client):
#     response = client.get(url_for("social.users"))
#     assert response.status_code == 200
#     # self.assert_valid(response)
#
#
# def test_groups(client):
#     response = client.get(url_for("social.groups"))
#     assert response.status_code == 200
#     # self.assert_valid(response)
#
#
# def test_user(client):
#     user = User.query.all()[0]
#     response = client.get(url_for("social.user", user_id=user.id))
#     assert response.status_code == 200
#     # self.assert_valid(response)
<commit_msg>remove useless assertions.<commit_after>from __future__ import absolute_import, print_function, unicode_literals

from flask import url_for

from abilian.core.models.subjects import User
from abilian.sbe.testing import BaseTestCase

class TestSocialViews(BaseTestCase):

    no_login = True

    # Tests start here
    def test_home(self):
        response = self.client.get(url_for("social.home"))
        assert response.status_code == 200

    def test_users(self):
        response = self.client.get(url_for("social.users"))
        assert response.status_code == 200

    def test_groups(self):
        response = self.client.get(url_for("social.groups"))
        assert response.status_code == 200

    def test_user(self):
        user = User.query.all()[0]
        response = self.client.get(url_for("social.user", user_id=user.id))
        assert response.status_code == 200

# def test_home(client):
#     response = client.get(url_for("social.home"))
#     assert response.status_code == 200
#     # self.assert_valid(response)
#
#
# def test_users(client):
#     response = client.get(url_for("social.users"))
#     assert response.status_code == 200
#     # self.assert_valid(response)
#
#
# def test_groups(client):
#     response = client.get(url_for("social.groups"))
#     assert response.status_code == 200
#     # self.assert_valid(response)
#
#
# def test_user(client):
#     user = User.query.all()[0]
#     response = client.get(url_for("social.user", user_id=user.id))
#     assert response.status_code == 200
#     # self.assert_valid(response)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import re, string, sys

with open("../stop_words.txt") as f:
    stops = set(f.read().split(",") + list(string.ascii_lowercase))
# The "database"
data = {}

# Internal functions of the "server"-side application
def error_state():
    return "Something wrong", ["get", "default", None]

# The "server"-side application handlers
def quit_handler(args):
    sys.exit("Goodbye cruel world...")

def default_get_handler(args):
        rep = "What would you like to do?"
        rep += "\n1 - Quit" + "\n2 - Upload file"
        links = {"1" : ["post", "execution", None], "2" : ["get", "file_form", None]}
        return rep, links

def upload_get_handler(args):
    return "Name of file to upload?", ["post", "file"]

def upload_post_handler(args):
    def create_data(filename):
        if filename in data:
            return
        word_freqs = {}
        print "Trying to open " + filename
        with open(filename) as f:
            for w in [x.lower() for x in re.split("[^a-zA-Z]+", f.read()) if len(x) > 0 and x.lower() not in stops]:
                word_freqs[w] = word_freqs.get(w, 0) + 1
        word_freqsl = word_freqs.items()
        word_freqsl.sort(lambda x, y: cmp(y[1], x[1]))
        data[filename] = word_freqsl

    if args == None:
        return error_state()

    filename = args[0]
    try:
        create_data(filename)
    except:
        return error_state()

    return word_get_handler([filename, 0])

def word_get_handler(args):
    def get_word(filename, word_index):
        if word_index < len(data[filename]):
            return data[filename][word_index]
        else:
            return ("no more words", 0)

    filename = args[0]; word_index = args[1]
    word_info = get_word(filename, word_index)
    rep = '\n#{0}: {1} - {2}'.format(word_index+1, word_info[0], word_info[1])
    rep += "\n\nWhat would you like to do next?"
    rep += "\n1 - Quit" + "\n2 - Upload file"
    rep += "\n3 - See next most-frequently occurring word"
    links = {"1" : ["post", "execution", None],
             "2" : ["get", "file_form", None],
             "3" : ["get", "word", [filename, word_index+1]]}
    return rep, links

# Handler registration
handlers = {"post_execution" : quit_handler,
            "get_default" : default_get_handler,
            "get_file_form" : upload_get_handler,
            "post_file" : upload_post_handler,
            "get_word" : word_get_handler }

# The "server" core
def handle_request(verb, uri, args):
    def handler_key(verb, uri):
        return verb + "_" + uri

    if handler_key(verb, uri) in handlers:
        return handlers[handler_key(verb, uri)](args)
    else:
        return handlers[handler_key("get", "default")](args)

# A very simple client "browser"
def render_and_get_input(state_representation, links):
    print state_representation
    sys.stdout.flush()
    if type(links) is dict: # many possible next states
        input = sys.stdin.readline().strip()
        if input in links:
            return links[input]
        else:
            return ["get", "default", None]
    elif type(links) is list: # only one possible next state
        if links[0] == "post": # get "form" data
            input = sys.stdin.readline().strip()
            links.append([input]) # add the data at the end
            return links
        else: # get action, don't get user input
            return links
    else:
        return ["get", "default", None]

request = ["get", "default", None]
while True:
    # "server"-side computation
    state_representation, links = handle_request(*request)
    # "client"-side computation
    request = render_and_get_input(state_representation, links)
<commit_msg>Minor<commit_after>#!/usr/bin/env python
import re, string, sys

with open("../stop_words.txt") as f:
    stops = set(f.read().split(",") + list(string.ascii_lowercase))
# The "database"
data = {}

# Internal functions of the "server"-side application
def error_state():
    return "Something wrong", ["get", "default", None]

# The "server"-side application handlers
def quit_handler(args):
    sys.exit("Goodbye cruel world...")

def default_get_handler(args):
        rep = "What would you like to do?"
        rep += "\n1 - Quit" + "\n2 - Upload file"
        links = {"1" : ["post", "execution", None], "2" : ["get", "file_form", None]}
        return rep, links

def upload_get_handler(args):
    return "Name of file to upload?", ["post", "file"]

def upload_post_handler(args):
    def create_data(filename):
        if filename in data:
            return
        word_freqs = {}
        with open(filename) as f:
            for w in [x.lower() for x in re.split("[^a-zA-Z]+", f.read()) if len(x) > 0 and x.lower() not in stops]:
                word_freqs[w] = word_freqs.get(w, 0) + 1
        word_freqsl = word_freqs.items()
        word_freqsl.sort(lambda x, y: cmp(y[1], x[1]))
        data[filename] = word_freqsl

    if args == None:
        return error_state()

    filename = args[0]
    try:
        create_data(filename)
    except:
        return error_state()

    return word_get_handler([filename, 0])

def word_get_handler(args):
    def get_word(filename, word_index):
        if word_index < len(data[filename]):
            return data[filename][word_index]
        else:
            return ("no more words", 0)

    filename = args[0]; word_index = args[1]
    word_info = get_word(filename, word_index)
    rep = '\n#{0}: {1} - {2}'.format(word_index+1, word_info[0], word_info[1])
    rep += "\n\nWhat would you like to do next?"
    rep += "\n1 - Quit" + "\n2 - Upload file"
    rep += "\n3 - See next most-frequently occurring word"
    links = {"1" : ["post", "execution", None],
             "2" : ["get", "file_form", None],
             "3" : ["get", "word", [filename, word_index+1]]}
    return rep, links

# Handler registration
handlers = {"post_execution" : quit_handler,
            "get_default" : default_get_handler,
            "get_file_form" : upload_get_handler,
            "post_file" : upload_post_handler,
            "get_word" : word_get_handler }

# The "server" core
def handle_request(verb, uri, args):
    def handler_key(verb, uri):
        return verb + "_" + uri

    if handler_key(verb, uri) in handlers:
        return handlers[handler_key(verb, uri)](args)
    else:
        return handlers[handler_key("get", "default")](args)

# A very simple client "browser"
def render_and_get_input(state_representation, links):
    print state_representation
    sys.stdout.flush()
    if type(links) is dict: # many possible next states
        input = sys.stdin.readline().strip()
        if input in links:
            return links[input]
        else:
            return ["get", "default", None]
    elif type(links) is list: # only one possible next state
        if links[0] == "post": # get "form" data
            input = sys.stdin.readline().strip()
            links.append([input]) # add the data at the end
            return links
        else: # get action, don't get user input
            return links
    else:
        return ["get", "default", None]

request = ["get", "default", None]
while True:
    # "server"-side computation
    state_representation, links = handle_request(*request)
    # "client"-side computation
    request = render_and_get_input(state_representation, links)
<|endoftext|>


<commit_before>"""Provide API client for TVDB API v2.0 services."""
import json
import logging

import cachecontrol
from cachecontrol import caches
from oslo_config import cfg
import requests
import six

from tvdbapi_client import exceptions
from tvdbapi_client import timeutil

LOG = logging.getLogger(__name__)

cfg.CONF.import_group('tvdb', 'tvdbapi_client.options')

DEFAULT_HEADERS = {
    'Accept-Language': 'en',
    'Content-Type': 'application/json',
}

SERIES_BY = [
    'name',
    'imdbId',
    'zap2itId',
]

EPISODES_BY = [
    'airedSeason',
    'airedEpisode',
    'imdbId',
    'dvdSeason',
    'dvdEpisode',
    'absoluteNumber',
    'page',
]

def requires_auth(func):
    """Handle authentication checks.

    .. py:decorator:: requires_auth

        Checks if the token has expired and performs authentication if needed.
    """
    @six.wraps(func)
    def wrapper(self, *args, **kwargs):
        if self.token_expired:
            self.authenticate()
        return func(self, *args, **kwargs)
    return wrapper

class TVDBClient(object):

    """TVDB Api client."""

    def __init__(self, apikey=None, username=None, userpass=None):
        """Create new instance of API client.

        :param str apikey: apikey from thetvdb
        :param str username: username used on thetvdb
        :param str userpass: password used on thetvdb
        """
        self.__apikey = apikey or cfg.CONF.tvdb.apikey
        self.__username = username or cfg.CONF.tvdb.username
        self.__userpass = userpass or cfg.CONF.tvdb.userpass
        self.__token = None

        self._token_timer = None
        self._session = None
        self._headers = DEFAULT_HEADERS
        self._language = 'en'

    @property
    def headers(self):
        """Provide access to updated headers."""
        self._headers.update(**{'Accept-Language': self.language})
        if self.__token:
            self._headers.update(
                **{'Authorization': 'Bearer %s' % self.__token})
        return self._headers

    @property
    def language(self):
        """Provide access to current language."""
        return self._language

    @language.setter
    def language(self, abbr):
        """Provide access to update language."""
        self._language = abbr

    @property
    def token_expired(self):
        """Provide access to flag indicating if token has expired."""
        if self._token_timer is None:
            return True
        return timeutil.is_newer_than(self._token_timer, timeutil.ONE_HOUR)

    @property
    def session(self):
        """Provide access to request session with local cache enabled."""
        if self._session is None:
            self._session = cachecontrol.CacheControl(
                requests.Session(),
                cache=caches.FileCache('.tvdb_cache'))
        return self._session

    @exceptions.error_map
    def _exec_request(self, service, method=None, path_args=None, data=None,
                      params=None):
        """Execute request."""
        if path_args is None:
            path_args = []

        req = {
            'method': method or 'get',
            'url': '/'.join(str(a).strip('/') for a in [
                cfg.CONF.tvdb.service_url, service] + path_args),
            'data': json.dumps(data) if data else None,
            'headers': self.headers,
            'params': params,
            'verify': cfg.CONF.tvdb.verify_ssl_certs,
        }

        LOG.debug('executing request (%s %s)', req['method'], req['url'])
        resp = self.session.request(**req)

        resp.raise_for_status()
        return resp.json() if resp.text else resp.text

    def _login(self):
        data = {'apikey': self.__apikey,
                'username': self.__username,
                'userpass': self.__userpass,
                }
        return self._exec_request('login', method='post', data=data)

    def _refresh_token(self):
        return self._exec_request('refresh_token')

    def authenticate(self):
        """Aquire authorization token for using thetvdb apis."""
        if self.__token:
            try:
                resp = self._refresh_token()
            except exceptions.TVDBRequestException as err:
                # if a 401 is the cause try to login
                if getattr(err.response, 'status_code', 0) == 401:
                    resp = self._login()
                else:
                    raise
        else:
            resp = self._login()

        self.__token = resp.get('token')
        self._token_timer = timeutil.utcnow()

    @requires_auth
    def search_series(self, **kwargs):
        """Provide the ability to search for a series.

        .. warning::

            authorization token required

        The following search arguments currently supported:

            * name
            * imdbId
            * zap2itId

        :param kwargs: keyword arguments to search for series
        :returns: series record or series records
        :rtype: dict
        """
        params = {}
        for arg, val in six.iteritems(kwargs):
            if arg in SERIES_BY:
                params[arg] = val
        resp = self._exec_request(
            'search', path_args=['series'], params=params)
        if cfg.CONF.tvdb.select_first:
            return resp['data'][0]
        return resp['data']

    @requires_auth
    def get_series(self, series_id):
        """Retrieve series record.

        .. warning::

            authorization token required

        :param str series_id: id of series as found on thetvdb
        :returns: series record
        :rtype: dict
        """
        return self._exec_request('series', path_args=[series_id])['data']

    @requires_auth
    def get_episodes(self, series_id, **kwargs):
        """All episodes for a given series.

        Paginated with 100 results per page.

        .. warning::

            authorization token required

        The following search arguments currently supported:

            * airedSeason
            * airedEpisode
            * imdbId
            * dvdSeason
            * dvdEpisode
            * absoluteNumber
            * page

        :param str series_id: id of series as found on thetvdb
        :parm kwargs: keyword args to search/filter episodes by (optional)
        :returns: series episode records
        :rtype: list
        """
        params = {'page': 1}
        for arg, val in six.iteritems(kwargs):
            if arg in EPISODES_BY:
                params[arg] = val
        return self._exec_request(
            'series',
            path_args=[series_id, 'episodes', 'query'], params=params)['data']

    @requires_auth
    def get_episodes_summary(self, series_id):
        """Return a summary of the episodes and seasons for the series.

        .. warning::

            authorization token required

        .. note::

            Season "0" is for all episodes that are considered to be specials.

        :param str series_id: id of series as found on thetvdb
        :returns: summary of the episodes and seasons for the series
        :rtype: dict
        """
        return self._exec_request(
            'series', path_args=[series_id, 'episodes', 'summary'])['data']

    @requires_auth
    def get_series_image_info(self, series_id):
        """Return a summary of the images for a particular series.

        .. warning::

            authorization token required

        :param str series_id: id of series as found on thetvdb
        :returns: summary of the images for the series
        :rtype: dict
        """
        return self._exec_request(
            'series', path_args=[series_id, 'images'])['data']

    @requires_auth
    def get_episode(self, episode_id):
        """Return the full information for a given episode id.

        .. warning::

            authorization token required

        :param str episode_id: id of episode as found on thetvdb
        :returns: episode record
        :rtype: dict
        """
        return self._exec_request('episodes', path_args=[episode_id])['data']
<commit_msg>change userpass to userkey so we can authenticate<commit_after>"""Provide API client for TVDB API v2.0 services."""
import json
import logging

import cachecontrol
from cachecontrol import caches
from oslo_config import cfg
import requests
import six

from tvdbapi_client import exceptions
from tvdbapi_client import timeutil

LOG = logging.getLogger(__name__)

cfg.CONF.import_group('tvdb', 'tvdbapi_client.options')

DEFAULT_HEADERS = {
    'Accept-Language': 'en',
    'Content-Type': 'application/json',
}

SERIES_BY = [
    'name',
    'imdbId',
    'zap2itId',
]

EPISODES_BY = [
    'airedSeason',
    'airedEpisode',
    'imdbId',
    'dvdSeason',
    'dvdEpisode',
    'absoluteNumber',
    'page',
]

def requires_auth(func):
    """Handle authentication checks.

    .. py:decorator:: requires_auth

        Checks if the token has expired and performs authentication if needed.
    """
    @six.wraps(func)
    def wrapper(self, *args, **kwargs):
        if self.token_expired:
            self.authenticate()
        return func(self, *args, **kwargs)
    return wrapper

class TVDBClient(object):

    """TVDB Api client."""

    def __init__(self, apikey=None, username=None, userpass=None):
        """Create new instance of API client.

        :param str apikey: apikey from thetvdb
        :param str username: username used on thetvdb
        :param str userpass: password used on thetvdb
        """
        self.__apikey = apikey or cfg.CONF.tvdb.apikey
        self.__username = username or cfg.CONF.tvdb.username
        self.__userpass = userpass or cfg.CONF.tvdb.userpass
        self.__token = None

        self._token_timer = None
        self._session = None
        self._headers = DEFAULT_HEADERS
        self._language = 'en'

    @property
    def headers(self):
        """Provide access to updated headers."""
        self._headers.update(**{'Accept-Language': self.language})
        if self.__token:
            self._headers.update(
                **{'Authorization': 'Bearer %s' % self.__token})
        return self._headers

    @property
    def language(self):
        """Provide access to current language."""
        return self._language

    @language.setter
    def language(self, abbr):
        """Provide access to update language."""
        self._language = abbr

    @property
    def token_expired(self):
        """Provide access to flag indicating if token has expired."""
        if self._token_timer is None:
            return True
        return timeutil.is_newer_than(self._token_timer, timeutil.ONE_HOUR)

    @property
    def session(self):
        """Provide access to request session with local cache enabled."""
        if self._session is None:
            self._session = cachecontrol.CacheControl(
                requests.Session(),
                cache=caches.FileCache('.tvdb_cache'))
        return self._session

    @exceptions.error_map
    def _exec_request(self, service, method=None, path_args=None, data=None,
                      params=None):
        """Execute request."""
        if path_args is None:
            path_args = []

        req = {
            'method': method or 'get',
            'url': '/'.join(str(a).strip('/') for a in [
                cfg.CONF.tvdb.service_url, service] + path_args),
            'data': json.dumps(data) if data else None,
            'headers': self.headers,
            'params': params,
            'verify': cfg.CONF.tvdb.verify_ssl_certs,
        }

        LOG.debug('executing request (%s %s)', req['method'], req['url'])
        resp = self.session.request(**req)

        resp.raise_for_status()
        return resp.json() if resp.text else resp.text

    def _login(self):
        data = {'apikey': self.__apikey,
                'username': self.__username,
                'userkey': self.__userpass,
                }
        return self._exec_request('login', method='post', data=data)

    def _refresh_token(self):
        return self._exec_request('refresh_token')

    def authenticate(self):
        """Aquire authorization token for using thetvdb apis."""
        if self.__token:
            try:
                resp = self._refresh_token()
            except exceptions.TVDBRequestException as err:
                # if a 401 is the cause try to login
                if getattr(err.response, 'status_code', 0) == 401:
                    resp = self._login()
                else:
                    raise
        else:
            resp = self._login()

        self.__token = resp.get('token')
        self._token_timer = timeutil.utcnow()

    @requires_auth
    def search_series(self, **kwargs):
        """Provide the ability to search for a series.

        .. warning::

            authorization token required

        The following search arguments currently supported:

            * name
            * imdbId
            * zap2itId

        :param kwargs: keyword arguments to search for series
        :returns: series record or series records
        :rtype: dict
        """
        params = {}
        for arg, val in six.iteritems(kwargs):
            if arg in SERIES_BY:
                params[arg] = val
        resp = self._exec_request(
            'search', path_args=['series'], params=params)
        if cfg.CONF.tvdb.select_first:
            return resp['data'][0]
        return resp['data']

    @requires_auth
    def get_series(self, series_id):
        """Retrieve series record.

        .. warning::

            authorization token required

        :param str series_id: id of series as found on thetvdb
        :returns: series record
        :rtype: dict
        """
        return self._exec_request('series', path_args=[series_id])['data']

    @requires_auth
    def get_episodes(self, series_id, **kwargs):
        """All episodes for a given series.

        Paginated with 100 results per page.

        .. warning::

            authorization token required

        The following search arguments currently supported:

            * airedSeason
            * airedEpisode
            * imdbId
            * dvdSeason
            * dvdEpisode
            * absoluteNumber
            * page

        :param str series_id: id of series as found on thetvdb
        :parm kwargs: keyword args to search/filter episodes by (optional)
        :returns: series episode records
        :rtype: list
        """
        params = {'page': 1}
        for arg, val in six.iteritems(kwargs):
            if arg in EPISODES_BY:
                params[arg] = val
        return self._exec_request(
            'series',
            path_args=[series_id, 'episodes', 'query'], params=params)['data']

    @requires_auth
    def get_episodes_summary(self, series_id):
        """Return a summary of the episodes and seasons for the series.

        .. warning::

            authorization token required

        .. note::

            Season "0" is for all episodes that are considered to be specials.

        :param str series_id: id of series as found on thetvdb
        :returns: summary of the episodes and seasons for the series
        :rtype: dict
        """
        return self._exec_request(
            'series', path_args=[series_id, 'episodes', 'summary'])['data']

    @requires_auth
    def get_series_image_info(self, series_id):
        """Return a summary of the images for a particular series.

        .. warning::

            authorization token required

        :param str series_id: id of series as found on thetvdb
        :returns: summary of the images for the series
        :rtype: dict
        """
        return self._exec_request(
            'series', path_args=[series_id, 'images'])['data']

    @requires_auth
    def get_episode(self, episode_id):
        """Return the full information for a given episode id.

        .. warning::

            authorization token required

        :param str episode_id: id of episode as found on thetvdb
        :returns: episode record
        :rtype: dict
        """
        return self._exec_request('episodes', path_args=[episode_id])['data']
<|endoftext|>


<commit_before># coding=utf-8
from __future__ import unicode_literals

base_drive = '\\\movies-pc'
<commit_msg>remove private settings file<commit_after><|endoftext|>