<commit_before>"""
Functions to calculate stats for each OSM contributor
"""
import MapGardening
import logging
from datetime import datetime, timedelta
import numpy

userstatstable = "usercounts"

class UserStats(object):

    conn = None
    cur = None
    utc_offset = 0
    nodetableobj = None
    blankspottableobj = None

    def __init__(self, connection, cursor, nodetableobj, blankspottableobj):
        """
        Initialize
        """
        self.conn = connection
        self.cur = cursor
        self.nodetableobj = nodetableobj
        self.blankspottableobj = blankspottableobj

    def drop_userstats_table(self):
        """
        Drop the userstats table.
        """

        querystring = "DROP TABLE IF EXISTS " + userstatstable
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            if str(inst).find("does not exist") != -1:
                # if exception is because table does not exist, do nothing
                pass
            else:
                logging.error("can't drop userstats table")
                logging.error(inst)
        self.conn.commit()

    def create_userstats_table(self):
        """
        Create a table called userstats in the currently connected database
        TODO: check whether table exists

        Populates table with uid, username, and count of number of times username
        was mentioned in the node table. (This is the number of edits the user is
        responsible for)
        """

        querystring = "CREATE TABLE " + userstatstable + " AS SELECT uid, username, count(username) from " +\
            self.nodetableobj.getTableName() + " GROUP BY uid, username ORDER BY count DESC"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't select username and count")
            logging.error(inst)
        self.conn.commit()

    def add_userstats_countedits(self, queryparam = "", newcolumn = "newcount"):
        """
        Count number of edits per user, then add as new column to userstats table.

        By default, counts all the edits in the database (using an empty query parameter).
        To count v1 edits, use queryparam = "WHERE version=1".
        This function can also be called by a number of convenience functions that
        specify which type of edits to count (all, only v1, etc).
        """

        temptablename = userstatstable + "_temp" + newcolumn

        querystring = "CREATE TEMP TABLE " + temptablename + \
            " AS " + \
            "SELECT a.uid, a.username, count(a.uid) AS " + newcolumn + " " + \
            "FROM " + self.nodetableobj.getTableName() + " a " + \
            "LEFT JOIN " + self.blankspottableobj.getTableName() + " b " + \
            "ON a.id = b.node_id " + queryparam + " " + \
            "GROUP BY a.uid, a.username ORDER BY " + newcolumn + " DESC"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            if str(inst).find("already exists") != -1:
                self.cur.execute("DROP TABLE " + temptablename) # drop the old table
                self.conn.commit()
                self.cur.execute(querystring) # and create a new one
                self.conn.commit()
            else:
                logging.error("can't create temp table")
                logging.error(inst)
        self.conn.commit()

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " integer"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        querystring = "UPDATE " + userstatstable + " " + \
            "SET " + newcolumn + " = (" + \
            "SELECT " + newcolumn + " FROM " + \
            temptablename + " WHERE " + \
            temptablename + ".uid = " + userstatstable + ".uid)"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't update new column")
            logging.error(inst)
        self.conn.commit()

    def add_userstats_v1edits(self):
        """
        Count number of v1 edits (creation of nodes) by each user
        """
        return self.add_userstats_countedits("WHERE a.version=1", "v1count")

    def add_userstats_blankedits(self):
        """
        Count number of "blank spot" edits by each user
        """
        return self.add_userstats_countedits("WHERE a.version=1 AND b.blank=true", "blankcount")

    def add_userstats_firstedit(self, queryparam = "", newcolumn = "firstedit"):
        """
        Add date of each user's first edit (of any version)
        If their first edit was not v1
        """

        # Note: select "distinct on" is postgresql-specific.

        temptablename = userstatstable + "_temp" + newcolumn

        querystring = "CREATE TEMP TABLE " + temptablename + \
            " AS " + \
            "SELECT DISTINCT ON (a.username) a.username, a.valid_from AS " + newcolumn + " " + \
            "FROM " + self.nodetableobj.getTableName() + " a " + \
            "LEFT JOIN " + self.blankspottableobj.getTableName() + " b " + \
            "ON a.id = b.node_id " + queryparam + " " + \
            "ORDER BY a.username, a.valid_from ASC"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            if str(inst).find("already exists") != -1:
                self.cur.execute("DROP TABLE " + temptablename) # drop the old table
                self.conn.commit()
                self.cur.execute(querystring) # and create a new one
                self.conn.commit()
            else:
                logging.error("can't create temp table")
                logging.error(inst)
        self.conn.commit()

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " date"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        querystring = "UPDATE " + userstatstable + " " \
            "SET " + newcolumn + " = (" + \
            "SELECT " + newcolumn + " from " + \
            temptablename + " WHERE " + \
            temptablename + ".username = " + userstatstable + ".username)"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't update new column")
            logging.error(inst)
        self.conn.commit()

    def add_userstats_firstedit_v1(self):
        """
        Add date of each user's first v1 edit... these are edits that
        create new features, not modify existing features.
        """

        return self.add_userstats_firstedit("WHERE a.version=1", "firsteditv1")

    def add_userstats_firstedit_blank(self):
        """
        Add date of each user's first blankspot edit... these are edits that
        create new features, in areas where no features previously existed
        """

        return self.add_userstats_firstedit("WHERE a.version=1 AND b.blank=true", "firsteditblank")

    def _days_since_epoch_obj(self, input_date_obj):
        """
        Given a date object, return number of days since epoch.
        This makes it easier to calculate the mean date of activity.
        """
        epoch = datetime.utcfromtimestamp(0)
        delta = input_date_obj - epoch
        return delta.days

    def _days_since_epoch(self, input_date_str):
        """
        Given a date string of format YYYY-MM-DD return number od days since epoch.
        """
        date_obj = datetime.strptime(input_date_str, "%Y-%m-%d")
        return self._days_since_epoch_obj(date_obj)

    def _date_from_days_since_epoch(self, days_since_epoch_int):
        """
        Reverse the calculation from _days_since_epoch(). Given an integer
        representing the number of days since Jan 1, 1970, return a date object
        for that date.
        """
        epoch = datetime.utcfromtimestamp(0)
        delta = timedelta(days=days_since_epoch_int)
        date_obj = epoch + delta
        return date_obj

    def get_dates_and_edit_counts(self, count_all=False):
        """
        Count number of edits for each date.
        Count all edits, v1 edits, and blankspot edits separately.
        Create two dictionaries, one to count edits for each user,
        and one to count edits overall.
        """

        print "gathering dates and edit counts"
        user_date_dict = {}
        all_date_dict = {}

        querystring = "SELECT a.username, a.valid_from, a.version, b.blank " + \
            "FROM " + self.nodetableobj.getTableName() + " a " + \
            "LEFT JOIN " + self.blankspottableobj.getTableName() + " b " + \
            "ON a.id = b.node_id "
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't select from nodes table")
            logging.error(inst)

        rows = self.cur.fetchall()

        if rows == None:
            return ""

        for row in rows:
            (username, valid_from, version, blank) = row
            if not username in user_date_dict:
                user_date_dict[username] = {}

            # Shift utc datetimes to local time
            td = timedelta(hours=self.utc_offset)
            edit_dt = valid_from + td

            # Return a date object (drop time info) from the datetime object
            edit_date_str = str(edit_dt.date())

            if not edit_date_str in user_date_dict[username]:
                user_date_dict[username][edit_date_str] = {'all': 0, 'v1': 0, 'blank': 0}
            if not edit_date_str in all_date_dict:
                all_date_dict[edit_date_str] = {'all': 0, 'v1': 0, 'blank': 0}

            user_date_dict[username][edit_date_str]['all'] += 1
            all_date_dict[edit_date_str]['all'] += 1
            if version == 1:
                user_date_dict[username][edit_date_str]['v1'] += 1
                all_date_dict[edit_date_str]['v1'] += 1
                if blank == True: # version must == 1, as well.
                    user_date_dict[username][edit_date_str]['blank'] += 1
                    all_date_dict[edit_date_str]['blank'] += 1
        if count_all:
            return all_date_dict
        else:
            return user_date_dict

    def add_userstats_mean_date(self, weighted=False, user_date_dict=None):
        """
        Of all the days the mapper is active, find the average (mean) date of activity
        """
        if user_date_dict == None:
            user_date_dict = self.get_dates_and_edit_counts()

        status = "calculating mean edit dates"

        newcolumn = "mean_date"
        if weighted == True:
            newcolumn = newcolumn + "_weighted"
            status += " (weighted)"

        print status

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " date"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        # Get keys for each user dict (list of dates)
        # convert keys to integers, find mean, convert back to date.
        for username in user_date_dict:
            date_keys = user_date_dict[username].keys()
            if weighted:
                weights = []
                for dk in date_keys:
                    weights.append(user_date_dict[username][dk]['all'])
            else:
                weights = None
            list_of_dates = map(self._days_since_epoch, date_keys)
            mean_date = round(numpy.average(list_of_dates, None, weights))
            mean_date_dt = self._date_from_days_since_epoch(mean_date)
            querystring = "UPDATE " + userstatstable + " " + \
                "SET " + newcolumn + " = %s " + \
                "WHERE username = %s"
            try:
                self.cur.execute(querystring, (mean_date_dt, username))
            except Exception, inst:
                logging.error("can't update mean_date column")
                logging.error(inst)
        self.conn.commit()

    def add_userstats_days_active(self, user_date_dict=None):
        """
        Count the number of days (calendar days in each region's timezone) the mapper edited the db
        """
        if user_date_dict == None:
            user_date_dict = self.get_dates_and_edit_counts()

        print "counting days active"

        newcolumn = "days_active"

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " integer"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        for username in user_date_dict:
            days_active = len(user_date_dict[username])
            querystring = "UPDATE " + userstatstable + " " + \
                "SET " + newcolumn + " = %s " + \
                "WHERE username = %s"
            try:
                self.cur.execute(querystring, (days_active, username))
            except Exception, inst:
                logging.error("can't update days_active column")
                logging.error(inst)
        self.conn.commit()

    def print_userstats(self, filename):
        querystring = "SELECT uid, username, count, blankcount, v1count, firstedit, firsteditv1, firsteditblank, days_active, mean_date, mean_date_weighted FROM " + userstatstable
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't select count and blankcount")
            logging.error(inst)

        localfile = open(filename, "w")

        print >> localfile, "uid\tusername\tcount\tblankcount\tv1count\tfirstedit\tfirsteditv1\tfirsteditblank\tdays_active\tmean_date\tmean_date_weighted"

        rows = self.cur.fetchall()

        nullvalue = "NULL"

        for row in rows:
            # assign nullvalue instead of None
            uid = row[0] or nullvalue
            username = row[1] or nullvalue
            count = row[2] or 0
            blankcount = row[3] or 0
            v1count = row[4] or 0
            firstedit = row[5] or nullvalue
            firsteditv1 = row[6] or nullvalue
            firsteditblank = row[7] or nullvalue
            days_active = row[8] or 0
            mean_date = row[9] or nullvalue
            mean_date_weighted = row[10] or nullvalue
            print >> localfile, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % (uid, username, count, blankcount, v1count, firstedit, firsteditv1, firsteditblank, days_active, mean_date, mean_date_weighted)

        localfile.close()

    def print_placestats(self, filename):
        all_date_dict = self.get_dates_and_edit_counts(count_all=True)

        localfile = open(filename, "w")

        print >> localfile, "date\tcount\tv1count\tblankcount"

        for date in sorted(all_date_dict):
            print >> localfile, "%s\t%s\t%s\t%s" % (date, all_date_dict[date]['all'], all_date_dict[date]['v1'], all_date_dict[date]['blank'])

        localfile.close()<commit_msg>Add more informative docstrings, drop temp tables<commit_after>"""
Functions to calculate stats for each OSM contributor.

The UserStats class examines two tables, the "node table" (usually hist_point)
and the "blank spot" table (usually taking the most recent table listed in
the "blank spot manager" table). The results of the analysis are stored in
a third table, the "userstatstable".

The UserStats class then prints user stats and place stats (taking all the
users together) as separate tsv files.
"""
import MapGardening
import logging
from datetime import datetime, timedelta
import numpy

userstatstable = "usercounts"

class UserStats(object):

    conn = None
    cur = None
    utc_offset = 0
    nodetableobj = None
    blankspottableobj = None

    def __init__(self, connection, cursor, nodetableobj, blankspottableobj):
        """
        Initialize
        """
        self.conn = connection
        self.cur = cursor
        self.nodetableobj = nodetableobj
        self.blankspottableobj = blankspottableobj

    def drop_userstats_table(self):
        """
        Drop the userstats table.
        """

        querystring = "DROP TABLE IF EXISTS " + userstatstable
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            if str(inst).find("does not exist") != -1:
                # if exception is because table does not exist, do nothing
                pass
            else:
                logging.error("can't drop userstats table")
                logging.error(inst)
        self.conn.commit()

    def create_userstats_table(self):
        """
        Create a table called userstats in the currently connected database

        NOTE: Call drop_userstats_table() first if there is a chance it already exists.

        Populates table with uid, username, and count of number of times username
        was mentioned in the node table. (This is the number of edits the user is
        responsible for)
        """

        print "creating userstats table"

        querystring = "CREATE TABLE " + userstatstable + " AS SELECT uid, username, count(username) from " +\
            self.nodetableobj.getTableName() + " GROUP BY uid, username ORDER BY count DESC"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't select username and count")
            logging.error(inst)
        self.conn.commit()

    def add_userstats_countedits(self, queryparam = "", newcolumn = "newcount"):
        """
        Count number of edits per user, then add as new column to userstats table.

        By default, counts all the edits in the database (using an empty query parameter).
        To count v1 edits, use queryparam = "WHERE version=1".
        This function can also be called by a number of convenience functions that
        specify which type of edits to count (all, only v1, etc).

        TODO: change this so we don't create the same almost-identical temp table
        every time the convenience functions are called.
        """

        temptablename = userstatstable + "_temp" + newcolumn

        querystring = "CREATE TEMP TABLE " + temptablename + \
            " AS " + \
            "SELECT a.uid, a.username, count(a.uid) AS " + newcolumn + " " + \
            "FROM " + self.nodetableobj.getTableName() + " a " + \
            "LEFT JOIN " + self.blankspottableobj.getTableName() + " b " + \
            "ON a.id = b.node_id " + queryparam + " " + \
            "GROUP BY a.uid, a.username ORDER BY " + newcolumn + " DESC"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            if str(inst).find("already exists") != -1:
                self.cur.execute("DROP TABLE " + temptablename) # drop the old table
                self.conn.commit()
                self.cur.execute(querystring) # and create a new one
                self.conn.commit()
            else:
                logging.error("can't create temp table")
                logging.error(inst)
        self.conn.commit()

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " integer"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        querystring = "UPDATE " + userstatstable + " " + \
            "SET " + newcolumn + " = (" + \
            "SELECT " + newcolumn + " FROM " + \
            temptablename + " WHERE " + \
            temptablename + ".uid = " + userstatstable + ".uid)"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't update new column")
            logging.error(inst)
        self.conn.commit()

        querystring = "DROP TABLE " + temptablename
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't drop temp table")
            logging.error(inst)
        self.conn.commit()

    def add_userstats_v1edits(self):
        """
        Count number of v1 edits (creation of nodes) by each user
        """
        print "counting v1 edits"
        return self.add_userstats_countedits("WHERE a.version=1", "v1count")

    def add_userstats_blankedits(self):
        """
        Count number of "blank spot" edits by each user
        """
        return self.add_userstats_countedits("WHERE a.version=1 AND b.blank=true", "blankcount")

    def add_userstats_firstedit(self, queryparam = "", newcolumn = "firstedit"):
        """
        Add date of each user's first edit.

        Type of edit (version 1 or blankspot) is specified in queryparam.

        TODO: change this so we don't create the same almost-identical temp table
        every time the convenience functions are called.
        """

        # Note: select "distinct on" is postgresql-specific.

        temptablename = userstatstable + "_temp" + newcolumn

        querystring = "CREATE TEMP TABLE " + temptablename + \
            " AS " + \
            "SELECT DISTINCT ON (a.username) a.username, a.valid_from AS " + newcolumn + " " + \
            "FROM " + self.nodetableobj.getTableName() + " a " + \
            "LEFT JOIN " + self.blankspottableobj.getTableName() + " b " + \
            "ON a.id = b.node_id " + queryparam + " " + \
            "ORDER BY a.username, a.valid_from ASC"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            if str(inst).find("already exists") != -1:
                self.cur.execute("DROP TABLE " + temptablename) # drop the old table
                self.conn.commit()
                self.cur.execute(querystring) # and create a new one
                self.conn.commit()
            else:
                logging.error("can't create temp table")
                logging.error(inst)
        self.conn.commit()

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " date"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        querystring = "UPDATE " + userstatstable + " " \
            "SET " + newcolumn + " = (" + \
            "SELECT " + newcolumn + " from " + \
            temptablename + " WHERE " + \
            temptablename + ".username = " + userstatstable + ".username)"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't update new column")
            logging.error(inst)
        self.conn.commit()

        querystring = "DROP TABLE " + temptablename
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't drop temp table")
            logging.error(inst)
        self.conn.commit()

    def add_userstats_firstedit_v1(self):
        """
        Add date of each user's first v1 edit... these are edits that
        create new features, not modify existing features.
        """

        return self.add_userstats_firstedit("WHERE a.version=1", "firsteditv1")

    def add_userstats_firstedit_blank(self):
        """
        Add date of each user's first blankspot edit... these are edits that
        create new features, in areas where no features previously existed
        """

        return self.add_userstats_firstedit("WHERE a.version=1 AND b.blank=true", "firsteditblank")

    def _days_since_epoch_obj(self, input_date_obj):
        """
        Given a date object, return number of days since epoch.
        This makes it easier to calculate the mean date of activity.
        """
        epoch = datetime.utcfromtimestamp(0)
        delta = input_date_obj - epoch
        return delta.days

    def _days_since_epoch(self, input_date_str):
        """
        Given a date string of format YYYY-MM-DD return number od days since epoch.
        """
        date_obj = datetime.strptime(input_date_str, "%Y-%m-%d")
        return self._days_since_epoch_obj(date_obj)

    def _date_from_days_since_epoch(self, days_since_epoch_int):
        """
        Reverse the calculation from _days_since_epoch(). Given an integer
        representing the number of days since Jan 1, 1970, return a date object
        for that date.
        """
        epoch = datetime.utcfromtimestamp(0)
        delta = timedelta(days=days_since_epoch_int)
        date_obj = epoch + delta
        return date_obj

    def get_dates_and_edit_counts(self, count_all=False):
        """
        Count number of edits for each date.
        Count all edits, v1 edits, and blankspot edits separately.
        Create two dictionaries, one to count edits for each user,
        and one to count edits overall.
        """

        print "gathering dates and edit counts"
        user_date_dict = {}
        all_date_dict = {}

        querystring = "SELECT a.username, a.valid_from, a.version, b.blank " + \
            "FROM " + self.nodetableobj.getTableName() + " a " + \
            "LEFT JOIN " + self.blankspottableobj.getTableName() + " b " + \
            "ON a.id = b.node_id "
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't select from nodes table")
            logging.error(inst)

        rows = self.cur.fetchall()

        if rows == None:
            return ""

        for row in rows:
            (username, valid_from, version, blank) = row
            if not username in user_date_dict:
                user_date_dict[username] = {}

            # Shift utc datetimes to local time
            td = timedelta(hours=self.utc_offset)
            edit_dt = valid_from + td

            # Return a date object (drop time info) from the datetime object
            edit_date_str = str(edit_dt.date())

            if not edit_date_str in user_date_dict[username]:
                user_date_dict[username][edit_date_str] = {'all': 0, 'v1': 0, 'blank': 0}
            if not edit_date_str in all_date_dict:
                all_date_dict[edit_date_str] = {'all': 0, 'v1': 0, 'blank': 0}

            user_date_dict[username][edit_date_str]['all'] += 1
            all_date_dict[edit_date_str]['all'] += 1
            if version == 1:
                user_date_dict[username][edit_date_str]['v1'] += 1
                all_date_dict[edit_date_str]['v1'] += 1
                if blank == True: # version must == 1, as well.
                    user_date_dict[username][edit_date_str]['blank'] += 1
                    all_date_dict[edit_date_str]['blank'] += 1
        if count_all:
            return all_date_dict
        else:
            return user_date_dict

    def add_userstats_mean_date(self, weighted=False, user_date_dict=None):
        """
        Of all the days the mapper is active, find the average (mean) date of activity
        """
        if user_date_dict == None:
            user_date_dict = self.get_dates_and_edit_counts()

        status = "calculating mean edit dates"

        newcolumn = "mean_date"
        if weighted == True:
            newcolumn = newcolumn + "_weighted"
            status += " (weighted)"

        print status

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " date"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        # Get keys for each user dict (list of dates)
        # convert keys to integers, find mean, convert back to date.
        for username in user_date_dict:
            date_keys = user_date_dict[username].keys()
            if weighted:
                weights = []
                for dk in date_keys:
                    weights.append(user_date_dict[username][dk]['all'])
            else:
                weights = None
            list_of_dates = map(self._days_since_epoch, date_keys)
            mean_date = round(numpy.average(list_of_dates, None, weights))
            mean_date_dt = self._date_from_days_since_epoch(mean_date)
            querystring = "UPDATE " + userstatstable + " " + \
                "SET " + newcolumn + " = %s " + \
                "WHERE username = %s"
            try:
                self.cur.execute(querystring, (mean_date_dt, username))
            except Exception, inst:
                logging.error("can't update mean_date column")
                logging.error(inst)
        self.conn.commit()

    def add_userstats_days_active(self, user_date_dict=None):
        """
        Count the number of days (calendar days in each region's timezone) the mapper edited the db
        """
        if user_date_dict == None:
            user_date_dict = self.get_dates_and_edit_counts()

        print "counting days active"

        newcolumn = "days_active"

        querystring = "ALTER TABLE " + userstatstable + " ADD " + newcolumn + " integer"
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            self.conn.rollback()
            logging.warning("can't add new column")
            logging.warning(inst)
        self.conn.commit()

        for username in user_date_dict:
            days_active = len(user_date_dict[username])
            querystring = "UPDATE " + userstatstable + " " + \
                "SET " + newcolumn + " = %s " + \
                "WHERE username = %s"
            try:
                self.cur.execute(querystring, (days_active, username))
            except Exception, inst:
                logging.error("can't update days_active column")
                logging.error(inst)
        self.conn.commit()

    def print_userstats(self, filename):
        querystring = "SELECT uid, username, count, blankcount, v1count, firstedit, firsteditv1, firsteditblank, days_active, mean_date, mean_date_weighted FROM " + userstatstable
        try:
            self.cur.execute(querystring)
        except Exception, inst:
            logging.error("can't select count and blankcount")
            logging.error(inst)

        localfile = open(filename, "w")

        print >> localfile, "uid\tusername\tcount\tblankcount\tv1count\tfirstedit\tfirsteditv1\tfirsteditblank\tdays_active\tmean_date\tmean_date_weighted"

        rows = self.cur.fetchall()

        nullvalue = "NULL"

        for row in rows:
            # assign nullvalue instead of None
            uid = row[0] or nullvalue
            username = row[1] or nullvalue
            count = row[2] or 0
            blankcount = row[3] or 0
            v1count = row[4] or 0
            firstedit = row[5] or nullvalue
            firsteditv1 = row[6] or nullvalue
            firsteditblank = row[7] or nullvalue
            days_active = row[8] or 0
            mean_date = row[9] or nullvalue
            mean_date_weighted = row[10] or nullvalue
            print >> localfile, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % (uid, username, count, blankcount, v1count, firstedit, firsteditv1, firsteditblank, days_active, mean_date, mean_date_weighted)

        localfile.close()

    def print_placestats(self, filename):
        all_date_dict = self.get_dates_and_edit_counts(count_all=True)

        localfile = open(filename, "w")

        print >> localfile, "date\tcount\tv1count\tblankcount"

        for date in sorted(all_date_dict):
            print >> localfile, "%s\t%s\t%s\t%s" % (date, all_date_dict[date]['all'], all_date_dict[date]['v1'], all_date_dict[date]['blank'])

        localfile.close()<|endoftext|>


<commit_before>from .data import Resource
import os
import sys
import pandas as pd
import gzip
import urllib3
import signal
import json
import Queue
import re
import corenlp.server
from bs4 import BeautifulSoup

class WikiListResource(Resource):
    """Manager of wikipedia page lists for each language model/event type."""

    def __init__(self):
        Resource.__init__(self)
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'wiki-lists')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

        self.type2cats = {
            u'accident': ['Category:Accidents'],
            u'impact event': ['Category:Astronomy'],
            u'bombing': ['Category:Terrorism'],
            u'hostage': ['Category:Terrorism'],
            u'shooting': ['Category:Murder'],
            u'protest': ['Category:Activism_by_type'],
            u'riot': ['Category:Activism_by_type'],
            u'storm': ['Category:Natural_disasters'],
            u'earthquake': ['Category:Natural_disasters'],
        }

        self.type2fname = {
            u'accident': 'accident.tsv.gz',
            u'impact event': 'astronomy.tsv.gz',
            u'bombing': 'terrorism.tsv.gz',
            u'hostage': 'terrorism.tsv.gz',
            u'shooting': 'murder.tsv.gz',
            u'protest': 'social-unrest.tsv.gz',
            u'riot': 'social-unrest.tsv.gz',
            u'storm': 'natural-disaster.tsv.gz',
            u'earthquake': 'natural-disaster.tsv.gz',
        }

    def __unicode__(self):
        return u"cuttsum.lm.WikiListResource"

    def get_list_path(self, event):
        return os.path.join(self.dir_, self.type2fname[event.type])

    def check_coverage(self, event, corpus, **kwargs):
        list_path = self.get_list_path(event)
        if os.path.exists(list_path):
            return 1
        else:
            return 0

    def _query(self, request, http):
        url = 'http://en.wikipedia.org/w/api.php'
        request['action'] = 'query'
        request['format'] = 'json'
        request['continue'] = ''
        last_continue = dict()
        while True:

            req = request.copy()
            req.update(last_continue)
            r = http.request_encode_url('GET', url, fields=req)
            result = json.loads(r.data)

            if 'error' in result:
                print result['error']
                break
            if 'warnings' in result: print(result['warnings'])
            if 'query' in result: yield result['query']
            if 'continue' not in result: break
            last_continue = result['continue']

    def get(self, event, corpus, max_depth=7, overwrite=False, n_procs=1,
            **kwargs):

        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)
        list_path = self.get_list_path(event)

        good_pages = []

        queue = [(cat, 0) for cat in self.type2cats[event.type]]
        url = 'http://en.wikipedia.org/w/api.php'
        http = urllib3.connection_from_url(url)

        visited = set()
        depth = 0
        while len(queue) > 0:

            cat, depth = queue.pop()
            sys.stdout.write('\r')
            sys.stdout.write(' ' * 79)
            sys.stdout.write('\r')
            sys.stdout.write("Categories in queue: {}".format(len(queue)))
            sys.stdout.flush()

            req = {
                'generator':'categorymembers',
                'gcmtitle': cat
            }

            for result in self._query(req, http):
                if 'pages' in result:
                    for page in result['pages']:
                        m = result['pages'][page]
                        if m['ns'] == 0:
                            title = m['title'].encode('utf-8')
                            if title not in visited:
                                visited.add(title)
                                good_pages.append(
                                    {u'depth': depth,
                                     u'category': cat,
                                     u'title': title})
                        elif m['ns'] == 14:
                            title = m['title'].encode('utf-8')
                            if depth + 1 < max_depth and title not in visited:
                                visited.add(title)
                                queue.append((title, depth + 1))

        good_pages.sort(key=lambda x: x['depth'])

        df = pd.DataFrame(good_pages, columns=[u'depth', u'category', u'title'])

        with gzip.open(list_path, 'w') as f:
            df.to_csv(f, sep='\t')
        return True

    def dependencies(self):
        return tuple([])

class DomainLMInputResource(Resource):
    """Manager of input (one sentence per line docs)
for each language model."""

    def __init__(self):
        Resource.__init__(self)
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'domain-lm-input')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

        self.type2fname = {
            u'accident': 'accident.txt.gz',
            u'impact event': 'astronomy.txt.gz',
            u'bombing': 'terrorism.txt.gz',
            u'hostage': 'terrorism.txt.gz',
            u'shooting': 'murder.txt.gz',
            u'protest': 'social-unrest.txt.gz',
            u'riot': 'social-unrest.txt.gz',
            u'storm': 'natural-disaster.txt.gz',
            u'earthquake': 'natural-disaster.txt.gz',
        }

    def get_domain_lm_input_path(self, event):
        return os.path.join(self.dir_, self.type2fname[event.type])

    def __unicode__(self):
        return u"cuttsum.lm.DomainLMInputResource"

    def check_coverage(self, event, corpus, **kwargs):
        path = self.get_domain_lm_input_path(event)
        if os.path.exists(path):
            return 1
        else:
            return 0

    def get(self, event, corpus, overwrite=False, progress_bar=False,
            n_procs=1, **kwargs):

        n_procs = min(n_procs, 5) # Wikipedia doesn't like us hammering them.

        wikilists = WikiListResource()
        list_path = wikilists.get_list_path(event)
        dmn_lm_input_path = self.get_domain_lm_input_path(event)

        with gzip.open(list_path, u'r') as f:
            df = pd.io.parsers.read_csv(
                f, sep='\t', quoting=3, header=0,
                names=[u'depth', u'category', u'title'])
        titles = df[u'title'].values.astype(list)

        if corenlp.server.check_status() is False:
            print "Starting Corenlp server"
            corenlp.server.start(
                mem="20G", threads=20,
                annotators=['tokenize', 'ssplit', 'pos', 'lemma', 'ner'])

        with gzip.open(dmn_lm_input_path, u'w') as f:
            self.do_work(domainlminput_worker_, titles, n_procs,
                progress_bar, result_handler=domainlminput_writer_(f))

        return True

    def dependencies(self):
        return tuple(['WikiListResource'])

def domainlminput_worker_(job_queue, result_queue):
    signal.signal(signal.SIGINT, signal.SIG_IGN)

    url = 'http://en.wikipedia.org/w/api.php'
    http = urllib3.connection_from_url(url)

    page_url_tmp = 'http://en.wikipedia.org/w/index.php?title={}&oldid={}'

    cnlp = corenlp.server.CoreNLPClient()

    while not job_queue.empty():
        try:

            title = job_queue.get(block=False)

            fields = {'action':'query', 'format':'json', 'prop':'revisions',
                      'titles':title, 'rvprop':'timestamp|ids',
                      'rvstart':'20111001000000', 'rvdir':'older',
                      'continue': ''}

            r = http.request_encode_url('GET', url, fields=fields)
            result = json.loads(r.data)
            pid = result['query']['pages'].keys()[0]
            revs = result['query']['pages'][pid].get('revisions', None)
            if revs is None:
                result_queue.put([])
                continue

            if len(revs) == 0:
                result_queue.put([])
                continue
            rev = revs[0]
            page_url = page_url_tmp.format(
                title.replace(' ', '_'), rev['parentid'])
            html_content = http.request_encode_url('GET', page_url).data

            soup = BeautifulSoup(html_content)

            results = []
            div = soup.find("div", {"id": "mw-content-text"})
            if div is None:
                result_queue.put([])
                continue
            for tag in div.find_all(True, recursive=False):
                if tag.name == 'p':
                    text = tag.get_text()
                    text = re.sub(r'\[\d+\]', '', tag.get_text())
                    text = cnlp.annotate(text)
                    for sent in text:
                        norm_tokens = []
                        for token in sent:
                            if token.ne == 'O':
                                words = token.lem.split(u'_')
                                for word in words:
                                    if word != u'':
                                        norm_tokens.append(word.lower())
                            else:
                                norm_tokens.append(
                                    u'__{}__'.format(token.ne.lower()))
                        results.append(
                            (u' '.join(norm_tokens)).encode(u'utf-8'))

            result_queue.put(results)
        except Queue.Empty:
            pass

def domainlminput_writer_(file):
        def wrapper(result):
            for sent in result:
                file.write(sent)
                file.write('\n')
            file.flush()
        return wrapper

class DomainLMResource(Resource):
    """Manager of arpa files for SRILM."""

    def __init__(self):
        Resource.__init__(self)
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'lm')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

        self.domain_lm_ = {
            u'accident': u'accidents_3.arpa.gz',
            u'shooting': u'murder_3.arpa.gz',
            u'storm': u'natural_disaster_3.arpa.gz',
            u'earthquake': u'natural_disaster_3.arpa.gz',
            u'bombing': u'terrorism_3.arpa.gz',
            u'riot': u'social_unrest_3.arpa.gz',
            u'protest': u'social_unrest_3.arpa.gz',
            u'hostage': u'terrorism_3.arpa.gz',
            u'impact event': u'astronomy_3.arpa.gz',
        }

        self.ports_ = {
            u'accidents_3.arpa.gz':        9998,
            u'terrorism_3.arpa.gz':        9997,
            u'natural_disaster_3.arpa.gz': 9996,
            u'murder_3.arpa.gz':           9995,
            u'social_unrest_3.arpa.gz':    9994,
            u'astronomy_3.arpa.gz':        9993,
        }

    def get_port(self, event):
        lm = self.domain_lm_[event.type]
        return self.ports_[lm]

    def get_arpa_path(self, event):
        return os.path.join(self.dir_, self.domain_lm_[event.type])

    def __unicode__(self):
        return u"cuttsum.lm.DomainLMResource"

    def check_coverage(self, event, corpus, **kwargs):
        if os.path.exists(self.get_arpa_path(event)):
            return 1.0
        else:
            return 0.0

    def get(self, event, corpus, **kwargs):
        lminputs = DomainLMInputResource()
        lminput_path = lminputs.get_domain_lm_input_path(event)
        arpa_path = self.get_arpa_path(event)

        cmd = 'ngram-count -order 3 -kndiscount -interpolate ' \
              '-text {} -lm {}'.format(lminput_path, arpa_path)

        os.system(cmd)

    def dependencies(self):
        return tuple(['DomainLMInputResource'])

class GigawordLMResource(Resource):
    def __init__(self):
        Resource.__init__(self)
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'lm')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

        self.gigaword_lm_ = u'gigaword_5.arpa.gz'
        self.ports_ = 9999

    def get_gigaword_port(self):
        return self.ports_

    def gigaword_path(self):
        return os.path.join(self.dir_, self.gigaword_lm_)

    def __unicode__(self):
        return u"cuttsum.lm.GigawordLMResource"

    def check_coverage(self, event, corpus, **kwargs):
        coverage = 0.0
        if os.path.exists(self.gigaword_path()):
            coverage = 1.0

        return coverage

    def get(self, event, corpus, **kwargs):
        raise NotImplementedError(
            "I don't know how to make a language model from scratch yet")

    def dependencies(self):
        return tuple([])
<commit_msg>Silenced queue status of wikilist getter when piped to log<commit_after>from .data import Resource
import os
import sys
import pandas as pd
import gzip
import urllib3
import signal
import json
import Queue
import re
import corenlp.server
from bs4 import BeautifulSoup

class WikiListResource(Resource):
    """Manager of wikipedia page lists for each language model/event type."""

    def __init__(self):
        Resource.__init__(self)
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'wiki-lists')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

        self.type2cats = {
            u'accident': ['Category:Accidents'],
            u'impact event': ['Category:Astronomy'],
            u'bombing': ['Category:Terrorism'],
            u'hostage': ['Category:Terrorism'],
            u'shooting': ['Category:Murder'],
            u'protest': ['Category:Activism_by_type'],
            u'riot': ['Category:Activism_by_type'],
            u'storm': ['Category:Natural_disasters'],
            u'earthquake': ['Category:Natural_disasters'],
        }

        self.type2fname = {
            u'accident': 'accident.tsv.gz',
            u'impact event': 'astronomy.tsv.gz',
            u'bombing': 'terrorism.tsv.gz',
            u'hostage': 'terrorism.tsv.gz',
            u'shooting': 'murder.tsv.gz',
            u'protest': 'social-unrest.tsv.gz',
            u'riot': 'social-unrest.tsv.gz',
            u'storm': 'natural-disaster.tsv.gz',
            u'earthquake': 'natural-disaster.tsv.gz',
        }

    def __unicode__(self):
        return u"cuttsum.lm.WikiListResource"

    def get_list_path(self, event):
        return os.path.join(self.dir_, self.type2fname[event.type])

    def check_coverage(self, event, corpus, **kwargs):
        list_path = self.get_list_path(event)
        if os.path.exists(list_path):
            return 1
        else:
            return 0

    def _query(self, request, http):
        url = 'http://en.wikipedia.org/w/api.php'
        request['action'] = 'query'
        request['format'] = 'json'
        request['continue'] = ''
        last_continue = dict()
        while True:

            req = request.copy()
            req.update(last_continue)
            r = http.request_encode_url('GET', url, fields=req)
            result = json.loads(r.data)

            if 'error' in result:
                print result['error']
                break
            if 'warnings' in result: print(result['warnings'])
            if 'query' in result: yield result['query']
            if 'continue' not in result: break
            last_continue = result['continue']

    def get(self, event, corpus, max_depth=7, overwrite=False, n_procs=1,
            **kwargs):

        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)
        list_path = self.get_list_path(event)

        good_pages = []

        queue = [(cat, 0) for cat in self.type2cats[event.type]]
        url = 'http://en.wikipedia.org/w/api.php'
        http = urllib3.connection_from_url(url)

        visited = set()
        depth = 0
        while len(queue) > 0:

            cat, depth = queue.pop()
            if sys.stdout.isatty():
                sys.stdout.write('\r')
                sys.stdout.write(' ' * 79)
                sys.stdout.write('\r')
                sys.stdout.write("Categories in queue: {}".format(len(queue)))
                sys.stdout.flush()

            req = {
                'generator':'categorymembers',
                'gcmtitle': cat
            }

            for result in self._query(req, http):
                if 'pages' in result:
                    for page in result['pages']:
                        m = result['pages'][page]
                        if m['ns'] == 0:
                            title = m['title'].encode('utf-8')
                            if title not in visited:
                                visited.add(title)
                                good_pages.append(
                                    {u'depth': depth,
                                     u'category': cat,
                                     u'title': title})
                        elif m['ns'] == 14:
                            title = m['title'].encode('utf-8')
                            if depth + 1 < max_depth and title not in visited:
                                visited.add(title)
                                queue.append((title, depth + 1))

        good_pages.sort(key=lambda x: x['depth'])

        df = pd.DataFrame(good_pages, columns=[u'depth', u'category', u'title'])

        with gzip.open(list_path, 'w') as f:
            df.to_csv(f, sep='\t')
        return True

    def dependencies(self):
        return tuple([])

class DomainLMInputResource(Resource):
    """Manager of input (one sentence per line docs)
for each language model."""

    def __init__(self):
        Resource.__init__(self)
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'domain-lm-input')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

        self.type2fname = {
            u'accident': 'accident.txt.gz',
            u'impact event': 'astronomy.txt.gz',
            u'bombing': 'terrorism.txt.gz',
            u'hostage': 'terrorism.txt.gz',
            u'shooting': 'murder.txt.gz',
            u'protest': 'social-unrest.txt.gz',
            u'riot': 'social-unrest.txt.gz',
            u'storm': 'natural-disaster.txt.gz',
            u'earthquake': 'natural-disaster.txt.gz',
        }

    def get_domain_lm_input_path(self, event):
        return os.path.join(self.dir_, self.type2fname[event.type])

    def __unicode__(self):
        return u"cuttsum.lm.DomainLMInputResource"

    def check_coverage(self, event, corpus, **kwargs):
        path = self.get_domain_lm_input_path(event)
        if os.path.exists(path):
            return 1
        else:
            return 0

    def get(self, event, corpus, overwrite=False, progress_bar=False,
            n_procs=1, **kwargs):

        n_procs = min(n_procs, 5) # Wikipedia doesn't like us hammering them.

        wikilists = WikiListResource()
        list_path = wikilists.get_list_path(event)
        dmn_lm_input_path = self.get_domain_lm_input_path(event)

        with gzip.open(list_path, u'r') as f:
            df = pd.io.parsers.read_csv(
                f, sep='\t', quoting=3, header=0,
                names=[u'depth', u'category', u'title'])
        titles = df[u'title'].values.astype(list)

        if corenlp.server.check_status() is False:
            print "Starting Corenlp server"
            corenlp.server.start(
                mem="20G", threads=20,
                annotators=['tokenize', 'ssplit', 'pos', 'lemma', 'ner'])

        with gzip.open(dmn_lm_input_path, u'w') as f:
            self.do_work(domainlminput_worker_, titles, n_procs,
                progress_bar, result_handler=domainlminput_writer_(f))

        return True

    def dependencies(self):
        return tuple(['WikiListResource'])

def domainlminput_worker_(job_queue, result_queue):
    signal.signal(signal.SIGINT, signal.SIG_IGN)

    url = 'http://en.wikipedia.org/w/api.php'
    http = urllib3.connection_from_url(url)

    page_url_tmp = 'http://en.wikipedia.org/w/index.php?title={}&oldid={}'

    cnlp = corenlp.server.CoreNLPClient()

    while not job_queue.empty():
        try:

            title = job_queue.get(block=False)

            fields = {'action':'query', 'format':'json', 'prop':'revisions',
                      'titles':title, 'rvprop':'timestamp|ids',
                      'rvstart':'20111001000000', 'rvdir':'older',
                      'continue': ''}

            r = http.request_encode_url('GET', url, fields=fields)
            result = json.loads(r.data)
            pid = result['query']['pages'].keys()[0]
            revs = result['query']['pages'][pid].get('revisions', None)
            if revs is None:
                result_queue.put([])
                continue

            if len(revs) == 0:
                result_queue.put([])
                continue
            rev = revs[0]
            page_url = page_url_tmp.format(
                title.replace(' ', '_'), rev['parentid'])
            html_content = http.request_encode_url('GET', page_url).data

            soup = BeautifulSoup(html_content)

            results = []
            div = soup.find("div", {"id": "mw-content-text"})
            if div is None:
                result_queue.put([])
                continue
            for tag in div.find_all(True, recursive=False):
                if tag.name == 'p':
                    text = tag.get_text()
                    text = re.sub(r'\[\d+\]', '', tag.get_text())
                    text = cnlp.annotate(text)
                    for sent in text:
                        norm_tokens = []
                        for token in sent:
                            if token.ne == 'O':
                                words = token.lem.split(u'_')
                                for word in words:
                                    if word != u'':
                                        norm_tokens.append(word.lower())
                            else:
                                norm_tokens.append(
                                    u'__{}__'.format(token.ne.lower()))
                        results.append(
                            (u' '.join(norm_tokens)).encode(u'utf-8'))

            result_queue.put(results)
        except Queue.Empty:
            pass

def domainlminput_writer_(file):
        def wrapper(result):
            for sent in result:
                file.write(sent)
                file.write('\n')
            file.flush()
        return wrapper

class DomainLMResource(Resource):
    """Manager of arpa files for SRILM."""

    def __init__(self):
        Resource.__init__(self)
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'lm')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

        self.domain_lm_ = {
            u'accident': u'accidents_3.arpa.gz',
            u'shooting': u'murder_3.arpa.gz',
            u'storm': u'natural_disaster_3.arpa.gz',
            u'earthquake': u'natural_disaster_3.arpa.gz',
            u'bombing': u'terrorism_3.arpa.gz',
            u'riot': u'social_unrest_3.arpa.gz',
            u'protest': u'social_unrest_3.arpa.gz',
            u'hostage': u'terrorism_3.arpa.gz',
            u'impact event': u'astronomy_3.arpa.gz',
        }

        self.ports_ = {
            u'accidents_3.arpa.gz':        9998,
            u'terrorism_3.arpa.gz':        9997,
            u'natural_disaster_3.arpa.gz': 9996,
            u'murder_3.arpa.gz':           9995,
            u'social_unrest_3.arpa.gz':    9994,
            u'astronomy_3.arpa.gz':        9993,
        }

    def get_port(self, event):
        lm = self.domain_lm_[event.type]
        return self.ports_[lm]

    def get_arpa_path(self, event):
        return os.path.join(self.dir_, self.domain_lm_[event.type])

    def __unicode__(self):
        return u"cuttsum.lm.DomainLMResource"

    def check_coverage(self, event, corpus, **kwargs):
        if os.path.exists(self.get_arpa_path(event)):
            return 1.0
        else:
            return 0.0

    def get(self, event, corpus, **kwargs):
        lminputs = DomainLMInputResource()
        lminput_path = lminputs.get_domain_lm_input_path(event)
        arpa_path = self.get_arpa_path(event)

        cmd = 'ngram-count -order 3 -kndiscount -interpolate ' \
              '-text {} -lm {}'.format(lminput_path, arpa_path)

        os.system(cmd)

    def dependencies(self):
        return tuple(['DomainLMInputResource'])

class GigawordLMResource(Resource):
    def __init__(self):
        Resource.__init__(self)
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'lm')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

        self.gigaword_lm_ = u'gigaword_5.arpa.gz'
        self.ports_ = 9999

    def get_gigaword_port(self):
        return self.ports_

    def gigaword_path(self):
        return os.path.join(self.dir_, self.gigaword_lm_)

    def __unicode__(self):
        return u"cuttsum.lm.GigawordLMResource"

    def check_coverage(self, event, corpus, **kwargs):
        coverage = 0.0
        if os.path.exists(self.gigaword_path()):
            coverage = 1.0

        return coverage

    def get(self, event, corpus, **kwargs):
        raise NotImplementedError(
            "I don't know how to make a language model from scratch yet")

    def dependencies(self):
        return tuple([])
<|endoftext|>


<commit_before>from gpiozero import MotionSensor
from picamera import PiCamera
from datetime import datetime
import RPi.GPIO as GPIO
import time
import sys
import subprocess

relay_pin = 17

GPIO.setmode(GPIO.BCM)

pir = MotionSensor(4)
camera = PiCamera()

camera.vflip = True
camera.hflip = True

GPIO.setwarnings(False)
GPIO.setup(relay_pin, GPIO.OUT)
GPIO.output(relay_pin, 1)

print("start")

try:
    while True:
        time.sleep(0.5)

        pir.wait_for_motion()
        print("motion")
        GPIO.output(relay_pin, 0)
        filename = "/home/pi/videos/" + datetime.now().strftime("%Y-%m-%d_%H.%M.%S.h264")
        camera.start_recording(filename)

        pir.wait_for_no_motion()
        print("no motion")
        GPIO.output(relay_pin, 1)
        camera.stop_recording()
        print("uploading to Dropbox")
        p = subprocess.Popen([sys.executable, 'dropboxUploader.py', filename],
                           stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT)
        p = subprocess.Popen(["mpack", "-s", "motion detected", filename, "nadapapa@gmail.com"],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)

        print("uploaded to Dropbox")
finally:
    camera.close()
    pir.close()
    GPIO.cleanup() # this ensures a clean exit<commit_msg>add email<commit_after>from gpiozero import MotionSensor
from picamera import PiCamera
from datetime import datetime
import RPi.GPIO as GPIO
import time
import sys
import subprocess

relay_pin = 17

GPIO.setmode(GPIO.BCM)

pir = MotionSensor(4)
camera = PiCamera()

camera.vflip = True
camera.hflip = True

GPIO.setwarnings(False)
GPIO.setup(relay_pin, GPIO.OUT)
GPIO.output(relay_pin, 1)

print("start")

try:
    while True:
        time.sleep(0.5)

        pir.wait_for_motion()
        print("motion")
        GPIO.output(relay_pin, 0)
        filename = "/home/pi/videos/" + datetime.now().strftime("%Y-%m-%d_%H.%M.%S.h264")
        camera.capture("/home/pi/motion.jpg")
        camera.start_recording(filename)

        pir.wait_for_no_motion()
        print("no motion")
        GPIO.output(relay_pin, 1)
        camera.stop_recording()
        print("uploading to Dropbox")
        p = subprocess.Popen([sys.executable, 'dropboxUploader.py', filename],
                           stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT)
        p = subprocess.Popen(["mpack", "-s", "motion detected", "/home/pi/motion.jpg", "nadapapa@gmail.com"],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)

        print("uploaded to Dropbox")
finally:
    camera.close()
    pir.close()
    GPIO.cleanup() # this ensures a clean exit<|endoftext|>


<commit_before># This file is part of Viper - https://github.com/botherder/viper
# See the file 'LICENSE' for copying permission.

import time
import datetime

from viper.common.out import *
from viper.common.objects import File
from viper.core.database import Database

class Session(object):
    def __init__(self):
        self.id = None
        # This will be assigned with the File object of the file currently
        # being analyzed.
        self.file = None
        # Timestamp of the creation of the session.
        self.created_at = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')

class Sessions(object):
    def __init__(self):
        self.current = None
        self.sessions = []
        # Store the results of the last "find" command.
        self.find = None

    def close(self):
        self.current = None

    def is_set(self):
        # Check if the session has been opened or not.
        if self.current:
            return True
        else:
            return False

    def switch(self, session):
        self.current = session
        print_info("Switched to session #{0} on {1}".format(self.current.id, self.current.file.path))

    def new(self, path):
        session = Session()

        total = len(self.sessions)
        session.id = total + 1

        # Open a section on the given file.
        session.file = File(path)

        # Try to lookup the file in the database. If it is already present
        # we get file name and
        row = Database().find(key='sha256', value=session.file.sha256)
        if row:
            session.file.name = row[0].name
            session.file.tags = ', '.join(tag.to_dict()['tag'] for tag in row[0].tag)

        self.sessions.append(session)
        self.current = session

        print_info("Session opened on {0}".format(path))

__sessions__ = Sessions()
<commit_msg>Added TODO note<commit_after># This file is part of Viper - https://github.com/botherder/viper
# See the file 'LICENSE' for copying permission.

import time
import datetime

from viper.common.out import *
from viper.common.objects import File
from viper.core.database import Database

class Session(object):
    def __init__(self):
        self.id = None
        # This will be assigned with the File object of the file currently
        # being analyzed.
        self.file = None
        # Timestamp of the creation of the session.
        self.created_at = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')

class Sessions(object):
    def __init__(self):
        self.current = None
        self.sessions = []
        # Store the results of the last "find" command.
        self.find = None

    def close(self):
        self.current = None

    def is_set(self):
        # Check if the session has been opened or not.
        if self.current:
            return True
        else:
            return False

    def switch(self, session):
        self.current = session
        print_info("Switched to session #{0} on {1}".format(self.current.id, self.current.file.path))

    def new(self, path):
        # TODO: sould replace an existing session if the file is the same
        # otherwise we will have multiple entries for the same file.

        session = Session()

        total = len(self.sessions)
        session.id = total + 1

        # Open a section on the given file.
        session.file = File(path)

        # Try to lookup the file in the database. If it is already present
        # we get file name and
        row = Database().find(key='sha256', value=session.file.sha256)
        if row:
            session.file.name = row[0].name
            session.file.tags = ', '.join(tag.to_dict()['tag'] for tag in row[0].tag)

        self.sessions.append(session)
        self.current = session

        print_info("Session opened on {0}".format(path))

__sessions__ = Sessions()
<|endoftext|>


<commit_before>#!/usr/bin/python

import argparse
import base64
import math
import json
from suffix_tree import SuffixTree
from Crypto.Cipher import AES
import hashlib
import random
import string

import prp

parser = argparse.ArgumentParser()
parser.add_argument('--filename', '-f', default='../files/shakespeare_poem.txt',
                    help='Path of file to encrypt')
args = parser.parse_args()

def getIndexOfInnerNode(node, length):
  while node.firstChild is not None:
    node = node.firstChild
  return length - len(node.pathLabel) + 1

def F(key, plaintext):
  keyHash = hashlib.sha256(key.encode()).hexdigest()
  ptxtHash = hashlib.sha256(plaintext.encode()).hexdigest()
  return hashlib.sha256((ptxtHash + keyHash).encode()).hexdigest()

BS = 16
pad = lambda s: s + (BS - len(s) % BS) * chr(BS - len(s) % BS)
unpad = lambda s : s[0:-ord(s[-1])]

K_1 = 'This is a key789'
K_3 = 'This is a key987'
K_4 = 'This is a key191'
K_D = 'This is a key123'
K_C = 'This is a key234'
K_L = 'This is a key777'
IV_D = 'This is an IV456'
IV_C = 'This is an IV567'
IV_L = 'This is an IV777'

K_1 += K_1
K_3 += K_3
K_4 += K_4
K_D += K_D
K_C += K_C
K_L += K_L

filename = args.filename

# key = 'This is a key12 This is a key12 '
# text = 'string of size16'
# iv = 'This is an IV456'
# aes = AES.new(key, AES.MODE_CBC, iv)
# ciphertext = aes.encrypt(pad(text))
# ciphertext = base64.b64encode(ciphertext)
#
# aes = AES.new(key, AES.MODE_CBC, iv)
# plaintext = aes.decrypt(base64.b64decode(ciphertext))
# print unpad(plaintext)

st = None
length = None
with open(filename) as f:
  print 'Constructing suffix tree...'
  st = SuffixTree(f.read())
  print 'done'
  length = len(st.string) - 1

print 'Encrypting suffix tree...'
D = {}
print '\tProcessing leaves...'
for leaf in st.leaves:
  leafPL = leaf.pathLabel[:len(leaf.pathLabel)-1] + '$'
  parentPL = leaf.parent.pathLabel
  initPath = leafPL[:len(parentPL)+1]
  key = F(K_1, initPath)

  index = str(length - len(leaf.pathLabel) + 1)
  leafpos = str(leaf.erdex)
  num_leaves = str(leaf.numLeaves)
  together = index + "---" + leafpos + "---" + num_leaves

  aes_D = AES.new(K_D, AES.MODE_CBC, IV_D)
  raw = aes_D.encrypt(pad(together))
  value = base64.b64encode(raw)

  D[key] = value

print '\tProcessing inner nodes...'
for innerNode in st.innerNodes:
  if innerNode.stringDepth == 0:
    continue
  nodePL = innerNode.pathLabel
  parentPL = innerNode.parent.pathLabel
  initPath = nodePL[:len(parentPL)+1]
  key = F(K_1, initPath)

  index = str(getIndexOfInnerNode(innerNode, length))
  leafpos = str(innerNode.erdex)
  num_leaves = str(innerNode.numLeaves)
  together = index + "---" + leafpos + "---" + num_leaves

  aes_D = AES.new(K_D, AES.MODE_CBC, IV_D)
  raw = aes_D.encrypt(pad(together))
  value = base64.b64encode(raw)

  D[key] = value

print 'done'

print 'constructing C...'
C = [None] * (len(st.string) - 1)
for i, c in enumerate(st.string[:-1]):
  aes_C = AES.new(K_C, AES.MODE_CBC, IV_C)
  raw = aes_C.encrypt(pad(c))
  value = base64.b64encode(raw)
  C[i] = value
print '\tdone'

print 'constructing L...'
L = [None] * len(st.string)
for leaf in st.leaves:
  index = str(length - len(leaf.pathLabel) + 1)
  aes_L = AES.new(K_L, AES.MODE_CBC, IV_L)
  raw = aes_L.encrypt(pad(index))
  value = base64.b64encode(raw)
  L[leaf.erdex] = value
print '\tdone'

print 'permuting C...'
C_p = [None] * (2 ** int(math.ceil(math.log(len(C), 2))))
for i in xrange(len(C)):
  p_i = prp.permuteSecure(i, len(C_p), K_3)
  C_p[p_i] = C[i]
for i in xrange(len(C), len(C_p)):
  p_i = prp.permuteSecure(i, len(C_p), K_3)
  c = random.choice(string.ascii_letters)
  aes_C = AES.new(K_C, AES.MODE_CBC, IV_C)
  raw = aes_C.encrypt(pad(c))
  C_p[p_i] = base64.b64encode(raw)
print '\tdone'

print 'permuting L...'
L_p = [None] * (2 ** int(math.ceil(math.log(len(L), 2))))
for i in xrange(len(L)):
  p_i = prp.permuteSecure(i, len(L_p), K_4)
  L_p[p_i] = L[i]
for i in xrange(len(L), len(L_p)):
  p_i = prp.permuteSecure(i, len(L_p), K_4)
  aes_L = AES.new(K_L, AES.MODE_CBC, IV_L)
  raw = aes_L.encrypt(pad(str(i)))
  L_p[p_i] = base64.b64encode(raw)
print '\tdone'

print 'saving to file...'
with open('ciphertext.txt', 'w') as fout:
  fout.write(str(len(D)) + '\n')
  for key in D:
    fout.write(json.dumps({key: D[key]}) + '\n')
  fout.write(str(len(C_p)) + '\n')
  for c in C_p:
    fout.write(c + '\n')
  fout.write(str(len(L_p)) + '\n')
  for l in L_p:
    fout.write(l + '\n')
print 'DONE'

# # to parse ciphertext
# D2 = {}
# C2 = []
# L2 = []
# with open('ciphertext.txt') as f:
#   # read D
#   sizeD = int(f.readline())
#   for i in xrange(sizeD):
#     data = json.loads(f.readline())
#     key = data.keys()[0]
#     D2[key] = data[key]
#   # # check dictionary D
#   # for key in D2:
#   #   aes_D = AES.new(K_D, AES.MODE_CBC, IV_D)
#   #   raw = base64.b64decode(D2[key])
#   #   print key, unpad(aes_D.decrypt(raw))
#
#   # read array C
#   sizeC = int(f.readline())
#   for i in xrange(sizeC):
#     C2.append(f.readline()[:-1])
#     C2[i] = base64.b64decode(C2[i])
#   # # check array C
#   # C3 = [None] * len(C2)
#   # for i, c in enumerate(C2):
#   #   i_unperm = prp.unpermuteSecure(i, len(C2), K_3)
#   #   aes_C = AES.new(K_C, AES.MODE_CBC, IV_C)
#   #   decrypted = unpad(aes_C.decrypt(c))[-1:]
#   #   C3[i_unperm] = decrypted
#   # C4 = ''.join(C3)
#   # print C4
#
#   # read array L
#   sizeL = int(f.readline())
#   for i in xrange(sizeL):
#     L2.append(f.readline()[:-1])
#     L2[i] = base64.b64decode(L2[i])
#   # check array L
#   L3 = [None] * len(L2)
#   for i, c in enumerate(L2):
#     i_unperm = prp.unpermuteSecure(i, len(L2), K_4)
#     aes_L = AES.new(K_L, AES.MODE_CBC, IV_L)
#     decrypted = unpad(aes_L.decrypt(c))
#     L3[i_unperm] = decrypted
#     L3[i] = decrypted
#   print L3
<commit_msg>store encrypted children indices<commit_after>#!/usr/bin/python

import argparse
import base64
import math
import json
from suffix_tree import SuffixTree
from Crypto.Cipher import AES
import hashlib
import random
import string

import prp

parser = argparse.ArgumentParser()
parser.add_argument('--filename', '-f', default='../files/shakespeare_poem.txt',
                    help='Path of file to encrypt')
args = parser.parse_args()

def getIndexOfInnerNode(node, length):
  while node.firstChild is not None:
    node = node.firstChild
  return length - len(node.pathLabel) + 1

def F(key, plaintext):
  keyHash = hashlib.sha256(key.encode()).hexdigest()
  ptxtHash = hashlib.sha256(plaintext.encode()).hexdigest()
  return hashlib.sha256((ptxtHash + keyHash).encode()).hexdigest()

BS = 16
pad = lambda s: s + (BS - len(s) % BS) * chr(BS - len(s) % BS)
unpad = lambda s : s[0:-ord(s[-1])]

K_1 = 'This is a key789' * 2
K_2 = 'This is a key173' * 2
K_3 = 'This is a key987' * 2
K_4 = 'This is a key191' * 2
K_D = 'This is a key123' * 2
K_C = 'This is a key234' * 2
K_L = 'This is a key777' * 2
IV_D = 'This is an IV456'
IV_C = 'This is an IV567'
IV_L = 'This is an IV777'

filename = args.filename

# key = 'This is a key12 This is a key12 '
# text = 'string of size16'
# iv = 'This is an IV456'
# aes = AES.new(key, AES.MODE_CBC, iv)
# ciphertext = aes.encrypt(pad(text))
# ciphertext = base64.b64encode(ciphertext)
#
# aes = AES.new(key, AES.MODE_CBC, iv)
# plaintext = aes.decrypt(base64.b64decode(ciphertext))
# print unpad(plaintext)

st = None
length = None
with open(filename) as f:
  print 'Constructing suffix tree...'
  st = SuffixTree(f.read())
  print 'done'
  length = len(st.string) - 1

print 'Encrypting suffix tree...'
D = {}
print '\tProcessing leaves...'
for leaf in st.leaves:
  leafPL = leaf.pathLabel[:len(leaf.pathLabel)-1] + '$'
  parentPL = leaf.parent.pathLabel
  initPath = leafPL[:len(parentPL)+1]
  key = F(K_1, initPath)

  index = str(length - len(leaf.pathLabel) + 1)
  leafpos = str(leaf.erdex)
  num_leaves = str(leaf.numLeaves)
  together = index + "---" + leafpos + "---" + num_leaves

  aes_D = AES.new(K_D, AES.MODE_CBC, IV_D)
  raw = aes_D.encrypt(pad(together))
  value = base64.b64encode(raw)

  D[key] = (value, [])

print '\tProcessing inner nodes...'
for innerNode in st.innerNodes:
  if innerNode.stringDepth == 0:
    continue
  nodePL = innerNode.pathLabel
  parentPL = innerNode.parent.pathLabel
  initPath = nodePL[:len(parentPL)+1]
  key = F(K_1, initPath)

  index = str(getIndexOfInnerNode(innerNode, length))
  leafpos = str(innerNode.erdex)
  num_leaves = str(innerNode.numLeaves)
  together = index + "---" + leafpos + "---" + num_leaves

  aes_D = AES.new(K_D, AES.MODE_CBC, IV_D)
  raw = aes_D.encrypt(pad(together))
  value = base64.b64encode(raw)

  child = innerNode.firstChild
  child_list = []
  while child:
    path_label = child.pathLabel[:-1] + '$'
    child_list.append(F(K_2, path_label[:len(innerNode.pathLabel)-1]))
    child = child.next
  D[key] = (value, child_list)
print 'done'

print 'constructing C...'
C = [None] * (len(st.string) - 1)
for i, c in enumerate(st.string[:-1]):
  aes_C = AES.new(K_C, AES.MODE_CBC, IV_C)
  raw = aes_C.encrypt(pad(c))
  value = base64.b64encode(raw)
  C[i] = value
print '\tdone'

print 'constructing L...'
L = [None] * len(st.string)
for leaf in st.leaves:
  index = str(length - len(leaf.pathLabel) + 1)
  aes_L = AES.new(K_L, AES.MODE_CBC, IV_L)
  raw = aes_L.encrypt(pad(index))
  value = base64.b64encode(raw)
  L[leaf.erdex] = value
print '\tdone'

#print 'permuting C...'
#C_p = [None] * (2 ** int(math.ceil(math.log(len(C), 2))))
#for i in xrange(len(C)):
#  p_i = prp.permuteSecure(i, len(C_p), K_3)
#  C_p[p_i] = C[i]
#for i in xrange(len(C), len(C_p)):
#  p_i = prp.permuteSecure(i, len(C_p), K_3)
#  c = random.choice(string.ascii_letters)
#  aes_C = AES.new(K_C, AES.MODE_CBC, IV_C)
#  raw = aes_C.encrypt(pad(c))
#  C_p[p_i] = base64.b64encode(raw)
#print '\tdone'
#
#print 'permuting L...'
#L_p = [None] * (2 ** int(math.ceil(math.log(len(L), 2))))
#for i in xrange(len(L)):
#  p_i = prp.permuteSecure(i, len(L_p), K_4)
#  L_p[p_i] = L[i]
#for i in xrange(len(L), len(L_p)):
#  p_i = prp.permuteSecure(i, len(L_p), K_4)
#  aes_L = AES.new(K_L, AES.MODE_CBC, IV_L)
#  raw = aes_L.encrypt(pad(str(i)))
#  L_p[p_i] = base64.b64encode(raw)
#print '\tdone'

print 'saving to file...'
with open('ciphertext.txt', 'w') as fout:
  fout.write(str(len(D)) + '\n')
  for key in D:
    fout.write(json.dumps({key: D[key]}) + '\n')
  fout.write(str(len(C)) + '\n')
  for c in C:
    fout.write(c + '\n')
  fout.write(str(len(L)) + '\n')
  for l in L:
    fout.write(l + '\n')
print 'DONE'

# # to parse ciphertext
# D2 = {}
# C2 = []
# L2 = []
# with open('ciphertext.txt') as f:
#   # read D
#   sizeD = int(f.readline())
#   for i in xrange(sizeD):
#     data = json.loads(f.readline())
#     key = data.keys()[0]
#     D2[key] = data[key]
#   # # check dictionary D
#   # for key in D2:
#   #   aes_D = AES.new(K_D, AES.MODE_CBC, IV_D)
#   #   raw = base64.b64decode(D2[key])
#   #   print key, unpad(aes_D.decrypt(raw))
#
#   # read array C
#   sizeC = int(f.readline())
#   for i in xrange(sizeC):
#     C2.append(f.readline()[:-1])
#     C2[i] = base64.b64decode(C2[i])
#   # # check array C
#   # C3 = [None] * len(C2)
#   # for i, c in enumerate(C2):
#   #   i_unperm = prp.unpermuteSecure(i, len(C2), K_3)
#   #   aes_C = AES.new(K_C, AES.MODE_CBC, IV_C)
#   #   decrypted = unpad(aes_C.decrypt(c))[-1:]
#   #   C3[i_unperm] = decrypted
#   # C4 = ''.join(C3)
#   # print C4
#
#   # read array L
#   sizeL = int(f.readline())
#   for i in xrange(sizeL):
#     L2.append(f.readline()[:-1])
#     L2[i] = base64.b64decode(L2[i])
#   # check array L
#   L3 = [None] * len(L2)
#   for i, c in enumerate(L2):
#     i_unperm = prp.unpermuteSecure(i, len(L2), K_4)
#     aes_L = AES.new(K_L, AES.MODE_CBC, IV_L)
#     decrypted = unpad(aes_L.decrypt(c))
#     L3[i_unperm] = decrypted
#     L3[i] = decrypted
#   print L3
<|endoftext|>


<commit_before>import logging
import urlparse

import requests

from django.conf import settings
from raven.contrib.django.raven_compat.models import client

from common.utils import full_url

from .utils import get_upload_dir

logger = logging.getLogger(__name__)

def start_task(user, source, task_params):
    # Target URL is automatically determined from relevant app label.
    task_url = '{}/'.format(
        urlparse.urljoin(settings.DATA_PROCESSING_URL, source))

    task_params.update({
        'member_id': user.member.member_id,
        's3_key_dir': get_upload_dir(source),
        's3_bucket_name': settings.AWS_STORAGE_BUCKET_NAME,
        'update_url': full_url('/data-import/task-update/'),
    })

    try:
        task_req = requests.post(
            task_url,
            json={'task_params': task_params})
    except requests.exceptions.RequestException:
        logger.error('Error in sending request to data processing')
        logger.error('These were the task params: %s', task_params)

        error_message = 'Error in call to Open Humans Data Processing.'

    if 'task_req' in locals() and not task_req.status_code == 200:
        logger.error('Non-200 response from data processing')
        logger.error('These were the task params: %s', task_params)

        error_message = 'Open Humans Data Processing not returning 200.'

    if 'error_message' in locals():
        if not settings.TESTING:
            client.captureMessage(error_message, error_data=task_params)

        return 'error'

def start_task_for_source(user, source):
    """
    Create a retrieval task for the given user and datafile type.
    """
    user_data = getattr(user, source)

    if hasattr(user_data, 'refresh_from_db'):
        user_data.refresh_from_db()

    return start_task(user=user,
                      source=source,
                      task_params=user_data.get_retrieval_params())
<commit_msg>🔧 change task start parameters<commit_after>import logging
import urlparse

import requests

from django.conf import settings
from raven.contrib.django.raven_compat.models import client

from common.utils import full_url

from .utils import get_upload_dir

logger = logging.getLogger(__name__)

def start_task(user, source, task_params):
    task_url = '{}/'.format(
        urlparse.urljoin(settings.DATA_PROCESSING_URL, source))

    task_params.update({
        'oh_user_id': user.id,
        'oh_member_id': user.member.member_id,
        'oh_update_url': full_url('/data-import/task-update/'),
        's3_key_dir': get_upload_dir(source),
        's3_bucket_name': settings.AWS_STORAGE_BUCKET_NAME,
    })

    try:
        task_req = requests.post(
            task_url,
            params={'key': settings.PRE_SHARED_KEY},
            json={'task_params': task_params})
    except requests.exceptions.RequestException:
        logger.error('Error in sending request to data processing')
        logger.error('These were the task params: %s', task_params)

        error_message = 'Error in call to Open Humans Data Processing.'

    if 'task_req' in locals() and not task_req.status_code == 200:
        logger.error('Non-200 response from data processing')
        logger.error('These were the task params: %s', task_params)

        error_message = 'Open Humans Data Processing not returning 200.'

    if 'error_message' in locals():
        if not settings.TESTING:
            client.captureMessage(error_message, error_data=task_params)

        return 'error'

def start_task_for_source(user, source):
    """
    Create a retrieval task for the given user and datafile type.
    """
    user_data = getattr(user, source)

    if hasattr(user_data, 'refresh_from_db'):
        user_data.refresh_from_db()

    return start_task(user=user,
                      source=source,
                      task_params=user_data.get_retrieval_params())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
        ('movie', '0001_initial'),
    ]

    operations = [
        migrations.AddField(
            model_name='movie',
            name='owner',
<<<<<<< HEAD
            field=models.ForeignKey(related_name='movies', default=1, to=settings.AUTH_USER_MODEL),
=======
            field=models.ForeignKey(default=1, to=settings.AUTH_USER_MODEL),
>>>>>>> 7a05b21aec24c88adbbd8f0b4d4e967d5eb4f11f
            preserve_default=False,
        ),
    ]
<commit_msg>Updated<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
        ('movie', '0001_initial'),
    ]

    operations = [
        migrations.AddField(
            model_name='movie',
            name='owner',
            field=models.ForeignKey(related_name='movies', default=1, to=settings.AUTH_USER_MODEL),
            preserve_default=False,
        ),
    ]
<|endoftext|>


<commit_before>from cloudshell.core.logger import qs_logger
from sandbox_scripts.helpers.Networking.save_restore_mgr import SaveRestoreManager
from sandbox_scripts.helpers.Networking.NetworkingSaveNRestore import *
from cloudshell.helpers.scripts import cloudshell_scripts_helpers as helpers
from sandbox_scripts.QualiEnvironmentUtils.Sandbox import SandboxBase
from cloudshell.api.common_cloudshell_api import CloudShellAPIError
from sandbox_scripts.QualiEnvironmentUtils.Resource import ResourceBase

class EnvironmentTeardownVM:
    REMOVE_DEPLOYED_RESOURCE_ERROR = 153

    def __init__(self):
        self.reservation_id = helpers.get_reservation_context_details().id
        self.logger = qs_logger.get_qs_logger(log_file_prefix="CloudShell Sandbox Teardown",
                                              log_group=self.reservation_id,
                                              log_category='Teardown')
        self.sandbox = SandboxBase(self.reservation_id, self.logger)

    # ---------------------------
    # ---------------------------
    def execute(self):
        self.sandbox.report_info("Beginning VMs cleanup")
        saveNRestoreTool = SaveRestoreManager(self.sandbox)

        filename = "Snapshot_"+self.reservation_id+".txt"
        is_snapshot = False

        # if the current reservation was saved as snapshot we look it by reservation_id
        if saveNRestoreTool.get_storage_manager():
            if saveNRestoreTool.is_snapshot(filename):
                is_snapshot = True
                saveNRestoreTool.delete_src_file(filename)
            else: #if this is a snapshot we look it by blueprint name
                if saveNRestoreTool.is_snapshot():
                    is_snapshot = True

        if is_snapshot:
            self.delete_VM_or_Power_off(to_delete=False)
        else:
            self.delete_VM_or_Power_off(to_delete=True)

    # ---------------------------
    # ---------------------------
    def delete_VM_or_Power_off(self, to_delete=False):
        """
        :param bool to_delete:
        :return:
        """
        # filter out resources not created in this reservation
        resources = self.sandbox.get_root_vm_resources()

        pool = ThreadPool()
        async_results = []
        lock = Lock()
        message_status = {
            "power_off": False,
            "delete": False
        }

        for resource in resources:
            result_obj = pool.apply_async(self._power_off_or_delete_deployed_app,
                                (resource, lock, message_status,to_delete))
            async_results.append(result_obj)

        pool.close()
        pool.join()

        resource_to_delete = []
        for async_result in async_results:
            result = async_result.get()
            if result is not None:
                resource_to_delete.append(result)

        # delete resource - bulk
        if resource_to_delete:
            try:
                self.sandbox.api_session.RemoveResourcesFromReservation(self.reservation_id, resource_to_delete)
            except CloudShellAPIError as exc:
                if exc.code == EnvironmentTeardownVM.REMOVE_DEPLOYED_RESOURCE_ERROR:
                    self.sandbox.report_error(error_message=exc.message,
                              log_message="Error executing RemoveResourcesFromReservation command. "
                                          "Error: {0}".format(exc.message),
                              raise_error=True, write_to_output_window=True)

    # ---------------------------
    # ---------------------------
    def _power_off_or_delete_deployed_app(self, resource, lock, message_status, to_delete):
        """
        :param Lock lock:
        :param (dict of str: Boolean) message_status:
        :param ResourceBase resource:
        :return:
        """
        if resource.model.lower() =="vcenter static vm":
            to_delete = False

        try:
            if to_delete:
                with lock:
                    if not message_status['delete']:
                        message_status['delete'] = True
                        self.sandbox.report_info("Apps are being powered off and deleted...",
                                                 write_to_output_window=True)

                self.sandbox.report_info("Executing 'Delete' on deployed app {0}".format(resource.name))
                return resource.name

            else:
                with lock:
                    if not message_status['power_off']:
                        message_status['power_off'] = True
                        self.sandbox.report_info('Apps are being powering off... ',
                                                 write_to_output_window=True)

                with lock:
                    self.sandbox.report_info("Executing 'Power Off' on deployed app {0}"
                                     .format(resource.name))
                    resource.execute_connected_command(self.sandbox.id,"PowerOff", "power")

                return None

        except Exception as exc:
            err_msg = "Error deleting or powering off deployed app {0}. Error: {1}".format(
                resource.name, str(exc))
            self.sandbox.report_error(err_msg,raise_error=False)
            return None

<commit_msg>reverted changes<commit_after>from cloudshell.core.logger import qs_logger
from sandbox_scripts.helpers.Networking.save_restore_mgr import SaveRestoreManager
from sandbox_scripts.helpers.Networking.NetworkingSaveNRestore import *
from sandbox_scripts.QualiEnvironmentUtils.Sandbox import SandboxBase
from cloudshell.helpers.scripts import cloudshell_scripts_helpers as helpers
from sandbox_scripts.QualiEnvironmentUtils.QualiUtils import QualiError

class EnvironmentTeardownVM:
    REMOVE_DEPLOYED_RESOURCE_ERROR = 153

    def __init__(self):
        self.reservation_id = helpers.get_reservation_context_details().id
        self.logger = qs_logger.get_qs_logger(log_file_prefix="CloudShell Sandbox Teardown",
                                              log_group=self.reservation_id,
                                              log_category='Teardown')
        self.sandbox = None

    def execute(self):

        self.sandbox = SandboxBase(self.reservation_id, self.logger)

        self.sandbox.report_info("Beginning VMs cleanup")

        reservation_details = self.sandbox.api_session.GetReservationDetails(self.reservation_id)

        saveNRestoreTool = SaveRestoreManager(self.sandbox)

        filename = "Snapshot_"+self.reservation_id+".txt"

        is_snapshot = False

        # if the current reservation was saved as snapshot we look it by reservation_id
        if saveNRestoreTool.get_storage_manager():
            if saveNRestoreTool.is_snapshot(filename):
                is_snapshot = True
                saveNRestoreTool.delete_src_file(filename)
            else: #if this is a snapshot we look it by blueprint name
                if saveNRestoreTool.is_snapshot():
                    is_snapshot = True

        if is_snapshot:
            self.delete_VM_or_Power_off(reservation_details, to_delete = False)

        else:
            self.delete_VM_or_Power_off(reservation_details, to_delete =True)

    def delete_VM_or_Power_off(self, reservation_details, to_delete = False):

        """
        :param GetReservationDescriptionResponseInfo reservation_details:
        :param str reservation_id:
        :return:
        """
        # filter out resources not created in this reservation
        resources = reservation_details.ReservationDescription.Resources

        pool = ThreadPool()
        async_results = []
        lock = Lock()
        message_status = {
            "power_off": False,
            "delete": False
        }

        for resource in resources:
            resource_details = self.sandbox.api_session.GetResourceDetails(resource.Name)
            vm_details = resource_details.VmDetails
            if vm_details and hasattr(vm_details, "UID") and vm_details.UID:
                result_obj = pool.apply_async(self._power_off_or_delete_deployed_app,
                                    (resource_details, lock, message_status,to_delete))
                async_results.append(result_obj)

        pool.close()
        pool.join()

        resource_to_delete = []
        for async_result in async_results:
            result = async_result.get()
            if result is not None:
                resource_to_delete.append(result)

        # delete resource - bulk
        if resource_to_delete:
            try:
                self.sandbox.api_session.RemoveResourcesFromReservation(self.reservation_id, resource_to_delete)
            except QualiError as exc:
                if exc.code == EnvironmentTeardownVM.REMOVE_DEPLOYED_RESOURCE_ERROR:
                    self.sandbox.report_error(error_message=exc.message,
                              log_message="Error executing RemoveResourcesFromReservation command. "
                                          "Error: {0}".format(exc.message),
                              raise_error=True, write_to_output_window=True)

    def _power_off_or_delete_deployed_app(self, resource_info, lock, message_status, to_delete):
        """
        :param Lock lock:
        :param (dict of str: Boolean) message_status:
        :param ResourceInfo resource_info:
        :return:
        """
        resource_name = resource_info.Name

        if resource_info.ResourceModelName.lower() =="vcenter static vm":
            to_delete = False

        try:

            if to_delete:
                with lock:
                    if not message_status['delete']:
                        message_status['delete'] = True
                        self.sandbox.report_info("Apps are being powered off and deleted...",
                                                 write_to_output_window=True)

                self.logger.info("Executing 'Delete' on deployed app {0} in reservation {1}"
                                 .format(resource_name, self.reservation_id))

                return resource_name

            else:
                with lock:
                    if not message_status['power_off']:
                        message_status['power_off'] = True
                        self.sandbox.report_info('Apps are being powering off... ',
                                                 write_to_output_window=True)

                with lock:
                    self.logger.info("Executing 'Power Off' on deployed app {0}"
                                     .format(resource_name, self.reservation_id))
                    self.sandbox.api_session.ExecuteResourceConnectedCommand(self.reservation_id, resource_name,
                                                                        "PowerOff", "power")

                return None

        except Exception as exc:
            self.logger.error("Error deleting or powering off deployed app {0} in reservation {1}. Error: {2}"
                              .format(resource_name, self.reservation_id, str(exc)))
            return None

<|endoftext|>


<commit_before>#!/usr/bin/python
#
# DEVELOPMENT CODE DO NOT USE
#
# SolveResolvePhaserClass.py
# Maintained by G.Winter

import os
import sys

if not os.environ['XIA2_ROOT'] in sys.path:
    sys.path.append(os.environ['XIA2_ROOT'])

from Wrappers.Solve.Solve import Solve
from Wrappers.Solve.Resolve import Resolve

from Wrappers.CCP4.Mtzdump import Mtzdump

# interface that this will implement
from HAPhaserClass import HAPhaserClass

class SolveResolvePhaserClass(HAPhaserClass):

    def __init__(self):
        HAPhaserClass.__init__(self)

        # check we have the things we need
        solve = Solve()
        resolve = Resolve()

        return

    def phase(self):

        solve = Solve()
        solve.set_hklin(self._mtz_file)
        for name in self._scalepack_files.keys():
            wavelength = self._input_dict[name]['wavelength']
            fp = self._input_dict[name]['fp']
            fpp = self._input_dict[name]['fpp']
            solve.add_wavelength(name, wavelength, fp, fpp)

        solve.set_n_sites(self._n_sites)
        solve.set_nres(self._nres)
        solve.set_atom(self._atom)

        # need to get the resolution range from the mtz file...

        mtzdump = Mtzdump()
        mtzdump.set_hklin(self._hklin)
        mtzdump.dump()
        resolution_range = mtzdump.get_resolution_range()
        solve.set_resolution_high(min(resolution_range))
        solve.set_resolution_low(max(resolution_range))

        # then run

        solve.close_wait()

        # then run resolve

        resolve = Resolve()
        resolve.set_solvent(self._solvent)
        resolve.close_wait()

        return

if __name__ == '__main__':
    spc = ShelxPhaserClass()
    spc.phase()

<commit_msg>typo shelx -> solve resolve<commit_after>#!/usr/bin/python
#
# DEVELOPMENT CODE DO NOT USE
#
# SolveResolvePhaserClass.py
# Maintained by G.Winter

import os
import sys

if not os.environ['XIA2_ROOT'] in sys.path:
    sys.path.append(os.environ['XIA2_ROOT'])

from Wrappers.Solve.Solve import Solve
from Wrappers.Solve.Resolve import Resolve

from Wrappers.CCP4.Mtzdump import Mtzdump

# interface that this will implement
from HAPhaserClass import HAPhaserClass

class SolveResolvePhaserClass(HAPhaserClass):

    def __init__(self):
        HAPhaserClass.__init__(self)

        # check we have the things we need
        solve = Solve()
        resolve = Resolve()

        return

    def phase(self):

        solve = Solve()
        solve.set_hklin(self._mtz_file)
        for name in self._scalepack_files.keys():
            wavelength = self._input_dict[name]['wavelength']
            fp = self._input_dict[name]['fp']
            fpp = self._input_dict[name]['fpp']
            solve.add_wavelength(name, wavelength, fp, fpp)

        solve.set_n_sites(self._n_sites)
        solve.set_nres(self._nres)
        solve.set_atom(self._atom)

        # need to get the resolution range from the mtz file...

        mtzdump = Mtzdump()
        mtzdump.set_hklin(self._hklin)
        mtzdump.dump()
        resolution_range = mtzdump.get_resolution_range()
        solve.set_resolution_high(min(resolution_range))
        solve.set_resolution_low(max(resolution_range))

        # then run

        solve.close_wait()

        # then run resolve

        resolve = Resolve()
        resolve.set_solvent(self._solvent)
        resolve.close_wait()

        return

if __name__ == '__main__':
    srpc = SolveResolvePhaserClass()
    srpc.phase()

<|endoftext|>