<commit_before>import datetime
import os
from neopysqlite.neopysqlite import Pysqlite, PysqliteCouldNotRetrieveData
from tqdm import tqdm

games = [
    {
        'name': 'Elite: Dangerous',
        'shorthand': 'ED'
     },
    {
        'name': 'Planet Coaster',
        'shorthand': 'PC'
    }
]

def get_stream_durations(stream_times):
    # pop one field out to avoid odd numbered lists
    time_deltas = []
    for i, field in enumerate(stream_times):
        # if i == 0 or i % 2 == 0:
        # current field
        # timestamp format: '2016-01-25 10:46:18'
        date_part = field.split(' ')[0].split('-')
        time_part = field.split(' ')[1].split(':')
        year, month, day = int(date_part[0]), int(date_part[1]), int(date_part[2])
        hour, minute, second = int(time_part[0]), int(time_part[1]), int(time_part[2])
        time1 = datetime.datetime(year=year, month=month, day=day, hour=hour, minute=minute, second=second)
        # caclulate a delta between the current and the next field
        if i+1 == len(stream_times):
            break
        next_field = stream_times[i+1]
        date_part = next_field.split(' ')[0].split('-')
        time_part = next_field.split(' ')[1].split(':')
        year, month, day = int(date_part[0]), int(date_part[1]), int(date_part[2])
        hour, minute, second = int(time_part[0]), int(time_part[1]), int(time_part[2])
        time2 = datetime.datetime(year=year, month=month, day=day, hour=hour, minute=minute, second=second)
        time_deltas.append([time1, time2])
    stream_durations = []
    current_stream_duration = 0
    for time_one, time_two in time_deltas:
        delta = time_two - time_one
        # operate on the assumption that different streams are a minimum of 1 hour away from each other
        if delta.seconds / (60 * 60) > 1:
            # print('DIFFERENT STREAM FOUND')
            stream_durations.append(current_stream_duration)
            current_stream_duration = 0
        else:
            # print('{} {}\n{} {}\n\t{}\t{}s'.format(i, time_one, i, time_two, delta, delta.seconds))
            current_stream_duration += delta.seconds
    else:
        # if the data ends then the last recorded stream will not be recorded, so add the deltas list anyway
        stream_durations.append(current_stream_duration)
    # change it from seconds to hours
    stream_durations = [round(stream / (60 * 60), 2) for stream in stream_durations]
    return stream_durations

class TwitchStatisticsOutput:
    # bounds for the tiers of streamers
    tier_one_bounds = {'upper': 999999, 'lower': 100}
    tier_two_bounds = {'upper': 99, 'lower': 50}
    tier_three_bounds = {'upper': 49, 'lower': 15}
    tier_four_bounds = {'upper': 14, 'lower': 0}

    def __init__(self, game_name, game_shorthand, verbose=False):
        self.name = game_name
        self.shorthand = game_shorthand
        self.db_file_path = os.path.join(os.getcwd(), 'data', '{}_stats.db'.format(game_shorthand))
        self.db = Pysqlite(database_name='twitch_stats', database_file=self.db_file_path)
        self.verbose = verbose

    def run(self):
        if self.verbose:
            print('Processing data for game: {}'.format(self.name))
        tables = self.db.get_table_names()
        tables = [table for table in tables if table not in ['test', 'sqlite_sequence']]
        # get the table names which do not start with a number
        valid_named_tables = [table for table in tables if not table[0][0].isdigit()]
        # get the table names which start with a number
        number_start_tables = [table for table in tables if table[0][0].isdigit()]
        # reod the original table names
        valid_named_tables.extend(['_' + table for table in number_start_tables])
        # initialise list for all the data
        all_streamer_data = []
        # list any streamers to ignore
        streamers_to_ignore = ['legenddolby1986']
        for streamer in tqdm(valid_named_tables):
            if streamer in streamers_to_ignore:
                # skip if its on the ignore list
                continue
            # get the db data from the table of the same name as the streamer and put it in the list^TM
            all_streamer_data.append(self.get_streamer_dict(streamer))
        # write the data to the text file
        self.write_text_file(streamer_data=all_streamer_data)

    def return_streamer_tier(self, average_viewers):
        if self.tier_one_bounds['upper'] >= average_viewers >= self.tier_one_bounds['lower']:
            return 1
        if self.tier_two_bounds['upper'] >= average_viewers >= self.tier_two_bounds['lower']:
            return 2
        if self.tier_three_bounds['upper'] >= average_viewers >= self.tier_three_bounds['lower']:
            return 3
        if self.tier_four_bounds['upper'] >= average_viewers >= self.tier_four_bounds['lower']:
            return 4
        return 0

    def get_streamer_dict(self, streamer):
        streamer_dict = dict()
        streamer_dict['name'] = streamer
        # catch an exception where the table cannot be found and return an empty dictionary instead
        try:
            data = self.db.get_all_rows(table=streamer)
        except PysqliteCouldNotRetrieveData:
            streamer_dict['partnership'] = False
            streamer_dict['tier'] = 4
            streamer_dict['viewers'] = []
            streamer_dict['viewers_max'] = 0
            streamer_dict['viewers_average'] = 0.0
            streamer_dict['followers'] = []
            streamer_dict['followers_max'] = 0
            streamer_dict['times'] = []
            streamer_dict['durations'] = []
            streamer_dict['durations_max'] = 0
            streamer_dict['durations_average'] = 0.0
            streamer_dict['durations_total'] = 0.0
            streamer_dict['stream_count'] = 0
            return streamer_dict
        streamer_dict['partnership'] = data[-1][3] == 1
        viewers = [field[1] for field in data]
        streamer_dict['viewers'] = [field[1] for field in data]
        streamer_dict['viewers_max'] = max(viewers)
        streamer_dict['viewers_average'] = sum(viewers) // len(viewers)
        streamer_dict['tier'] = self.return_streamer_tier(streamer_dict['viewers_average'])
        followers = [field[2] for field in data]
        streamer_dict['followers'] = followers
        streamer_dict['followers_max'] = followers[-1]
        streamer_dict['times'] = [field[4] for field in data]  # times
        streamer_dict['durations'] = get_stream_durations(streamer_dict['times'])
        streamer_dict['durations_max'] = max(streamer_dict['durations'])
        streamer_dict['durations_average'] = round(sum(streamer_dict['durations']), 2)
        streamer_dict['durations_total'] = round(sum(streamer_dict['durations']), 2)
        streamer_dict['stream_count'] = len(streamer_dict['durations'])
        return streamer_dict

    def write_text_file(self, streamer_data):
        durations = [streamer['durations'] for streamer in streamer_data]
        # get the longest consecutive stream
        non_empty_durations = [duration for duration in durations if not duration == []]
        longest_stream = max([max(duration_set) for duration_set in non_empty_durations])
        # calculate total time streamed over all streamers
        total_duration_sums = sum([sum(duration_set) for duration_set in non_empty_durations])
        total_duration = round(total_duration_sums, 2)
        total_streams = 0
        # calculate the total number of discrete streams
        for streamer in streamer_data:
            total_streams += streamer['stream_count']
        text_file_path = os.path.join(os.getcwd(), 'data', '{}_Twitch_Stats.txt'.format(self.shorthand))
        with open(text_file_path, mode='w', encoding='utf-8') as file:
            file.write('{} Twitch Streamer Statistics\n'.format(self.name))
            file.write('Data recorded 24/7 via twitch\'s public API every ~20 seconds\n')
            file.write('Script written by Simon Agius Muscat / CMDR Purrcat\n')
            file.write('More information can be found at: https://github.com/purrcat259/twitch-statistics\n')
            file.write('Total streamers recorded: {}\n'.format(len(streamer_data)))
            file.write('Total streams recorded: {}\n'.format(total_streams))
            file.write('Total time streamed: {} hours\n'.format(total_duration))
            file.write('Longest single stream: {} hours\n'.format(round(longest_stream, 2)))
            file.write('Tier One Bounds: {} >= Average Viewers >= {}\n'.format(self.tier_one_bounds['upper'], self.tier_one_bounds['lower']))
            file.write('Tier One Streamers: {}\n'.format(len([s for s in streamer_data if self.return_streamer_tier(s['viewers_average']) == 1])))
            file.write('Tier Two Bounds: {} >= Average Viewers >= {}\n'.format(self.tier_two_bounds['upper'], self.tier_two_bounds['lower']))
            file.write('Tier Two Streamers: {}\n'.format(len([s for s in streamer_data if self.return_streamer_tier(s['viewers_average']) == 2])))
            file.write('Tier Three Bounds: {} >= Average Viewers >= {}\n'.format(self.tier_three_bounds['upper'], self.tier_three_bounds['lower']))
            file.write('Tier Three Streamers: {}\n'.format(len([s for s in streamer_data if self.return_streamer_tier(s['viewers_average']) == 3])))
            file.write('Tier Four Bounds: {} >= Average Viewers >= {}\n'.format(self.tier_four_bounds['upper'], self.tier_four_bounds['lower']))
            file.write('Tier Four Streamers: {}\n'.format(len([s for s in streamer_data if self.return_streamer_tier(s['viewers_average']) == 4])))
            for streamer in streamer_data:
                # skip streamers with total durations less than 10 minutes
                if streamer['durations_total'] < 0.2:
                    continue
                file.write('\nStreamer: {} (T{})\n'.format(streamer['name'], streamer['tier']))
                file.write('Partnered: {} \n'.format(streamer['partnership']))
                file.write('Average Viewers: {}\n'.format(streamer['viewers_average']))
                file.write('Peak Viewers: {}\n'.format(streamer['viewers_max']))
                file.write('Followers: {}\n'.format(streamer['followers_max']))
                file.write('Stream count: {}\n'.format(streamer['stream_count']))
                file.write('Average Stream duration: {} hours\n'.format(streamer['durations_average']))
                file.write('Longest Stream duration: {} hours\n'.format(streamer['durations_max']))
                file.write('Total time streamed: {} hours\n'.format(streamer['durations_total']))
                time_percentage = round((streamer['durations_total'] / total_duration) * 100, 3)
                file.write('Percentage streamed of total duration: {}%\n'.format(time_percentage))
                file.write('Stream durations:\n')
                for duration in streamer['durations']:
                    if duration < 1.0:
                        duration = round(duration * 60, 2)
                        # skip stream durations less than 5 m inutes
                        if duration < 5.0:
                            continue
                        file.write('\t{} minutes\n'.format(duration))
                    else:
                        file.write('\t{} hours\n'.format(duration))

def main():
    for game in games:
        out = TwitchStatisticsOutput(game_name=game['name'], game_shorthand=game['shorthand'], verbose=True)
        out.run()

if __name__ == '__main__':
    main()
<commit_msg>Add mid directory parameter<commit_after>import datetime
import os
from neopysqlite.neopysqlite import Pysqlite, PysqliteCouldNotRetrieveData
from tqdm import tqdm

games = [
    {
        'name': 'Elite: Dangerous',
        'shorthand': 'ED'
     },
    {
        'name': 'Planet Coaster',
        'shorthand': 'PC'
    }
]

def get_stream_durations(stream_times):
    # pop one field out to avoid odd numbered lists
    time_deltas = []
    for i, field in enumerate(stream_times):
        # if i == 0 or i % 2 == 0:
        # current field
        # timestamp format: '2016-01-25 10:46:18'
        date_part = field.split(' ')[0].split('-')
        time_part = field.split(' ')[1].split(':')
        year, month, day = int(date_part[0]), int(date_part[1]), int(date_part[2])
        hour, minute, second = int(time_part[0]), int(time_part[1]), int(time_part[2])
        time1 = datetime.datetime(year=year, month=month, day=day, hour=hour, minute=minute, second=second)
        # caclulate a delta between the current and the next field
        if i+1 == len(stream_times):
            break
        next_field = stream_times[i+1]
        date_part = next_field.split(' ')[0].split('-')
        time_part = next_field.split(' ')[1].split(':')
        year, month, day = int(date_part[0]), int(date_part[1]), int(date_part[2])
        hour, minute, second = int(time_part[0]), int(time_part[1]), int(time_part[2])
        time2 = datetime.datetime(year=year, month=month, day=day, hour=hour, minute=minute, second=second)
        time_deltas.append([time1, time2])
    stream_durations = []
    current_stream_duration = 0
    for time_one, time_two in time_deltas:
        delta = time_two - time_one
        # operate on the assumption that different streams are a minimum of 1 hour away from each other
        if delta.seconds / (60 * 60) > 1:
            # print('DIFFERENT STREAM FOUND')
            stream_durations.append(current_stream_duration)
            current_stream_duration = 0
        else:
            # print('{} {}\n{} {}\n\t{}\t{}s'.format(i, time_one, i, time_two, delta, delta.seconds))
            current_stream_duration += delta.seconds
    else:
        # if the data ends then the last recorded stream will not be recorded, so add the deltas list anyway
        stream_durations.append(current_stream_duration)
    # change it from seconds to hours
    stream_durations = [round(stream / (60 * 60), 2) for stream in stream_durations]
    return stream_durations

class TwitchStatisticsOutput:
    # bounds for the tiers of streamers
    tier_one_bounds = {'upper': 999999, 'lower': 100}
    tier_two_bounds = {'upper': 99, 'lower': 50}
    tier_three_bounds = {'upper': 49, 'lower': 15}
    tier_four_bounds = {'upper': 14, 'lower': 0}

    def __init__(self, game_name, game_shorthand, db_mid_directory, verbose=False):
        self.name = game_name
        self.shorthand = game_shorthand
        self.db_file_path = os.path.join(os.getcwd(), db_mid_directory, '{}_stats.db'.format(game_shorthand))
        self.db = Pysqlite(database_name='twitch_stats', database_file=self.db_file_path)
        self.verbose = verbose

    def run(self):
        if self.verbose:
            print('Processing data for game: {}'.format(self.name))
        tables = self.db.get_table_names()
        tables = [table for table in tables if table not in ['test', 'sqlite_sequence']]
        # get the table names which do not start with a number
        valid_named_tables = [table for table in tables if not table[0][0].isdigit()]
        # get the table names which start with a number
        number_start_tables = [table for table in tables if table[0][0].isdigit()]
        # reod the original table names
        valid_named_tables.extend(['_' + table for table in number_start_tables])
        # initialise list for all the data
        all_streamer_data = []
        # list any streamers to ignore
        streamers_to_ignore = ['legenddolby1986']
        for streamer in tqdm(valid_named_tables):
            if streamer in streamers_to_ignore:
                # skip if its on the ignore list
                continue
            # get the db data from the table of the same name as the streamer and put it in the list^TM
            all_streamer_data.append(self.get_streamer_dict(streamer))
        # write the data to the text file
        self.write_text_file(streamer_data=all_streamer_data)

    def return_streamer_tier(self, average_viewers):
        if self.tier_one_bounds['upper'] >= average_viewers >= self.tier_one_bounds['lower']:
            return 1
        if self.tier_two_bounds['upper'] >= average_viewers >= self.tier_two_bounds['lower']:
            return 2
        if self.tier_three_bounds['upper'] >= average_viewers >= self.tier_three_bounds['lower']:
            return 3
        if self.tier_four_bounds['upper'] >= average_viewers >= self.tier_four_bounds['lower']:
            return 4
        return 0

    def get_streamer_dict(self, streamer):
        streamer_dict = dict()
        streamer_dict['name'] = streamer
        # catch an exception where the table cannot be found and return an empty dictionary instead
        try:
            data = self.db.get_all_rows(table=streamer)
        except PysqliteCouldNotRetrieveData:
            streamer_dict['partnership'] = False
            streamer_dict['tier'] = 4
            streamer_dict['viewers'] = []
            streamer_dict['viewers_max'] = 0
            streamer_dict['viewers_average'] = 0.0
            streamer_dict['followers'] = []
            streamer_dict['followers_max'] = 0
            streamer_dict['times'] = []
            streamer_dict['durations'] = []
            streamer_dict['durations_max'] = 0
            streamer_dict['durations_average'] = 0.0
            streamer_dict['durations_total'] = 0.0
            streamer_dict['stream_count'] = 0
            return streamer_dict
        streamer_dict['partnership'] = data[-1][3] == 1
        viewers = [field[1] for field in data]
        streamer_dict['viewers'] = [field[1] for field in data]
        streamer_dict['viewers_max'] = max(viewers)
        streamer_dict['viewers_average'] = sum(viewers) // len(viewers)
        streamer_dict['tier'] = self.return_streamer_tier(streamer_dict['viewers_average'])
        followers = [field[2] for field in data]
        streamer_dict['followers'] = followers
        streamer_dict['followers_max'] = followers[-1]
        streamer_dict['times'] = [field[4] for field in data]  # times
        streamer_dict['durations'] = get_stream_durations(streamer_dict['times'])
        streamer_dict['durations_max'] = max(streamer_dict['durations'])
        streamer_dict['durations_average'] = round(sum(streamer_dict['durations']), 2)
        streamer_dict['durations_total'] = round(sum(streamer_dict['durations']), 2)
        streamer_dict['stream_count'] = len(streamer_dict['durations'])
        return streamer_dict

    def write_text_file(self, streamer_data):
        durations = [streamer['durations'] for streamer in streamer_data]
        # get the longest consecutive stream
        non_empty_durations = [duration for duration in durations if not duration == []]
        longest_stream = max([max(duration_set) for duration_set in non_empty_durations])
        # calculate total time streamed over all streamers
        total_duration_sums = sum([sum(duration_set) for duration_set in non_empty_durations])
        total_duration = round(total_duration_sums, 2)
        total_streams = 0
        # calculate the total number of discrete streams
        for streamer in streamer_data:
            total_streams += streamer['stream_count']
        text_file_path = os.path.join(os.getcwd(), 'data', '{}_Twitch_Stats.txt'.format(self.shorthand))
        with open(text_file_path, mode='w', encoding='utf-8') as file:
            file.write('{} Twitch Streamer Statistics\n'.format(self.name))
            file.write('Data recorded 24/7 via twitch\'s public API every ~20 seconds\n')
            file.write('Script written by Simon Agius Muscat / CMDR Purrcat\n')
            file.write('More information can be found at: https://github.com/purrcat259/twitch-statistics\n')
            file.write('Total streamers recorded: {}\n'.format(len(streamer_data)))
            file.write('Total streams recorded: {}\n'.format(total_streams))
            file.write('Total time streamed: {} hours\n'.format(total_duration))
            file.write('Longest single stream: {} hours\n'.format(round(longest_stream, 2)))
            file.write('Tier One Bounds: {} >= Average Viewers >= {}\n'.format(self.tier_one_bounds['upper'], self.tier_one_bounds['lower']))
            file.write('Tier One Streamers: {}\n'.format(len([s for s in streamer_data if self.return_streamer_tier(s['viewers_average']) == 1])))
            file.write('Tier Two Bounds: {} >= Average Viewers >= {}\n'.format(self.tier_two_bounds['upper'], self.tier_two_bounds['lower']))
            file.write('Tier Two Streamers: {}\n'.format(len([s for s in streamer_data if self.return_streamer_tier(s['viewers_average']) == 2])))
            file.write('Tier Three Bounds: {} >= Average Viewers >= {}\n'.format(self.tier_three_bounds['upper'], self.tier_three_bounds['lower']))
            file.write('Tier Three Streamers: {}\n'.format(len([s for s in streamer_data if self.return_streamer_tier(s['viewers_average']) == 3])))
            file.write('Tier Four Bounds: {} >= Average Viewers >= {}\n'.format(self.tier_four_bounds['upper'], self.tier_four_bounds['lower']))
            file.write('Tier Four Streamers: {}\n'.format(len([s for s in streamer_data if self.return_streamer_tier(s['viewers_average']) == 4])))
            for streamer in streamer_data:
                # skip streamers with total durations less than 10 minutes
                if streamer['durations_total'] < 0.2:
                    continue
                file.write('\nStreamer: {} (T{})\n'.format(streamer['name'], streamer['tier']))
                file.write('Partnered: {} \n'.format(streamer['partnership']))
                file.write('Average Viewers: {}\n'.format(streamer['viewers_average']))
                file.write('Peak Viewers: {}\n'.format(streamer['viewers_max']))
                file.write('Followers: {}\n'.format(streamer['followers_max']))
                file.write('Stream count: {}\n'.format(streamer['stream_count']))
                file.write('Average Stream duration: {} hours\n'.format(streamer['durations_average']))
                file.write('Longest Stream duration: {} hours\n'.format(streamer['durations_max']))
                file.write('Total time streamed: {} hours\n'.format(streamer['durations_total']))
                time_percentage = round((streamer['durations_total'] / total_duration) * 100, 3)
                file.write('Percentage streamed of total duration: {}%\n'.format(time_percentage))
                file.write('Stream durations:\n')
                for duration in streamer['durations']:
                    if duration < 1.0:
                        duration = round(duration * 60, 2)
                        # skip stream durations less than 5 m inutes
                        if duration < 5.0:
                            continue
                        file.write('\t{} minutes\n'.format(duration))
                    else:
                        file.write('\t{} hours\n'.format(duration))

def main():
    for game in games:
        out = TwitchStatisticsOutput(game_name=game['name'], game_shorthand=game['shorthand'], db_mid_directory='data', verbose=True)
        out.run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""Identifier Model Module"""

from ..database import db
from sqlalchemy import UniqueConstraint
from sqlalchemy.dialects.postgresql import ENUM

identifier_use = ENUM('usual', 'official', 'temp', 'secondary',
                      name='id_use', create_type=False)

class Identifier(db.Model):
    """Identifier ORM, for FHIR Identifier resources"""
    __tablename__ = 'identifiers'
    id = db.Column(db.Integer, primary_key=True)
    use = db.Column('id_use', identifier_use)
    system = db.Column(db.String(255), nullable=False)
    _value = db.Column('value', db.Text, nullable=False)
    assigner = db.Column(db.String(255))

    __table_args__ = (UniqueConstraint('system', 'value',
        name='_identifier_system_value'),)

    @property
    def value(self):
        return self._value

    @value.setter
    def value(self, value):
        # Force to text
        self._value = unicode(value)
        # Don't allow empty string
        if not len(self._value):
            raise TypeError("<empty string>")

    @classmethod
    def from_fhir(cls, data):
        instance = cls()
        # if we aren't given a 'use', call it 'usual'
        instance.use = data['use'] if 'use' in data else 'usual'
        instance.system = data['system']
        instance.value = data['value']
        # FHIR changed - assigner needs to reference object in system
        # not storing at this time
        #if 'assigner' in data:
        #    instance.assigner = data['assigner']
        return instance

    def __str__(self):
        return 'Identifier {0.use} {0.system} {0.value}'.format(self)

    def __eq__(self, other):
        ## Only compare (system, value), as per unique constraint
        return self.system == other.system and self.value == other.value

    def as_fhir(self):
        d = {}
        for k in ('use', 'system', 'value', 'assigner'):
            if getattr(self, k, None):
                d[k] = getattr(self, k)
        return d

    def add_if_not_found(self, commit_immediately=False):
        """Add self to database, or return existing

        Queries for similar, matching on **system** and **value** alone.
        Note the database unique constraint to match.

        @return: the new or matched Identifier

        """
        existing = Identifier.query.filter_by(system=self.system,
                                              _value=self.value).first()
        if not existing:
            db.session.add(self)
            if commit_immediately:
                db.session.commit()
        else:
            self.id = existing.id
        self = db.session.merge(self)
        return self

class UserIdentifier(db.Model):
    """ORM class for user_identifiers data

    Holds links to any additional identifiers a user may have,
    such as study participation.

    """
    __tablename__ = 'user_identifiers'
    id = db.Column(db.Integer, primary_key=True)
    user_id = db.Column(db.ForeignKey('users.id'), nullable=False)
    identifier_id = db.Column(
        db.ForeignKey('identifiers.id'), nullable=False)
    identifier = db.relationship(Identifier, cascade="save-update")
    user = db.relationship('User', cascade="save-update")

    __table_args__ = (UniqueConstraint(
        'user_id', 'identifier_id', name='_user_identifier'),)

    def __str__(self):
        return ("user_identifier {} for {}".format(
            self.identifier, self.user))
<commit_msg>Minor import order fix.<commit_after>"""Identifier Model Module"""
from sqlalchemy import UniqueConstraint
from sqlalchemy.dialects.postgresql import ENUM

from ..database import db

identifier_use = ENUM('usual', 'official', 'temp', 'secondary',
                      name='id_use', create_type=False)

class Identifier(db.Model):
    """Identifier ORM, for FHIR Identifier resources"""
    __tablename__ = 'identifiers'
    id = db.Column(db.Integer, primary_key=True)
    use = db.Column('id_use', identifier_use)
    system = db.Column(db.String(255), nullable=False)
    _value = db.Column('value', db.Text, nullable=False)
    assigner = db.Column(db.String(255))

    __table_args__ = (UniqueConstraint('system', 'value',
        name='_identifier_system_value'),)

    @property
    def value(self):
        return self._value

    @value.setter
    def value(self, value):
        # Force to text
        self._value = unicode(value)
        # Don't allow empty string
        if not len(self._value):
            raise TypeError("<empty string>")

    @classmethod
    def from_fhir(cls, data):
        instance = cls()
        # if we aren't given a 'use', call it 'usual'
        instance.use = data['use'] if 'use' in data else 'usual'
        instance.system = data['system']
        instance.value = data['value']
        # FHIR changed - assigner needs to reference object in system
        # not storing at this time
        #if 'assigner' in data:
        #    instance.assigner = data['assigner']
        return instance

    def __str__(self):
        return 'Identifier {0.use} {0.system} {0.value}'.format(self)

    def __eq__(self, other):
        ## Only compare (system, value), as per unique constraint
        return self.system == other.system and self.value == other.value

    def as_fhir(self):
        d = {}
        for k in ('use', 'system', 'value', 'assigner'):
            if getattr(self, k, None):
                d[k] = getattr(self, k)
        return d

    def add_if_not_found(self, commit_immediately=False):
        """Add self to database, or return existing

        Queries for similar, matching on **system** and **value** alone.
        Note the database unique constraint to match.

        @return: the new or matched Identifier

        """
        existing = Identifier.query.filter_by(system=self.system,
                                              _value=self.value).first()
        if not existing:
            db.session.add(self)
            if commit_immediately:
                db.session.commit()
        else:
            self.id = existing.id
        self = db.session.merge(self)
        return self

class UserIdentifier(db.Model):
    """ORM class for user_identifiers data

    Holds links to any additional identifiers a user may have,
    such as study participation.

    """
    __tablename__ = 'user_identifiers'
    id = db.Column(db.Integer, primary_key=True)
    user_id = db.Column(db.ForeignKey('users.id'), nullable=False)
    identifier_id = db.Column(
        db.ForeignKey('identifiers.id'), nullable=False)
    identifier = db.relationship(Identifier, cascade="save-update")
    user = db.relationship('User', cascade="save-update")

    __table_args__ = (UniqueConstraint(
        'user_id', 'identifier_id', name='_user_identifier'),)

    def __str__(self):
        return ("user_identifier {} for {}".format(
            self.identifier, self.user))
<|endoftext|>


<commit_before>from lxml.html.soupparser import fromstring
import logging
from django.core.management import setup_environ
import bookworm.settings
setup_environ(bookworm.settings)

import bookworm.search.constants as constants

logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger('search.epubindexer')

def get_searchable_content(content):
    '''Returns the content of a chapter as a searchable field'''
    html = fromstring(content)
    ns = get_namespace(content)
    headers = ('h1', 'h2', 'h3', 'h4', 'h5', 'h6')
    if ns is not None:
        temp_para = [ p.xpath('.//text()') for p in html.iter(tag='{%s}p' % ns)]
        for h in headers:
            temp_para += [ p.xpath('.//text()') for p in html.iter(tag='{%s}%s' % (ns, h))]
    else:
        temp_para = [ p.xpath('.//text()') for p in html.iter(tag='p')]
        for h in headers:
            temp_para += [ p.xpath('.//text()') for p in html.iter(tag='%s' % h)]
    paragraphs = []
    for p in temp_para:
        if p is not None:
            paragraphs.append(' '.join([i.strip().replace('\n',' ') for i in p]))

    return '\n'.join(paragraphs)

def get_namespace(content):
    '''Determines whether this content has a namespace or not'''
    html = fromstring(content)
    if html.find('{%s}p' % constants.XHTML_NS) is not None:
        return constants.XHTML_NS
    return None

<commit_msg>Work around bug in lxml soupparser.py<commit_after>from lxml.html.soupparser import fromstring
import logging
from django.core.management import setup_environ
import bookworm.settings
setup_environ(bookworm.settings)

import bookworm.search.constants as constants

logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger('search.epubindexer')

def get_searchable_content(content):
    '''Returns the content of a chapter as a searchable field'''
    try:
        html = fromstring(str(content))
    except TypeError: # soupparser.py bug
        return None
    ns = get_namespace(content)
    headers = ('h1', 'h2', 'h3', 'h4', 'h5', 'h6')
    if ns is not None:
        temp_para = [ p.xpath('.//text()') for p in html.iter(tag='{%s}p' % ns)]
        for h in headers:
            temp_para += [ p.xpath('.//text()') for p in html.iter(tag='{%s}%s' % (ns, h))]
    else:
        temp_para = [ p.xpath('.//text()') for p in html.iter(tag='p')]
        for h in headers:
            temp_para += [ p.xpath('.//text()') for p in html.iter(tag='%s' % h)]
    paragraphs = []
    for p in temp_para:
        if p is not None:
            paragraphs.append(' '.join([i.strip().replace('\n',' ') for i in p]))

    return '\n'.join(paragraphs)

def get_namespace(content):
    '''Determines whether this content has a namespace or not'''
    html = fromstring(content)
    if html.find('{%s}p' % constants.XHTML_NS) is not None:
        return constants.XHTML_NS
    return None

<|endoftext|>


<commit_before>#!/usr/bin/python3
#
# DESCRIPTION
#
#       getaudio is used to sequentially prompt the user for dictations of displayed sentences.
#
# DEPENDENCIES
#
#       python3-pyaudio, python3
#
# USAGE
#
#       python3 getaudio.py sentence-file /output/folder recording-repetitions model-name
#
# LIBRARY IMPORTS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

import sys, os, _thread, pyaudio, wave, contextlib

# VARIABLE DEFINITIONS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

pronunciation_dictionary="cmudict-en-us.dict"

chunk = 1024
FORMAT = pyaudio.paInt16
CHANNELS = 1
RATE = 16000

sentence_file = sys.argv[1]

output_folder = sys.argv[2].rstrip(os.sep)

reps = int(sys.argv[3])

model_name = sys.argv[4]

# FUNCTION DEFINITIONS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

# ignore sdterr messages: as from pyaudio
@contextlib.contextmanager
def ignore_stderr():
    devnull = os.open(os.devnull, os.O_WRONLY)
    old_stderr = os.dup(2)
    sys.stderr.flush()
    os.dup2(devnull, 2)
    os.close(devnull)
    try:
        yield
    finally:
        os.dup2(old_stderr, 2)
        os.close(old_stderr)

def record_until_keypress(audio_filepath):

    # detect keypress [enter]
    def input_thread(L):
        input()
        L.append(None)

    # initialize audio stream - and keep it quiet
    with ignore_stderr():
        p = pyaudio.PyAudio()
        stream = p.open(format = FORMAT,
                channels = CHANNELS,
                rate = RATE,
                input = True,
                frames_per_buffer = chunk)

    # create interrupt thread
    L = []
    _thread.start_new_thread(input_thread, (L,))

    # record data during loop
    frames = []
    while True:
        data = stream.read(chunk)
        frames.append(data)
        if L:
            stream.stop_stream()
            break

    # exit cleanly after break
    stream.close()
    p.terminate()

    # write data to WAVE file
    data = b''.join(frames)
    wf = wave.open(audio_filepath, 'wb')
    wf.setnchannels(CHANNELS)
    wf.setsampwidth(p.get_sample_size(FORMAT))
    wf.setframerate(RATE)
    wf.writeframes(data)
    wf.close()

# LOGIC ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

if not os.path.exists(output_folder):
    os.makedirs(output_folder)

# create list of sentences for prompt
sentence_list = []
with open(sentence_file) as f:
    for line in f:
        sentence_list.append(line)

num_recs = len(sentence_list)*reps

# collect audio files
try:

    input("Press [enter], read text, & press [enter].")

    j=0

    for sentence in sentence_list*reps:
        #recording number
        j+=1

        # record audio with visual
        print("Recording no. %04d of %04d: \n\n\t%s" % (j, num_recs, sentence), end='\r')

        # recording file should look like this (e.g.): ./bespoke_training_data/audio/arctic_0001.wav
        record_until_keypress(str(output_folder + os.sep + model_name + "_%04d.wav" % j))

except KeyboardInterrupt:
    pass

<commit_msg>Delete getaudio.py<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python

import os
# import ssl
import json
import urllib

# ENV VARIABLE
home = os.getenv('HOME')

# GET OS VARIABLE
from sys import platform

# CONDITIONALLY SET OS SPECIFIC ENV
if platform == 'linux' or platform == 'linux2':

	print 'getspace is NOT YET running as getspace_linux'
	quit() # exit until draft is no draft any longer

	import getspace_linux as getspace
	path = home + '/GetSpace'

elif platform == 'darwin':

	print 'getspace is running as getspace_osx'

	import getspace_osx as getspace
	path = home + '/Pictures/GetSpace'

else:

	print 'exiting (no compatible os found)'
	quit()

# DEF GET COLOR BASED ON https://gist.github.com/zollinger/1722663
def getcolor(img):
	img = img.resize((150, 150))
	result = img.convert('P', palette=Image.ADAPTIVE, colors=1)
	result.putalpha(255)
	colors = result.getcolors(22500)
	color = colors[0][1]
	return color

# OPEN URL ON SRV - ADD YOUR API KEY HERE
url = 'https://api.nasa.gov/planetary/apod?api_key=DEMO_KEY'

try:
    response = urllib.urlopen(url)

    # LOAD JSON DATA FROM OBJECT
    jsonstuff = json.load(response)

except IOError, e:

	print 'Using CURL fallback.'

	import subprocess

	process = subprocess.Popen(['curl', url], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
	stdout, stderr = process.communicate()
	response = stdout

	# LOAD JSON DATA FROM STRING
	jsonstuff = json.loads(response)

# CHECK IF JSON DATA EXISTS
if 'jsonstuff' not in locals():
	print 'Failed! APOD info could not be downloaded.'
	quit()

else:
	print 'APOD info downloaded.'
	media_type = (jsonstuff['media_type'])

# CHECK IF AN IMAGE IS AVAILABLE
if media_type == 'image':

	# IF 'PICTURES/GETSPACE' DOES NOT EXISTS, CREATE IT
	if not os.path.exists(path):
		os.makedirs(path)

	# EXTRACT VARIABLES FROM JSON DATA
	date = (jsonstuff['date'])
	title = (jsonstuff['title'])
	explanation = (jsonstuff['explanation'])
	hdurl = (jsonstuff['hdurl'])
	# (maybe there's not always a HQ version? if so, add fallback for 'url')

	# DEFINE FILE PATH VARIABLES
	imgname = os.path.basename(hdurl)
	# imgname = 'testimage2.jpg' # debugging
	filename, file_extension = os.path.splitext(imgname)
	savein = path + '/' + imgname
	saveout = path + '/' + filename + '_info' + file_extension

	# CHECK IF IMAGE IS NEW
	lastspace = path + '/lastspace.txt'
	if os.path.exists(lastspace):
		lastimage = open(lastspace, 'r')
		print 'lastimg is: ' + lastimage.read()
		print 'imgname is: ' + imgname
		if lastimage.read() == imgname:
			file = open(lastspace, 'w')
			file.write(imgname)
			file.close()
			print 'exiting (there\'s no new image available)'
			quit()
	else:
		file = open(lastspace, 'w')
		file.write(imgname)
		file.close()

	# DOWNLOAD THE HQ IMAGE FROM hdurl TO savein
	urllib.urlretrieve(hdurl, savein)
	print 'APOD image downloaded.'

	# CHECK IF PIL/PILLOW IS INSTALLED
	try:
		import PIL
	except ImportError, e:
		getspace.notify("Please read the source code.", "This thing requires a spaceship.", "So, no space info on the desktop for you.", "error")
		pass
	else:

		# GET THE SCREEN SIZE
		(vw, vh) = getspace.getscreensize()

		# CALCULATE RATIO OF THE SCREEN
		ratioscreen = float(vw) / vh
		print("Screenratio is %s (%sx%spx)" % (ratioscreen, vw, vh))

		# MANIPULATE THE IMAGE!
		from PIL import Image
		from PIL import ImageOps
		from PIL import ImageDraw
		# from PIL import ImageFilter

		Image.MAX_IMAGE_PIXELS = 1000000000

		from math import floor

		# OPEN IMG + INVERSION IMAGE
		img = Image.open(savein)
		inv = ImageOps.invert(img)

		# GET IMG + INVERSION COLOR
		imgcolor = getcolor(img)
		invcolor = getcolor(inv)

		# ADDING TRANSPARENCY TO IMGCOLOR
		lst = list(imgcolor)
		lst[3] = 180
		imgcolor = tuple(lst)

		# SET INVCOLOR TO EITHER BLACK OR WHITE
		lst = list(invcolor)
		if lst[0] + lst[1] + lst[2] > 382:
			lst[0] = 255
			lst[1] = 255
			lst[2] = 255
		else:
			lst[0] = 0
			lst[1] = 0
			lst[2] = 0
		invcolor = tuple(lst)

		# OPEN DOWNLOADED IMAGE
		img = img.convert('RGBA')

		# GET IMAGE SIZE
		(width, height) = img.size

		# CALCULATE RATIO OF THE IMAGE
		ratioimage = float(width) / height
		print("Imageratio is %s (%sx%spx)" % (ratioimage, width, height))

		# CUT THE IMAGE TO THE SCREEN RATIO
		if ratioscreen > ratioimage:

			newheight = floor(width / ratioscreen)
			img = ImageOps.fit(img, (width,newheight), centering = (0.5,0.5))
			(width, height) = img.size

			print 'Image cut.'

		elif ratioscreen < ratioimage:

			newwidth = floor(height * ratioscreen)

			img = ImageOps.fit(img, (newwidth,height), centering = (0.5,0.5))
			(width, height) = img.size

			print 'Image cut.'

		# UPSCALE IMAGE TO DISPLAY SIZE
		if width < vw:

			img = img.resize((int(vw), int(vh)), Image.BICUBIC)
			width = vw
			height = vh

			print 'Image upscaled.'

		# BEGIN PILLOW'S DRAWING
		overlay = Image.new('RGBA', img.size)
		# draw = ImageDraw.Draw(img)
		draw = ImageDraw.Draw(overlay)

		# GET FONT VARIABLES
		(fsizehead, fsizetext, wrapped, headfont, textfont) = getspace.getfontvars(height, explanation)

		# CALCULATE THE HEADING SIZE
		whead, hhead = draw.textsize(title,font=headfont)

		# SET POSITIONG VARS
		golden = hhead * 1.618
		silver = golden - hhead

		# CALCULATE THE TEXT SIZE AND DEDUCE POSITIONING VARS FOR NEXT 2 STEPS
		wtext, htext = draw.textsize(wrapped,font=textfont)
		x = (2 * golden - silver)
		y = (height - htext - golden - silver)
		x2 = (x + wtext + silver)
		y2 = (y + htext + silver)

		# DRAW A RECTANGULAR BACKGROUND SHAPE ONTO THE IMAGE
		# draw.rectangle([x - silver, y - silver, x2, y2 ],fill=(255,255,255,188))
		draw.rectangle([x - silver, y - silver, x2, y2 ],fill=imgcolor)

		# DRAW THE 'EXPLANATION'-TEXT ONTO THE BACKGROUND
		draw.text((x + 1, y + 1),wrapped,invcolor,font=textfont,align="left")
		# draw.text((x, y),wrapped,(0,0,0),font=textfont,align="left")

		# DRAW THE HEADING
		draw.text((x, (y - 2 * silver - hhead)),title,invcolor,font=headfont)
		# draw.text((golden, golden),title,(255,255,255),font=headfont)

		# CHANGE LOGINIMAGE
		# (add stuff for changing loginimage)

		# SET THE OVERLAY ON TOP OF THE IMG
		img = Image.alpha_composite(img, overlay)

		print 'Text written.'

		# WRITE THE IMAGE TO saveout
		img.save(saveout)

		print 'Image saved.'

		# SET savein TO saveout SO THE MODIFIED IMAGE IS SET AS BG IN THE NEXT STEP
		savein = saveout

	# SETTING THE DESKTOP BG
	getspace.dosetbackground(savein)

	print 'Background set.'

	# SETTING

	getspace.notify("Boldly go where only aliens have gone before", title.replace("'", ""), "Background image has been set.", "success")

else:

	getspace.notify("Space, the final frontier!", "Apparently, there is no new image today.", "Try investing in space travel or something.", "error")<commit_msg>added test for video (know not resulting in error message, need to add test for newness)<commit_after>#!/usr/bin/env python

import os
# import ssl
import json
import urllib

# ENV VARIABLE
home = os.getenv('HOME')

# GET OS VARIABLE
from sys import platform

# CONDITIONALLY SET OS SPECIFIC ENV
if platform == 'linux' or platform == 'linux2':

	print 'getspace is NOT YET running as getspace_linux'
	quit() # exit until draft is no draft any longer

	import getspace_linux as getspace
	path = home + '/GetSpace'

elif platform == 'darwin':

	print 'getspace is running as getspace_osx'

	import getspace_osx as getspace
	path = home + '/Pictures/GetSpace'

else:

	print 'exiting (no compatible os found)'
	quit()

# DEF GET COLOR BASED ON https://gist.github.com/zollinger/1722663
def getcolor(img):
	img = img.resize((150, 150))
	result = img.convert('P', palette=Image.ADAPTIVE, colors=1)
	result.putalpha(255)
	colors = result.getcolors(22500)
	color = colors[0][1]
	return color

# OPEN URL ON SRV - ADD YOUR API KEY HERE
url = 'https://api.nasa.gov/planetary/apod?api_key=DEMO_KEY'

try:
    response = urllib.urlopen(url)

    # LOAD JSON DATA FROM OBJECT
    jsonstuff = json.load(response)

except IOError, e:

	print 'Using CURL fallback.'

	import subprocess

	process = subprocess.Popen(['curl', url], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
	stdout, stderr = process.communicate()
	response = stdout

	# LOAD JSON DATA FROM STRING
	jsonstuff = json.loads(response)

# CHECK IF JSON DATA EXISTS
if 'jsonstuff' not in locals():
	print 'Failed! APOD info could not be downloaded.'
	quit()

else:
	print 'APOD info downloaded.'
	media_type = (jsonstuff['media_type'])

# CHECK IF AN VIDEO OR IMNAGE IS AVAILABLE
if media_type == 'video':
	getspace.notify("Space traveling now..", "There is an APOD video today.", "Watch it at apod.nasa.gov!", "success")

elif media_type == 'image':

	# IF 'PICTURES/GETSPACE' DOES NOT EXISTS, CREATE IT
	if not os.path.exists(path):
		os.makedirs(path)

	# EXTRACT VARIABLES FROM JSON DATA
	date = (jsonstuff['date'])
	title = (jsonstuff['title'])
	explanation = (jsonstuff['explanation'])
	hdurl = (jsonstuff['hdurl'])
	# (maybe there's not always a HQ version? if so, add fallback for 'url')

	# DEFINE FILE PATH VARIABLES
	imgname = os.path.basename(hdurl)
	# imgname = 'testimage2.jpg' # debugging
	filename, file_extension = os.path.splitext(imgname)
	savein = path + '/' + imgname
	saveout = path + '/' + filename + '_info' + file_extension

	# CHECK IF IMAGE IS NEW
	lastspace = path + '/lastspace.txt'
	if os.path.exists(lastspace):
		lastimage = open(lastspace, 'r')
		print 'lastimg is: ' + lastimage.read()
		print 'imgname is: ' + imgname
		if lastimage.read() == imgname:
			file = open(lastspace, 'w')
			file.write(imgname)
			file.close()
			print 'exiting (there\'s no new image available)'
			quit()
	else:
		file = open(lastspace, 'w')
		file.write(imgname)
		file.close()

	# DOWNLOAD THE HQ IMAGE FROM hdurl TO savein
	urllib.urlretrieve(hdurl, savein)
	print 'APOD image downloaded.'

	# CHECK IF PIL/PILLOW IS INSTALLED
	try:
		import PIL
	except ImportError, e:
		getspace.notify("Please read the source code.", "This thing requires a spaceship.", "So, no space info on the desktop for you.", "error")
		pass
	else:

		# GET THE SCREEN SIZE
		(vw, vh) = getspace.getscreensize()

		# CALCULATE RATIO OF THE SCREEN
		ratioscreen = float(vw) / vh
		print("Screenratio is %s (%sx%spx)" % (ratioscreen, vw, vh))

		# MANIPULATE THE IMAGE!
		from PIL import Image
		from PIL import ImageOps
		from PIL import ImageDraw
		# from PIL import ImageFilter

		Image.MAX_IMAGE_PIXELS = 1000000000

		from math import floor

		# OPEN IMG + INVERSION IMAGE
		img = Image.open(savein)
		inv = ImageOps.invert(img)

		# GET IMG + INVERSION COLOR
		imgcolor = getcolor(img)
		invcolor = getcolor(inv)

		# ADDING TRANSPARENCY TO IMGCOLOR
		lst = list(imgcolor)
		lst[3] = 180
		imgcolor = tuple(lst)

		# SET INVCOLOR TO EITHER BLACK OR WHITE
		lst = list(invcolor)
		if lst[0] + lst[1] + lst[2] > 382:
			lst[0] = 255
			lst[1] = 255
			lst[2] = 255
		else:
			lst[0] = 0
			lst[1] = 0
			lst[2] = 0
		invcolor = tuple(lst)

		# OPEN DOWNLOADED IMAGE
		img = img.convert('RGBA')

		# GET IMAGE SIZE
		(width, height) = img.size

		# CALCULATE RATIO OF THE IMAGE
		ratioimage = float(width) / height
		print("Imageratio is %s (%sx%spx)" % (ratioimage, width, height))

		# CUT THE IMAGE TO THE SCREEN RATIO
		if ratioscreen > ratioimage:

			newheight = floor(width / ratioscreen)
			img = ImageOps.fit(img, (width,newheight), centering = (0.5,0.5))
			(width, height) = img.size

			print 'Image cut.'

		elif ratioscreen < ratioimage:

			newwidth = floor(height * ratioscreen)

			img = ImageOps.fit(img, (newwidth,height), centering = (0.5,0.5))
			(width, height) = img.size

			print 'Image cut.'

		# UPSCALE IMAGE TO DISPLAY SIZE
		if width < vw:

			img = img.resize((int(vw), int(vh)), Image.BICUBIC)
			width = vw
			height = vh

			print 'Image upscaled.'

		# BEGIN PILLOW'S DRAWING
		overlay = Image.new('RGBA', img.size)
		# draw = ImageDraw.Draw(img)
		draw = ImageDraw.Draw(overlay)

		# GET FONT VARIABLES
		(fsizehead, fsizetext, wrapped, headfont, textfont) = getspace.getfontvars(height, explanation)

		# CALCULATE THE HEADING SIZE
		whead, hhead = draw.textsize(title,font=headfont)

		# SET POSITIONG VARS
		golden = hhead * 1.618
		silver = golden - hhead

		# CALCULATE THE TEXT SIZE AND DEDUCE POSITIONING VARS FOR NEXT 2 STEPS
		wtext, htext = draw.textsize(wrapped,font=textfont)
		x = (2 * golden - silver)
		y = (height - htext - golden - silver)
		x2 = (x + wtext + silver)
		y2 = (y + htext + silver)

		# DRAW A RECTANGULAR BACKGROUND SHAPE ONTO THE IMAGE
		# draw.rectangle([x - silver, y - silver, x2, y2 ],fill=(255,255,255,188))
		draw.rectangle([x - silver, y - silver, x2, y2 ],fill=imgcolor)

		# DRAW THE 'EXPLANATION'-TEXT ONTO THE BACKGROUND
		draw.text((x + 1, y + 1),wrapped,invcolor,font=textfont,align="left")
		# draw.text((x, y),wrapped,(0,0,0),font=textfont,align="left")

		# DRAW THE HEADING
		draw.text((x, (y - 2 * silver - hhead)),title,invcolor,font=headfont)
		# draw.text((golden, golden),title,(255,255,255),font=headfont)

		# CHANGE LOGINIMAGE
		# (add stuff for changing loginimage)

		# SET THE OVERLAY ON TOP OF THE IMG
		img = Image.alpha_composite(img, overlay)

		print 'Text written.'

		# WRITE THE IMAGE TO saveout
		img.save(saveout)

		print 'Image saved.'

		# SET savein TO saveout SO THE MODIFIED IMAGE IS SET AS BG IN THE NEXT STEP
		savein = saveout

	# SETTING THE DESKTOP BG
	getspace.dosetbackground(savein)

	print 'Background set.'

	# SETTING

	getspace.notify("Boldly go where only aliens have gone before", title.replace("'", ""), "Background image has been set.", "success")

else:

	getspace.notify("Space, the final frontier!", "Apparently, there is no new image today.", "Try investing in space travel or something.", "error")<|endoftext|>


<commit_before>try:
    # Python 2.x
    from urlparse import urlsplit, urlunsplit
except:
    # Python 3.x
    from urllib import parse as urlsplit
    from urllib import parse as urlunsplit

from django.conf import settings
from django.http import HttpResponsePermanentRedirect

class SSLifyMiddleware(object):
    """Force all requests to use HTTPs. If we get an HTTP request, we'll just
    force a redirect to HTTPs.

    .. note::
        This will only take effect if ``settings.DEBUG`` is False.

    .. note::
        You can also disable this middleware when testing by setting
        ``settings.SSLIFY_DISABLE`` to True.
    """
    def process_request(self, request):
        # If the user has explicitly disabled SSLify, do nothing.
        if getattr(settings, 'SSLIFY_DISABLE', settings.DEBUG):
            return None

        # If we get here, proceed as normal.
        if not request.is_secure():
            url = request.build_absolute_uri(request.get_full_path())
            url_split = urlsplit(url)
            scheme = 'https' if url_split.scheme == 'http' else url_split.scheme
            ssl_port = getattr(settings, 'SSL_PORT', 443)
            url_secure_split = (scheme, "%s:%d" % (url_split.hostname or '', ssl_port)) + url_split[2:]
            secure_url = urlunsplit(url_secure_split)
            return HttpResponsePermanentRedirect(secure_url)
<commit_msg>Trying for python 3 compat again =)<commit_after>try:
    # Python 2.x
    from urlparse import urlsplit, urlunsplit
except:
    # Python 3.x
    from urllib.parse import urlsplit
    from urllib.parse import urlunsplit

from django.conf import settings
from django.http import HttpResponsePermanentRedirect

class SSLifyMiddleware(object):
    """Force all requests to use HTTPs. If we get an HTTP request, we'll just
    force a redirect to HTTPs.

    .. note::
        This will only take effect if ``settings.DEBUG`` is False.

    .. note::
        You can also disable this middleware when testing by setting
        ``settings.SSLIFY_DISABLE`` to True.
    """
    def process_request(self, request):
        # If the user has explicitly disabled SSLify, do nothing.
        if getattr(settings, 'SSLIFY_DISABLE', settings.DEBUG):
            return None

        # If we get here, proceed as normal.
        if not request.is_secure():
            url = request.build_absolute_uri(request.get_full_path())
            url_split = urlsplit(url)
            scheme = 'https' if url_split.scheme == 'http' else url_split.scheme
            ssl_port = getattr(settings, 'SSL_PORT', 443)
            url_secure_split = (scheme, "%s:%d" % (url_split.hostname or '', ssl_port)) + url_split[2:]
            secure_url = urlunsplit(url_secure_split)
            return HttpResponsePermanentRedirect(secure_url)
<|endoftext|>


<commit_before>import os
import time
import datetime
import hashlib
import urllib
import sqlite3
import json
import tweepy as tp
import detector

fileno = 0
file_md5 = []

def reset(dbfile, mode):
    """保存用のフォルダーを生成し、必要な変数を初期化する"""
    try:
        dbfile.execute("drop table result")
    except:
        pass
    try:
        dbfile.execute("drop table {}".format(mode))
    except:
        pass
    dbfile.execute("vacuum")
    dbfile.execute("create table {} (filename, image, username, url, tags, time, facex, facey, facew, faceh)".format(mode))
    dbfile.execute("create table result (mode, time, image_count, tweet_count)")
    dbfile.commit()

def on_status(status, dbfile, mode):
    """UserStreamから飛んできたStatusを処理する"""
    global fileno
    global file_md5
    # ツイートについてる画像枚数
    image_count = 0
    # Tweetに画像がついているか
    is_media = False
    # TweetがRTかどうか
    if hasattr(status, "retweeted_status"):
        status = status.retweeted_status
    # Tweetが引用ツイートかどうか
    if hasattr(status, "quoted_status"):
        status = status.quoted_status
    # 複数枚の画像ツイートのとき
    if hasattr(status, "extended_entities"):
        if 'media' in status.extended_entities:
            status_media = status.extended_entities
            is_media = True
    # 一枚の画像ツイートのとき
    elif hasattr(status, "entities"):
        if 'media' in status.entities:
            status_media = status.entities
            is_media = True

    # 画像がついていたとき
    if is_media:
        for image in status_media['media']:
            if image['type'] != 'photo':
                break
            # URL, ファイル名
            media_url = image['media_url']
            filename = str(fileno).zfill(5)
            # ダウンロード
            try:
                temp_file = urllib.request.urlopen(media_url+":small").read()
            except:
                continue
            # md5の取得
            current_md5 = hashlib.md5(temp_file).hexdigest()
            # すでに取得済みの画像は飛ばす
            if current_md5 in file_md5:
                continue
            # 画像判定呼出
            current_hash = None
            current_hash, facex, facey, facew, faceh = detector.face_2d(temp_file, status.user.screen_name, filename)
            if current_hash is not None:
                # 取得済みとしてハッシュ値を保存
                file_md5.append(current_md5)
                # ハッシュタグがあれば保存する
                tags = []
                if hasattr(status, "entities"):
                    if "hashtags" in status.entities:
                        for hashtag in status.entities['hashtags']:
                            tags.append(hashtag['text'])
                # データベースに保存
                SQL = "insert into {} values (?,?,?,?,?,?,?,?,?,?)".format(mode)
                url = "https://twitter.com/" + status.user.screen_name + "/status/" + status.id_str
                value = (filename, media_url, status.user.screen_name, url, str(tags).replace("'",""),
                        str(datetime.datetime.now()), str(facex), str(facey), str(facew), str(faceh))
                dbfile.execute(SQL, value)
                dbfile.commit()
                fileno += 1
            temp_file = None
            image_count += 1
    return image_count

def getTweets(api, mode, count, query):
    start = time.time()

    dbpath = os.path.abspath(__file__).replace(os.path.basename(__file__),"/DB/user/"+ api.me().id_str + ".db")
    dbfile = sqlite3.connect(dbpath)

    # DBのリセット等
    reset(dbfile, mode)
    tweet_count = image_count = 0
    tweet_id = []
    # 取得モード
    if mode == "timeline":
        for status in tp.Cursor(api.home_timeline).items(count):
            if status.id in tweet_id:
                continue
            else:
                tweet_id.append(status.id)
            image_count += on_status(status, dbfile, mode)
            tweet_count += 1
    elif mode == "fav":
        for status in tp.Cursor(api.favorites).items(count):
            if status.id in tweet_id:
                continue
            else:
                tweet_id.append(status.id)
            image_count += on_status(status, dbfile, mode)
            tweet_count += 1
    elif mode == "user":
        for status in tp.Cursor(api.user_timeline, screen_name=query).items(count):
            if status.id in tweet_id:
                continue
            else:
                tweet_id.append(status.id)
            image_count += on_status(status, dbfile, mode)
            tweet_count += 1
    elif mode == "list":
        listurl = query.replace("https://","")
        owner = listurl.split("/")[1]
        slug = listurl.split("/")[3]
        for status in tp.Cursor(api.list_timeline, owner_screen_name=owner, slug=slug).items(count):
            if status.id in tweet_id:
                continue
            else:
                tweet_id.append(status.id)
            image_count += on_status(status, dbfile, mode)
            tweet_count += 1
    elif mode == "tag":
        for status in tp.Cursor(api.search, q="#" + query).items(count):
            if status.id in tweet_id:
                continue
            else:
                tweet_id.append(status.id)
            image_count += on_status(status, dbfile, mode)
            tweet_count += 1
    elif mode == "keyword":
        for status in tp.Cursor(api.search, q=query).items(count):
            if status.id in tweet_id:
                continue
            else:
                tweet_id.append(status.id)
            image_count += on_status(status, dbfile, mode)
            tweet_count += 1

    elapsed_time = time.time() - start
    # 実行時間,全ツイート数,全画像枚数をデータベースに
    SQL = "insert into result values (?,?,?,?)"
    value = (mode, str(elapsed_time), str(image_count), str(tweet_count))
    dbfile.execute(SQL, value)
    dbfile.commit()

<commit_msg>グローバル変数の削除<commit_after>import os
import time
import datetime
import hashlib
import urllib
import sqlite3
import json
import tweepy as tp
import detector

def reset(dbfile, mode):
    """保存用のフォルダーを生成し、必要な変数を初期化する"""
    try:
        dbfile.execute("drop table result")
    except:
        pass
    try:
        dbfile.execute("drop table {}".format(mode))
    except:
        pass
    dbfile.execute("vacuum")
    dbfile.execute("create table {} (filename, image, username, url, tags, time, facex, facey, facew, faceh)".format(mode))
    dbfile.execute("create table result (mode, time, image_count, tweet_count)")
    dbfile.commit()

def on_status(status, dbfile, mode, id, file_md5):
    """UserStreamから飛んできたStatusを処理する"""
    return_md5 = []
    # ツイートについてる画像枚数
    image_count = 0
    # Tweetに画像がついているか
    is_media = False
    # TweetがRTかどうか
    if hasattr(status, "retweeted_status"):
        status = status.retweeted_status
    # Tweetが引用ツイートかどうか
    if hasattr(status, "quoted_status"):
        status = status.quoted_status
    # 複数枚の画像ツイートのとき
    if hasattr(status, "extended_entities"):
        if 'media' in status.extended_entities:
            status_media = status.extended_entities
            is_media = True
    # 一枚の画像ツイートのとき
    elif hasattr(status, "entities"):
        if 'media' in status.entities:
            status_media = status.entities
            is_media = True

    # 画像がついていたとき
    if is_media:
        for image in status_media['media']:
            if image['type'] != 'photo':
                break
            # URL, ファイル名
            media_url = image['media_url']
            filename = str(id).zfill(5)
            # ダウンロード
            try:
                temp_file = urllib.request.urlopen(media_url+":small").read()
            except:
                continue
            # md5の取得
            current_md5 = hashlib.md5(temp_file).hexdigest()
            # すでに取得済みの画像は飛ばす
            if current_md5 in file_md5:
                continue
            # 画像判定呼出
            current_hash = None
            current_hash, facex, facey, facew, faceh = detector.face_2d(temp_file, status.user.screen_name, filename)
            if current_hash is not None:
                # 取得済みとしてハッシュ値を保存
                return_md5.append(current_md5)
                # ハッシュタグがあれば保存する
                tags = []
                if hasattr(status, "entities"):
                    if "hashtags" in status.entities:
                        for hashtag in status.entities['hashtags']:
                            tags.append(hashtag['text'])
                # データベースに保存
                SQL = "insert into {} values (?,?,?,?,?,?,?,?,?,?)".format(mode)
                url = "https://twitter.com/" + status.user.screen_name + "/status/" + status.id_str
                value = (filename, media_url, status.user.screen_name, url, str(tags).replace("'",""),
                        str(datetime.datetime.now()), str(facex), str(facey), str(facew), str(faceh))
                dbfile.execute(SQL, value)
                dbfile.commit()
                id += 1
            temp_file = None
            image_count += 1
    return id, image_count, return_md5

def getTweets(api, mode, count, query):
    start = time.time()

    dbpath = os.path.abspath(__file__).replace(os.path.basename(__file__),"/DB/user/"+ api.me().id_str + ".db")
    dbfile = sqlite3.connect(dbpath)

    # DBのリセット等
    reset(dbfile, mode)
    tweet_count = 0
    image_count = 0
    temp_count = 0
    id = 0
    tweet_id = []
    file_md5 = []
    return_md5 = []
    # 取得モード
    if mode == "timeline":
        for status in tp.Cursor(api.home_timeline).items(count):
            if status.id in tweet_id:
                continue
            else:
                tweet_id.append(status.id)
            id, temp_count, return_md5 = on_status(status, dbfile, mode, id, file_md5)
            file_md5 += return_md5
            image_count += temp_count
            tweet_count += 1
    elif mode == "fav":
        for status in tp.Cursor(api.favorites).items(count):
            if status.id in tweet_id:
                continue
            else:
                tweet_id.append(status.id)
            id, temp_count, return_md5 = on_status(status, dbfile, mode, id, file_md5)
            file_md5 += return_md5
            image_count += temp_count
    elif mode == "user":
        for status in tp.Cursor(api.user_timeline, screen_name=query).items(count):
            if status.id in tweet_id:
                continue
            else:
                tweet_id.append(status.id)
            id, temp_count, return_md5 = on_status(status, dbfile, mode, id, file_md5)
            file_md5 += return_md5
            image_count += temp_count
            tweet_count += 1
    elif mode == "list":
        listurl = query.replace("https://","")
        owner = listurl.split("/")[1]
        slug = listurl.split("/")[3]
        for status in tp.Cursor(api.list_timeline, owner_screen_name=owner, slug=slug).items(count):
            if status.id in tweet_id:
                continue
            else:
                tweet_id.append(status.id)
            id, temp_count, return_md5 = on_status(status, dbfile, mode, id, file_md5)
            file_md5 += return_md5
            image_count += temp_count
            tweet_count += 1
    elif mode == "tag":
        for status in tp.Cursor(api.search, q="#" + query).items(count):
            if status.id in tweet_id:
                continue
            else:
                tweet_id.append(status.id)
            id, temp_count, return_md5 = on_status(status, dbfile, mode, id, file_md5)
            file_md5 += return_md5
            image_count += temp_count
            tweet_count += 1
    elif mode == "keyword":
        for status in tp.Cursor(api.search, q=query).items(count):
            if status.id in tweet_id:
                continue
            else:
                tweet_id.append(status.id)
            id, temp_count, return_md5 = on_status(status, dbfile, mode, id, file_md5)
            file_md5 += return_md5
            image_count += temp_count
            tweet_count += 1

    elapsed_time = time.time() - start
    # 実行時間,全ツイート数,全画像枚数をデータベースに
    SQL = "insert into result values (?,?,?,?)"
    value = (mode, str(elapsed_time), str(image_count), str(tweet_count))
    dbfile.execute(SQL, value)
    dbfile.commit()

<|endoftext|>


<commit_before>import argparse
import getpass
import glob
import json
import os
import time
import urllib2
from PIL import Image
import pwd
import platform

def parse_args():
        parser = argparse.ArgumentParser(
            description='Set wallpaper from your choice of subreddit!')
        parser.add_argument(''
            '--subreddit', type=str, help='Your choice of subreddit to download Images')
        parser.add_argument('-hq', action='store_true',
                            help='If you want to download only high quality photos')
        parser.add_argument(
            '-download', action='store_true', help='Download the photos now!')
        args = parser.parse_args()
        return args

# URL of the wallpaper JSON
# Default url - if no subreddit specified then use this
url = 'http://www.reddit.com/r/wallpaper.json'
WallpaperCount = 0
hqchoice = 0
downloadNow = 0
ops=0
directory = ""

def getOs():
    global directory
    global ops
    print("Inside getOs")
    username = pwd.getpwuid(os.getuid()).pw_name
    if platform.system()=="Darwin":
        directory = "/Users/" + username + "/Pictures/Wallpapers/"
        ops=1
    elif platform.system()=="Linux":
        directory = "/home/"+username+"/Wallpapers/"
        ops=0

def removeUnwantedPhotos(listwallpaper):
    '''To remove non-jpg downloaded photos'''
    for paths in listwallpaper:
        if os.path.getsize(paths) < 102400:
            np = "rm " + paths
            os.system(np)
        if hqchoice == 1:
            with Image.open(paths) as im:
                width, height = im.size
                if width < 1500 or height < 1500:
                    np = "rm" + paths
                    os.system(np)

def returnwallpaper():
    '''Returns a list of wallpapers from the folder'''
    directoryWall = directory + "*.jpg"
    print(directoryWall)
    listWallpaper = glob.glob(directoryWall)
    print("Wallpapers Returned : ", listWallpaper)
    return listWallpaper

def setwallpaper(listwallpaper, count):
    '''Gets the list of wallpaper and sets a new wallpaper based on count'''
    if ops==0:
        filepath = "gsettings set org.gnome.desktop.background picture-uri file:" + listwallpaper[count]
    elif ops==1:
        filepath = "osascript -e 'tell application \"Finder\" to set desktop picture to POSIX file \"" + listwallpaper[count] +"\""
    os.system(filepath)

def download(dCount):
    checkVar = 0
    # To avoid 'Too many requests error - 2 second wait and try again in case error
    while(checkVar == dCount):
        try:
            obj = urllib2.urlopen(url)
            data = json.load(obj)
            print("Connection Established! Hurray!")
            checkVar += 1
        except:
            print("Request Error, Trying again!")
            time.sleep(2)
            pass

    if not os.path.exists(directory):
        os.makedirs(directory)

    for i in data["data"]["children"]:

        imurl = i["data"]["url"]
        checkUrl = str(imurl)

        try:

            print imurl

            req = urllib2.Request(imurl+".jpg")
            time.sleep(2)
            imgdata = urllib2.urlopen(req).read()
            time.sleep(2)
            print("Downloaded Image!")
            fp = open((directory + "%s"+".jpg") % (dCount), "wb")

            fp.write(imgdata)

            fp.close()

            dCount = dCount + 1

        except:
            pass
            time.sleep(2)
    print dCount
    return dCount

def countdownDownload(timerDownload, timerupdate, count):
    '''Time Countdown function to check wallpaper change time or download time'''

    stu = timerupdate

    listwallpaper = returnwallpaper()
    removeUnwantedPhotos(listwallpaper)
    listwallpaper = returnwallpaper()
    WallpaperCount = len(listwallpaper)

    setwallpaper(listwallpaper, count)
    count += 1

    # 24 hour loop
    while timerDownload:
            # 1 Minute Loop
        while timerupdate:
            time.sleep(1)
            timerupdate -= 1
        print("End of 15 Minutes, Wallpaper changed!")

        if(count >= WallpaperCount):
            count = 0

        setwallpaper(listwallpaper, count)
        count += 1

        timerupdate = stu
        timerDownload = timerDownload - stu

    dCount = download()
    return dCount

def fetchPreferences():
    '''Function to fetch preferences from file'''
    Preferences = list()
    pref = open("WpPreferences.txt", "r")
    for word in pref.read().split():
        Preferences.append(int(word))
    return Preferences

def saveToPreferences(Preferences):
    '''Function to the Index of Latest downloaded Wallpaper'''
    for i in range(0, 5):
        Preferences[i] = str(Preferences[i])
        Preferences[i] = Preferences[i] + "\n"
    with open('WpPreferences.txt', 'w') as file:
        file.writelines(Preferences)

def firstTimeDownload(Preferences):
    '''To download wallpapers automatically on first run'''
    if Preferences[3] == 0 or downloadNow == 1:
        print("Downloading")
        dCount = download(0)
        Preferences[3] = 1
        Preferences[4] = dCount
    saveToPreferences(Preferences)
    return Preferences

def main():
    getOs()
    Preferences = fetchPreferences()
    Preferences = firstTimeDownload(Preferences)
    Preferences = fetchPreferences()

    timerDownload = Preferences[2] * 60 * 60
    timerupdate = Preferences[1] * 60

    while(1):
        count = 0
        dCount = countdownDownload(timerDownload, timerupdate, count)
        Preferences[4] = dCount
        saveToPreferences(Preferences)

if __name__ == '__main__':
    user_choice = parse_args()
    if not user_choice.subreddit:
        url = 'http://www.reddit.com/r/wallpaper.json'
    else:
        url = 'http://www.reddit.com/r/' + user_choice.subreddit + '.json'
    if user_choice.hq == True:
        hqchoice = 1
    print(user_choice.download)
    if user_choice.download == True:
        downloadNow = 1
    main()
<commit_msg>Support for mac added successfully<commit_after>import argparse
import getpass
import glob
import json
import os
import time
import urllib2
from PIL import Image
import pwd
import platform

def parse_args():
        parser = argparse.ArgumentParser(
            description='Set wallpaper from your choice of subreddit!')
        parser.add_argument(''
            '--subreddit', type=str, help='Your choice of subreddit to download Images')
        parser.add_argument('-hq', action='store_true',
                            help='If you want to download only high quality photos')
        parser.add_argument(
            '-download', action='store_true', help='Download the photos now!')
        args = parser.parse_args()
        return args

# URL of the wallpaper JSON
# Default url - if no subreddit specified then use this
url = 'http://www.reddit.com/r/wallpaper.json'
WallpaperCount = 0
hqchoice = 0
downloadNow = 0
ops=0
directory = ""

def getOs():
    global directory
    global ops
    print("Inside getOs")
    username = pwd.getpwuid(os.getuid()).pw_name
    if platform.system()=="Darwin":
        directory = "/Users/" + username + "/Pictures/Wallpapers/"
        ops=1
    elif platform.system()=="Linux":
        directory = "/home/"+username+"/Wallpapers/"
        ops=0

def removeUnwantedPhotos(listwallpaper):
    '''To remove non-jpg downloaded photos'''
    for paths in listwallpaper:
        if os.path.getsize(paths) < 102400:
            np = "rm " + paths
            os.system(np)
        if hqchoice == 1:
            with Image.open(paths) as im:
                width, height = im.size
                if width < 1500 or height < 1500:
                    np = "rm" + paths
                    os.system(np)

def returnwallpaper():
    '''Returns a list of wallpapers from the folder'''
    directoryWall = directory + "*.jpg"
    print(directoryWall)
    listWallpaper = glob.glob(directoryWall)
    print("Wallpapers Returned : ", listWallpaper)
    return listWallpaper

def setwallpaper(listwallpaper, count):
    '''Gets the list of wallpaper and sets a new wallpaper based on count'''
    if ops==0:
        filepath = "gsettings set org.gnome.desktop.background picture-uri file:" + listwallpaper[count]
    elif ops==1:
        filepath = "osascript -e 'tell application \"Finder\" to set desktop picture to POSIX file \"" + listwallpaper[count] +"\"'"
    os.system(filepath)

def download(dCount):
    checkVar = 0
    # To avoid 'Too many requests error - 2 second wait and try again in case error
    while(checkVar == dCount):
        try:
            obj = urllib2.urlopen(url)
            data = json.load(obj)
            print("Connection Established! Hurray!")
            checkVar += 1
        except:
            print("Request Error, Trying again!")
            time.sleep(2)
            pass

    if not os.path.exists(directory):
        os.makedirs(directory)

    for i in data["data"]["children"]:

        imurl = i["data"]["url"]
        checkUrl = str(imurl)

        try:

            print imurl

            req = urllib2.Request(imurl+".jpg")
            time.sleep(2)
            imgdata = urllib2.urlopen(req).read()
            time.sleep(2)
            print("Downloaded Image!")
            fp = open((directory + "%s"+".jpg") % (dCount), "wb")

            fp.write(imgdata)

            fp.close()

            dCount = dCount + 1

        except:
            pass
            time.sleep(2)
    print dCount
    return dCount

def countdownDownload(timerDownload, timerupdate, count):
    '''Time Countdown function to check wallpaper change time or download time'''

    stu = timerupdate

    listwallpaper = returnwallpaper()
    removeUnwantedPhotos(listwallpaper)
    listwallpaper = returnwallpaper()
    WallpaperCount = len(listwallpaper)

    setwallpaper(listwallpaper, count)
    count += 1

    # 24 hour loop
    while timerDownload:
            # 1 Minute Loop
        while timerupdate:
            time.sleep(1)
            timerupdate -= 1
        print("End of 15 Minutes, Wallpaper changed!")

        if(count >= WallpaperCount):
            count = 0

        setwallpaper(listwallpaper, count)
        count += 1

        timerupdate = stu
        timerDownload = timerDownload - stu

    dCount = download()
    return dCount

def fetchPreferences():
    '''Function to fetch preferences from file'''
    Preferences = list()
    pref = open("WpPreferences.txt", "r")
    for word in pref.read().split():
        Preferences.append(int(word))
    return Preferences

def saveToPreferences(Preferences):
    '''Function to the Index of Latest downloaded Wallpaper'''
    for i in range(0, 5):
        Preferences[i] = str(Preferences[i])
        Preferences[i] = Preferences[i] + "\n"
    with open('WpPreferences.txt', 'w') as file:
        file.writelines(Preferences)

def firstTimeDownload(Preferences):
    '''To download wallpapers automatically on first run'''
    if Preferences[3] == 0 or downloadNow == 1:
        print("Downloading")
        dCount = download(0)
        Preferences[3] = 1
        Preferences[4] = dCount
    saveToPreferences(Preferences)
    return Preferences

def main():
    getOs()
    Preferences = fetchPreferences()
    Preferences = firstTimeDownload(Preferences)
    Preferences = fetchPreferences()

    timerDownload = Preferences[2] * 60 * 60
    timerupdate = Preferences[1] * 60

    while(1):
        count = 0
        dCount = countdownDownload(timerDownload, timerupdate, count)
        Preferences[4] = dCount
        saveToPreferences(Preferences)

if __name__ == '__main__':
    user_choice = parse_args()
    if not user_choice.subreddit:
        url = 'http://www.reddit.com/r/wallpaper.json'
    else:
        url = 'http://www.reddit.com/r/' + user_choice.subreddit + '.json'
    if user_choice.hq == True:
        hqchoice = 1
    print(user_choice.download)
    if user_choice.download == True:
        downloadNow = 1
    main()
<|endoftext|>


<commit_before>import hashlib
import io
import os
import struct
import zlib

import btreedb4
import sbvj01
import sbon

class KeyStore(btreedb4.FileBTreeDB4):
    """A B-tree database that uses SHA-256 hashes for key lookup.

    """
    def get(self, key):
        # Get the SHA-256 hash of the key.
        key = hashlib.sha256(key.encode('utf-8')).digest()
        return super(KeyStore, self).get(key)

class Package(KeyStore):
    """A B-tree database representing a package of files.

    """
    DIGEST_KEY = '_digest'
    INDEX_KEY = '_index'

    def __init__(self, path):
        super(Package, self).__init__(path)
        self._index = None

    def get_digest(self):
        return self.get(Package.DIGEST_KEY)

    def get_index(self):
        if self._index:
            return self._index

        stream = io.BytesIO(self.get(Package.INDEX_KEY))
        self._index = sbon.read_string_list(stream)
        return self._index

class Player(sbvj01.FileSBVJ01):
    """A Starbound character.

    """
    def __init__(self, path):
        super(Player, self).__init__(path)
        self.name = None

    def open(self):
        super(Player, self).open()
        assert self.identifier == 'PlayerEntity', 'Invalid player file'
        self.name = self.data['identity']['name']

class World(btreedb4.FileBTreeDB4):
    """A single Starbound world.

    """
    DATA_KEY = '\x00\x00\x00\x00\x00'

    TILES_X = 32
    TILES_Y = 32
    TILES_PER_REGION = TILES_X * TILES_Y

    def __init__(self, path):
        super(World, self).__init__(path)
        self._world_data = None

    def get_entities(self, x, y):
        stream = io.BytesIO(self.get_region_data(x, y, 2))
        return sbon.read_document_list(stream)

    def get_region_data(self, x, y, layer=1):
        """Get the raw data for a region. Regions are sets of 32x32 in-game
        tiles. Layer 1 holds tile data and layer 2 holds entity data.

        """
        key = struct.pack('>BHH', layer, x, y)
        return zlib.decompress(self.get(key))

    def get_tiles(self, x, y):
        stream = io.BytesIO(self.get_region_data(x, y, 1))
        unknown = stream.read(3)
        # There are 1024 (32x32) tiles in a region.
        return [sbon.read_tile(stream) for _ in xrange(World.TILES_PER_REGION)]

    def get_world_data(self):
        if self._world_data:
            return self._world_data

        stream = io.BytesIO(self.get(World.DATA_KEY))

        # Not sure what these values mean.
        unknown_1, unknown_2 = struct.unpack('>ii', stream.read(8))

        name, data = sbon.read_document(stream)
        assert name == 'WorldMetadata', 'Invalid world data'

        self._world_data = data
        return data

    def open(self):
        super(World, self).open()
        assert self.identifier == 'World2', 'Tried to open non-world BTreeDB4 file'

def open(path):
    _, extension = os.path.splitext(path)
    if extension == '.clientcontext':
        file = sbvj01.FileSBVJ01(path)
    elif extension == '.db':
        file = KeyStore(path)
    elif extension in ('.modpak', '.pak'):
        file = Package(path)
    elif extension == '.player':
        file = Player(path)
    elif extension in ('.shipworld', '.world'):
        file = World(path)
    else:
        raise ValueError('Unrecognized file extension')

    file.open()
    return file
<commit_msg>Improve KeyError for hashed keys<commit_after>import hashlib
import binascii
import io
import os
import struct
import zlib

import btreedb4
import sbvj01
import sbon

class KeyStore(btreedb4.FileBTreeDB4):
    """A B-tree database that uses SHA-256 hashes for key lookup.

    """
    def get(self, key):
        # Get the SHA-256 hash of the key.
        hashed_key = hashlib.sha256(key.encode('utf-8')).digest()
        try:
            return super(KeyStore, self).get(hashed_key)
        except KeyError:
            raise KeyError('%s (%s)' % (key, binascii.hexlify(hashed_key)))

class Package(KeyStore):
    """A B-tree database representing a package of files.

    """
    DIGEST_KEY = '_digest'
    INDEX_KEY = '_index'

    def __init__(self, path):
        super(Package, self).__init__(path)
        self._index = None

    def get_digest(self):
        return self.get(Package.DIGEST_KEY)

    def get_index(self):
        if self._index:
            return self._index

        stream = io.BytesIO(self.get(Package.INDEX_KEY))
        self._index = sbon.read_string_list(stream)
        return self._index

class Player(sbvj01.FileSBVJ01):
    """A Starbound character.

    """
    def __init__(self, path):
        super(Player, self).__init__(path)
        self.name = None

    def open(self):
        super(Player, self).open()
        assert self.identifier == 'PlayerEntity', 'Invalid player file'
        self.name = self.data['identity']['name']

class World(btreedb4.FileBTreeDB4):
    """A single Starbound world.

    """
    DATA_KEY = '\x00\x00\x00\x00\x00'

    TILES_X = 32
    TILES_Y = 32
    TILES_PER_REGION = TILES_X * TILES_Y

    def __init__(self, path):
        super(World, self).__init__(path)
        self._world_data = None

    def get_entities(self, x, y):
        stream = io.BytesIO(self.get_region_data(x, y, 2))
        return sbon.read_document_list(stream)

    def get_region_data(self, x, y, layer=1):
        """Get the raw data for a region. Regions are sets of 32x32 in-game
        tiles. Layer 1 holds tile data and layer 2 holds entity data.

        """
        key = struct.pack('>BHH', layer, x, y)
        return zlib.decompress(self.get(key))

    def get_tiles(self, x, y):
        stream = io.BytesIO(self.get_region_data(x, y, 1))
        unknown = stream.read(3)
        # There are 1024 (32x32) tiles in a region.
        return [sbon.read_tile(stream) for _ in xrange(World.TILES_PER_REGION)]

    def get_world_data(self):
        if self._world_data:
            return self._world_data

        stream = io.BytesIO(self.get(World.DATA_KEY))

        # Not sure what these values mean.
        unknown_1, unknown_2 = struct.unpack('>ii', stream.read(8))

        name, data = sbon.read_document(stream)
        assert name == 'WorldMetadata', 'Invalid world data'

        self._world_data = data
        return data

    def open(self):
        super(World, self).open()
        assert self.identifier == 'World2', 'Tried to open non-world BTreeDB4 file'

def open(path):
    _, extension = os.path.splitext(path)
    if extension == '.clientcontext':
        file = sbvj01.FileSBVJ01(path)
    elif extension == '.db':
        file = KeyStore(path)
    elif extension in ('.modpak', '.pak'):
        file = Package(path)
    elif extension == '.player':
        file = Player(path)
    elif extension in ('.shipworld', '.world'):
        file = World(path)
    else:
        raise ValueError('Unrecognized file extension')

    file.open()
    return file
<|endoftext|>


<commit_before>

__author__ = "Ryan Faulkner"
__date__ = "July 27th, 2012"
__license__ = "GPL (version 2 or later)"

from dateutil.parser import parse as date_parse
import datetime
import user_metric as um

LAST_EDIT = -1
REGISTRATION = 0

class TimeToThreshold(um.UserMetric):
    """
        Produces an integer value representing the number of minutes taken to reach a threshold.

            `https://meta.wikimedia.org/wiki/Research:Metrics//time_to_threshold`

        As a UserMetric type this class utilizes the process() function attribute to produce an internal list of metrics by
        user handle (typically ID but user names may also be specified). The execution of process() produces a nested list that
        stores in each element:

            * User ID
            * Time difference in minutes between events

        Below is an example of how we can generate the time taken to reach the first edit and the last edit from registration : ::

            >>> import src.metrics.time_to_threshold as t
            >>> t.TimeToThreshold(t.TimeToThreshold.EditCountThreshold, first_edit=0, threshold_edit=1).process([13234584]).__iter__().next()
            [13234584, 3318]
            >>> t.TimeToThreshold(t.TimeToThreshold.EditCountThreshold, first_edit=0, threshold_edit=-1).process([13234584]).__iter__().next()
            [13234584, 18906]

        If the termination event never occurs the number of minutes returned is -1.
    """

    def __init__(self,
                 threshold_type_class=None,
                 **kwargs):

        um.UserMetric.__init__(self, **kwargs)
        if threshold_type_class is None:
             self._threshold_obj_ = self.EditCountThreshold(**kwargs)
        else:
            try:
                self._threshold_obj_ = threshold_type_class(**kwargs)
            except NameError:
                self._threshold_obj_ = self.EditCountThreshold(**kwargs)
                print str(datetime.datetime.now()) + ' - Invalid threshold class. Using default (EditCountThreshold).'

    @staticmethod
    def header(): return ['user_id', 'minutes_diff']

    def process(self, user_handle, is_id=True, **kwargs):
        """ Wrapper for specific threshold objects """
        self._results =  self._threshold_obj_.process(user_handle, self, is_id=is_id, **kwargs)
        return self

    class EditCountThreshold():
        """
            Nested Class. Objects of this class are to be created by the constructor of TimeToThreshold.  The class has one method,
            process(), which computes the time, in minutes, taken between making N edits and M edits for a user.  N < M.
        """

        def __init__(self, first_edit=REGISTRATION, threshold_edit=1):
            """
                Object constructor.  There are two required parameters:

                    - Parameters:
                        - **first_edit** - Integer.  The numeric value of the first edit from which to measure the threshold.
                        - **threshold_edit** - Integer.  The numeric value of the threshold edit from which to measure the threshold
            """

            try:
                self._first_edit_ = int(first_edit)
                self._threshold_edit_ = int(threshold_edit)

            except ValueError:
                raise um.UserMetric.UserMetricError(str(self.__class__()) + ': Invalid init params.')

        def process(self, user_handle, threshold_obj, is_id=True, **kwargs):
            """
                First determine if the user has made an adequate number of edits.  If so, compute the number of minutes that passed
                between the Nth and Mth edit.

                    - Parameters:
                        - **user_handle** - List(int).  List of user ids.
                        - **first_edit** - Integer.  The numeric value of the first edit from which to measure the threshold.
                        - **threshold_edit** - Integer.  The numeric value of the threshold edit from which to measure the threshold
            """

            minutes_to_threshold = list()

            # Operate on either user ids or names
            if is_id:
                user_revs_SQL = 'select rev_timestamp from %(project)s.revision where rev_user = "%(user_handle)s" order by 1 asc'
            else:
                user_revs_SQL = 'select rev_timestamp from %(project)s.revision where rev_user_text = "%(user_handle)s" order by 1 asc'

            if not hasattr(user_handle, '__iter__'): user_handle = [user_handle] # ensure the handles are iterable

            # For each user gather their revisions
            for user in user_handle:
                sql = user_revs_SQL % {'user_handle' : str(user), 'project' : threshold_obj._project_}
                revs = [rev[0] for rev in threshold_obj._data_source_.execute_SQL(sql)]
                minutes_to_threshold.append([user, self._get_minute_diff_result(revs)])

            return minutes_to_threshold

        def _get_minute_diff_result(self, results):
            """
                Private method for this class.  This computes the minutes to threshold for the timestamp results.

                    - Parameters:
                        - **results** - list.  list of revision records with timestamp for a given user.
            """
            if self._threshold_edit_ == REGISTRATION and len(results):
                dat_obj_end = date_parse(results[0])
            elif self._threshold_edit_ == LAST_EDIT and len(results):
                dat_obj_end = date_parse(results[len(results) - 1])
            elif self._threshold_edit_ < len(results):
                dat_obj_end = date_parse(results[self._threshold_edit_])
            else:
                return -1

            if self._first_edit_ == REGISTRATION and len(results) > 0:
                dat_obj_start = date_parse(results[0])
            elif self._first_edit_ == LAST_EDIT and len(results):
                dat_obj_start = date_parse(results[len(results) - 1])
            elif self._first_edit_ < len(results):
                dat_obj_start = date_parse(results[self._first_edit_])
            else:
                return -1

            time_diff = dat_obj_end - dat_obj_start
            return int(time_diff.seconds / 60) + abs(time_diff.days) * 24<commit_msg>ADD: parameter types structure for TimeToThreshold<commit_after>

__author__ = "Ryan Faulkner"
__date__ = "July 27th, 2012"
__license__ = "GPL (version 2 or later)"

from dateutil.parser import parse as date_parse
import datetime
import user_metric as um

LAST_EDIT = -1
REGISTRATION = 0

class TimeToThreshold(um.UserMetric):
    """
        Produces an integer value representing the number of minutes taken to reach a threshold.

            `https://meta.wikimedia.org/wiki/Research:Metrics//time_to_threshold`

        As a UserMetric type this class utilizes the process() function attribute to produce an internal list of metrics by
        user handle (typically ID but user names may also be specified). The execution of process() produces a nested list that
        stores in each element:

            * User ID
            * Time difference in minutes between events

        Below is an example of how we can generate the time taken to reach the first edit and the last edit from registration : ::

            >>> import src.metrics.time_to_threshold as t
            >>> t.TimeToThreshold(t.TimeToThreshold.EditCountThreshold, first_edit=0, threshold_edit=1).process([13234584]).__iter__().next()
            [13234584, 3318]
            >>> t.TimeToThreshold(t.TimeToThreshold.EditCountThreshold, first_edit=0, threshold_edit=-1).process([13234584]).__iter__().next()
            [13234584, 18906]

        If the termination event never occurs the number of minutes returned is -1.
    """

    # Structure that defines parameters for TimeToThreshold class
    _param_types = {
        'init' : {
            'threshold_type_class' : ['str', 'Type of threshold to use.'],
            },
        'process' : {
            'is_id' : ['bool', 'Are user ids or names being passed.'],
        }
    }

    def __init__(self,
                 threshold_type_class=None,
                 **kwargs):

        if not threshold_type_class:
             self._threshold_obj_ = self.EditCountThreshold(**kwargs)
        else:
            try:
                self._threshold_obj_ = self.__threshold_types[threshold_type_class](**kwargs)
            except NameError:
                self._threshold_obj_ = self.EditCountThreshold(**kwargs)
                print str(datetime.datetime.now()) + ' - Invalid threshold class. Using default (EditCountThreshold).'

        um.UserMetric.__init__(self, **kwargs)
        self.append_params(um.UserMetric)   # add params from base class
        self._param_types['threshold_type_params'] = self._threshold_obj_._param_types

    @staticmethod
    def header(): return ['user_id', 'minutes_diff']

    def process(self, user_handle, is_id=True, **kwargs):
        """ Wrapper for specific threshold objects """
        self._results =  self._threshold_obj_.process(user_handle, self, is_id=is_id, **kwargs)
        return self

    class EditCountThreshold():
        """
            Nested Class. Objects of this class are to be created by the constructor of TimeToThreshold.  The class has one method,
            process(), which computes the time, in minutes, taken between making N edits and M edits for a user.  N < M.
        """

        # Structure that defines parameters for TimeToThreshold class
        _param_types = {
            'init' : {
                'first_edit' : ['int', 'Event that initiates measurement period.'],
                'threshold_edit' : ['int', 'Threshold event.'],
                },
            'process' : {
                'is_id' : ['bool', 'Are user ids or names being passed.'],
                }
        }

        def __init__(self, first_edit=REGISTRATION, threshold_edit=1):
            """
                Object constructor.  There are two required parameters:

                    - Parameters:
                        - **first_edit** - Integer.  The numeric value of the first edit from which to measure the threshold.
                        - **threshold_edit** - Integer.  The numeric value of the threshold edit from which to measure the threshold
            """

            try:
                self._first_edit_ = int(first_edit)
                self._threshold_edit_ = int(threshold_edit)

            except ValueError:
                raise um.UserMetric.UserMetricError(str(self.__class__()) + ': Invalid init params.')

        def process(self, user_handle, threshold_obj, is_id=True, **kwargs):
            """
                First determine if the user has made an adequate number of edits.  If so, compute the number of minutes that passed
                between the Nth and Mth edit.

                    - Parameters:
                        - **user_handle** - List(int).  List of user ids.
                        - **first_edit** - Integer.  The numeric value of the first edit from which to measure the threshold.
                        - **threshold_edit** - Integer.  The numeric value of the threshold edit from which to measure the threshold
            """

            minutes_to_threshold = list()

            # Operate on either user ids or names
            if is_id:
                user_revs_SQL = 'select rev_timestamp from %(project)s.revision where rev_user = "%(user_handle)s" order by 1 asc'
            else:
                user_revs_SQL = 'select rev_timestamp from %(project)s.revision where rev_user_text = "%(user_handle)s" order by 1 asc'

            if not hasattr(user_handle, '__iter__'): user_handle = [user_handle] # ensure the handles are iterable

            # For each user gather their revisions
            for user in user_handle:
                sql = user_revs_SQL % {'user_handle' : str(user), 'project' : threshold_obj._project_}
                revs = [rev[0] for rev in threshold_obj._data_source_.execute_SQL(sql)]
                minutes_to_threshold.append([user, self._get_minute_diff_result(revs)])

            return minutes_to_threshold

        def _get_minute_diff_result(self, results):
            """
                Private method for this class.  This computes the minutes to threshold for the timestamp results.

                    - Parameters:
                        - **results** - list.  list of revision records with timestamp for a given user.
            """
            if self._threshold_edit_ == REGISTRATION and len(results):
                dat_obj_end = date_parse(results[0])
            elif self._threshold_edit_ == LAST_EDIT and len(results):
                dat_obj_end = date_parse(results[len(results) - 1])
            elif self._threshold_edit_ < len(results):
                dat_obj_end = date_parse(results[self._threshold_edit_])
            else:
                return -1

            if self._first_edit_ == REGISTRATION and len(results) > 0:
                dat_obj_start = date_parse(results[0])
            elif self._first_edit_ == LAST_EDIT and len(results):
                dat_obj_start = date_parse(results[len(results) - 1])
            elif self._first_edit_ < len(results):
                dat_obj_start = date_parse(results[self._first_edit_])
            else:
                return -1

            time_diff = dat_obj_end - dat_obj_start
            return int(time_diff.seconds / 60) + abs(time_diff.days) * 24

    __threshold_types = { 'edit_count_threshold' : EditCountThreshold }<|endoftext|>