<commit_before>from __future__ import unicode_literals

import json

import pytz
import time
from dash.utils import is_dict_equal
from dash.utils.sync import BaseSyncer, sync_local_to_set, sync_local_to_changes
from django.core.cache import cache
from django.utils import timezone

from ureport.contacts.models import ContactField, Contact
from ureport.locations.models import Boundary
from ureport.polls.models import PollResult
from ureport.utils import datetime_to_json_date
from . import BaseBackend

class FieldSyncer(BaseSyncer):
    """
    Syncer for contact fields
    """
    model = ContactField
    local_id_attr = 'key'
    remote_id_attr = 'key'

    def local_kwargs(self, org, remote):
        return {
            'org': org,
            'key': remote.key,
            'label': remote.label,
            'value_type': self.model.TEMBA_TYPES.get(remote.value_type, self.model.TYPE_TEXT)
        }

    def update_required(self, local, remote, local_kwags):
        return local.label != remote.label or local.value_type != self.model.TEMBA_TYPES.get(remote.value_type)

    def delete_locale(self, local):
        local.release()

class BoundarySyncer(BaseSyncer):
    """
    syncer for location boundaries
    """
    model = Boundary
    local_id_attr = 'osm_id'
    remote_id_attr = 'boundary'

    def local_kwargs(self, org, remote):
        geometry = json.dumps(dict(type=remote.geometry.type, coordinates=remote.geometry.coordinates))

        parent = Boundary.objects.filter(osm_id__iexact=remote.parent, org=org).first()

        return {
            'org': org,
            'geometry': geometry,
            'parent': parent,
            'level': remote.level,
            'name': remote.name,
            'osm_id': remote.boundary
        }

    def update_required(self, local, remote, local_kwargs):
        if local.name != remote.name:
            return True

        if local.level != remote.level:
            return True

        if remote.parent:
            if not local.parent:
                return True
            elif local.parent.osm_id != remote.parent:
                return True

        return not is_dict_equal(json.loads(local.geometry),
                                 dict(type=remote.geometry.type, coordinates=remote.geometry.coordinates))

    def delete_locale(self, local):
        local.release()

class ContactSyncer(BaseSyncer):
    model = Contact

    def get_boundaries_data(self, org):

        cache_attr = '__boundaries__%d' % org.pk
        if hasattr(self, cache_attr):
            return getattr(self, cache_attr)

        org_state_boundaries_data = dict()
        org_district_boundaries_data = dict()
        state_boundaries = Boundary.objects.filter(org=org, level=1)
        for state in state_boundaries:
            org_state_boundaries_data[state.name.lower()] = state.osm_id
            state_district_data = dict()
            district_boundaries = Boundary.objects.filter(org=org, level=2, parent=state)
            for district in district_boundaries:
                state_district_data[district.name.lower()] = district.osm_id

            org_district_boundaries_data[state.osm_id] = state_district_data

        setattr(self, cache_attr, (org_state_boundaries_data, org_district_boundaries_data))
        return org_state_boundaries_data, org_district_boundaries_data

    def get_contact_fields(self, org):
        cache_attr = '__contact_fields__%d' % org.pk
        if hasattr(self, cache_attr):
            return getattr(self, cache_attr)
        contact_fields = ContactField.objects.filter(org=org)
        contact_fields_data = {elt.label.lower(): elt.key for elt in contact_fields}

        setattr(self, cache_attr, contact_fields_data)
        return contact_fields_data

    def local_kwargs(self, org, remote):
        from ureport.utils import json_date_to_datetime

        reporter_group = org.get_config('reporter_group')
        contact_groups_names = [group.name.lower() for group in remote.groups]

        if not reporter_group.lower() in contact_groups_names:
            return None

        org_state_boundaries_data, org_district_boundaries_data = self.get_boundaries_data(org)
        contact_fields = self.get_contact_fields(org)

        state = ''
        district = ''

        state_field = org.get_config('state_label')
        if state_field:
            state_field = state_field.lower()
            if org.get_config('is_global'):
                state_name = remote.fields.get(contact_fields.get(state_field), None)
                if state_name:
                    state = state_name

            else:
                state_name = remote.fields.get(contact_fields.get(state_field), None)
                if state_name:
                    state_name = state_name.lower()
                    state = org_state_boundaries_data.get(state_name, '')

                district_field = org.get_config('district_label')
                if district_field:
                    district_field = district_field.lower()
                    district_name = remote.fields.get(contact_fields.get(district_field), None)
                    if district_name:
                        district_name = district_name.lower()
                        district = org_district_boundaries_data.get(state, dict()).get(district_name, '')

        registered_on = None
        registration_field = org.get_config('registration_label')
        if registration_field:
            registration_field = registration_field.lower()
            registered_on = remote.fields.get(contact_fields.get(registration_field), None)
            if registered_on:
                registered_on = json_date_to_datetime(registered_on)

        occupation = ''
        occupation_field = org.get_config('occupation_label')
        if occupation_field:
            occupation_field = occupation_field.lower()
            occupation = remote.fields.get(contact_fields.get(occupation_field), '')
            if not occupation:
                occupation = ''

        born = 0
        born_field = org.get_config('born_label')
        if born_field:
            born_field = born_field.lower()
            try:
                born = int(remote.fields.get(contact_fields.get(born_field), 0))

                # support only positive django integer field valid values
                if born < 0 or born > 2147483647:
                    born = 0

            except ValueError:
                pass
            except TypeError:
                pass

        gender = ''
        gender_field = org.get_config('gender_label')
        female_label = org.get_config('female_label')
        male_label = org.get_config('male_label')

        if gender_field:
            gender_field = gender_field.lower()
            gender = remote.fields.get(contact_fields.get(gender_field), '')

            if gender and gender.lower() == female_label.lower():
                gender = self.model.FEMALE
            elif gender and gender.lower() == male_label.lower():
                gender = self.model.MALE
            else:
                gender = ''

        return {
            'org': org,
            'uuid': remote.uuid,
            'gender': gender,
            'born': born,
            'occupation': occupation,
            'registered_on': registered_on,
            'state': state,
            'district': district
        }

    def update_required(self, local, remote, local_kwargs):
        if not local_kwargs:
            return True

        update = local.gender != local_kwargs['gender'] or local.born != local_kwargs['born']
        update = update or local.occupation != local_kwargs['occupation']
        update = update or local.registered_on != local_kwargs['registered_on']
        update = update or local.state != local_kwargs['state'] or local.district != local_kwargs['district']
        return update

class RapidProBackend(BaseBackend):
    """
    RapidPro instance as a backend
    """
    @staticmethod
    def _get_client(org, api_version):
        return org.get_temba_client(api_version=api_version)

    def pull_fields(self, org):
        client = self._get_client(org, 2)
        incoming_objects = client.get_fields().all(retry_on_rate_exceed=True)

        return sync_local_to_set(org, FieldSyncer(), incoming_objects)

    def pull_boundaries(self, org):

        if org.get_config('is_global'):
            incoming_objects = Boundary.build_global_boundaries()
        else:
            client = self._get_client(org, 1)
            incoming_objects = client.get_boundaries()

        return sync_local_to_set(org, BoundarySyncer(), incoming_objects)

    def pull_contacts(self, org, modified_after, modified_before, progress_callback=None):
        client = self._get_client(org, 2)

        # all contacts created or modified in RapidPro in the time window
        active_query = client.get_contacts(after=modified_after, before=modified_before)
        fetches = active_query.iterfetches(retry_on_rate_exceed=True)

        # all contacts deleted in RapidPro in the same time window
        deleted_query = client.get_contacts(deleted=True, after=modified_after, before=modified_before)
        deleted_fetches = deleted_query.iterfetches(retry_on_rate_exceed=True)

        return sync_local_to_changes(org, ContactSyncer(), fetches, deleted_fetches, progress_callback)

    def pull_results(self, poll, modified_after, modified_before, progress_callback=None):
        org = poll.org
        client = self._get_client(org, 2)

        start = time.time()
        print "Start fetching runs for poll #%d on org #%d" % (poll.pk, org.pk)

        # ignore the TaskState time and use the time we stored in redis
        now = timezone.now()
        after = cache.get(PollResult.POLL_RESULTS_LAST_PULL_CACHE_KEY % (org.pk, poll.pk), None)

        poll_runs_query = client.get_runs(flow=poll.flow_uuid, responded=True, after=after, before=now)
        fetches = poll_runs_query.iterfetches(retry_on_rate_exceed=True)

        num_created = 0
        num_updated = 0
        num_ignored = 0
        num_synced = 0

        existing_poll_results = PollResult.objects.filter(flow=poll.flow_uuid, org=poll.org_id)

        poll_results_map = dict()
        for res in existing_poll_results:
            if res.contact not in poll_results_map:
                poll_results_map[res.contact] = {res.ruleset: res}
            else:
                poll_results_map[res.contact][res.ruleset] = res

        fetch_start = time.time()
        for fetch in fetches:

            print "RapidPro API fetch for poll #%d on org #%d %d - %d took %ds" % (poll.pk, org.pk, num_synced,
                                                                                num_synced + len(fetch),
                                                                                time.time() - fetch_start)

            local_sync_start = time.time()

            contact_uuids = [run.contact.uuid for run in fetch]
            contacts = Contact.objects.filter(org=org, uuid__in=contact_uuids)
            contacts_map = {c.uuid: c for c in contacts}

            new_poll_results = []

            for temba_run in fetch:
                flow_uuid = temba_run.flow.uuid
                contact_uuid = temba_run.contact.uuid
                completed = temba_run.exit_type == 'completed'

                contact_obj = contacts_map.get(contact_uuid, None)

                state = ''
                district = ''
                if contact_obj is not None:
                    state = contact_obj.state
                    district = contact_obj.district

                for temba_step in temba_run.steps:
                    ruleset_uuid = temba_step.node
                    category = temba_step.category
                    text = temba_step.text

                    existing_poll_result = poll_results_map.get(contact_uuid, dict()).get(ruleset_uuid, None)

                    if existing_poll_result is not None:

                        update_required = existing_poll_result.category != category or existing_poll_result.text != text
                        update_required = update_required or existing_poll_result.state != state
                        update_required = update_required or existing_poll_result.district != district
                        update_required = update_required or existing_poll_result.completed != completed

                        if update_required:
                            PollResult.objects.filter(pk=existing_poll_result.pk).update(category=category, text=text,
                                                                                state=state, district=district,
                                                                                completed=completed)

                            num_updated += 1
                        else:
                            num_ignored += 1

                    else:
                        new_poll_results.append(PollResult(org=org, flow=flow_uuid, ruleset=ruleset_uuid,
                                                           contact=contact_uuid, category=category, text=text,
                                                           state=state, date=temba_step.arrived_on,
                                                           district=district, completed=completed))

                        num_created += 1

            PollResult.objects.bulk_create(new_poll_results)

            num_synced += len(fetch)
            if progress_callback:
                progress_callback(num_synced)

            print "Local sync ops for poll #%d on org #%d %d - %d took %ds" % (poll.pk, org.pk, num_synced - len(fetch),
                                                                               num_synced,
                                                                               time.time() - local_sync_start)
            print "Total synced for poll #%d on org #%d %d runs in %ds" % (poll.pk, org.pk, num_synced,
                                                                           time.time() - start)
            fetch_start = time.time()
            print "=" * 40

        # update the time for this poll from which we fetch next time
        cache.set(PollResult.POLL_RESULTS_LAST_PULL_CACHE_KEY % (org.pk, poll.pk),
                  datetime_to_json_date(now.replace(tzinfo=pytz.utc)))

        # from django.db import connection as db_connection, reset_queries
        # slowest_queries = sorted(db_connection.queries, key=lambda q: q['time'], reverse=True)[:10]
        # for q in slowest_queries:
        #     print "=" * 60
        #     print "\n\n\n"
        #     print "%s -- %s" % (q['time'], q['sql'])
        # reset_queries()

        print "Finished pulling results for poll #%d on org #%d runs in %ds, " \
              "created %d, updated %d, ignored %d" % (poll.pk, org.pk, time.time() - start, num_created,
                                                      num_updated, num_ignored)
        return num_created, num_updated, num_ignored<commit_msg>explicitly set timeout to None to cache forever<commit_after>from __future__ import unicode_literals

import json

import pytz
import time
from dash.utils import is_dict_equal
from dash.utils.sync import BaseSyncer, sync_local_to_set, sync_local_to_changes
from django.core.cache import cache
from django.utils import timezone

from ureport.contacts.models import ContactField, Contact
from ureport.locations.models import Boundary
from ureport.polls.models import PollResult
from ureport.utils import datetime_to_json_date
from . import BaseBackend

class FieldSyncer(BaseSyncer):
    """
    Syncer for contact fields
    """
    model = ContactField
    local_id_attr = 'key'
    remote_id_attr = 'key'

    def local_kwargs(self, org, remote):
        return {
            'org': org,
            'key': remote.key,
            'label': remote.label,
            'value_type': self.model.TEMBA_TYPES.get(remote.value_type, self.model.TYPE_TEXT)
        }

    def update_required(self, local, remote, local_kwags):
        return local.label != remote.label or local.value_type != self.model.TEMBA_TYPES.get(remote.value_type)

    def delete_locale(self, local):
        local.release()

class BoundarySyncer(BaseSyncer):
    """
    syncer for location boundaries
    """
    model = Boundary
    local_id_attr = 'osm_id'
    remote_id_attr = 'boundary'

    def local_kwargs(self, org, remote):
        geometry = json.dumps(dict(type=remote.geometry.type, coordinates=remote.geometry.coordinates))

        parent = Boundary.objects.filter(osm_id__iexact=remote.parent, org=org).first()

        return {
            'org': org,
            'geometry': geometry,
            'parent': parent,
            'level': remote.level,
            'name': remote.name,
            'osm_id': remote.boundary
        }

    def update_required(self, local, remote, local_kwargs):
        if local.name != remote.name:
            return True

        if local.level != remote.level:
            return True

        if remote.parent:
            if not local.parent:
                return True
            elif local.parent.osm_id != remote.parent:
                return True

        return not is_dict_equal(json.loads(local.geometry),
                                 dict(type=remote.geometry.type, coordinates=remote.geometry.coordinates))

    def delete_locale(self, local):
        local.release()

class ContactSyncer(BaseSyncer):
    model = Contact

    def get_boundaries_data(self, org):

        cache_attr = '__boundaries__%d' % org.pk
        if hasattr(self, cache_attr):
            return getattr(self, cache_attr)

        org_state_boundaries_data = dict()
        org_district_boundaries_data = dict()
        state_boundaries = Boundary.objects.filter(org=org, level=1)
        for state in state_boundaries:
            org_state_boundaries_data[state.name.lower()] = state.osm_id
            state_district_data = dict()
            district_boundaries = Boundary.objects.filter(org=org, level=2, parent=state)
            for district in district_boundaries:
                state_district_data[district.name.lower()] = district.osm_id

            org_district_boundaries_data[state.osm_id] = state_district_data

        setattr(self, cache_attr, (org_state_boundaries_data, org_district_boundaries_data))
        return org_state_boundaries_data, org_district_boundaries_data

    def get_contact_fields(self, org):
        cache_attr = '__contact_fields__%d' % org.pk
        if hasattr(self, cache_attr):
            return getattr(self, cache_attr)
        contact_fields = ContactField.objects.filter(org=org)
        contact_fields_data = {elt.label.lower(): elt.key for elt in contact_fields}

        setattr(self, cache_attr, contact_fields_data)
        return contact_fields_data

    def local_kwargs(self, org, remote):
        from ureport.utils import json_date_to_datetime

        reporter_group = org.get_config('reporter_group')
        contact_groups_names = [group.name.lower() for group in remote.groups]

        if not reporter_group.lower() in contact_groups_names:
            return None

        org_state_boundaries_data, org_district_boundaries_data = self.get_boundaries_data(org)
        contact_fields = self.get_contact_fields(org)

        state = ''
        district = ''

        state_field = org.get_config('state_label')
        if state_field:
            state_field = state_field.lower()
            if org.get_config('is_global'):
                state_name = remote.fields.get(contact_fields.get(state_field), None)
                if state_name:
                    state = state_name

            else:
                state_name = remote.fields.get(contact_fields.get(state_field), None)
                if state_name:
                    state_name = state_name.lower()
                    state = org_state_boundaries_data.get(state_name, '')

                district_field = org.get_config('district_label')
                if district_field:
                    district_field = district_field.lower()
                    district_name = remote.fields.get(contact_fields.get(district_field), None)
                    if district_name:
                        district_name = district_name.lower()
                        district = org_district_boundaries_data.get(state, dict()).get(district_name, '')

        registered_on = None
        registration_field = org.get_config('registration_label')
        if registration_field:
            registration_field = registration_field.lower()
            registered_on = remote.fields.get(contact_fields.get(registration_field), None)
            if registered_on:
                registered_on = json_date_to_datetime(registered_on)

        occupation = ''
        occupation_field = org.get_config('occupation_label')
        if occupation_field:
            occupation_field = occupation_field.lower()
            occupation = remote.fields.get(contact_fields.get(occupation_field), '')
            if not occupation:
                occupation = ''

        born = 0
        born_field = org.get_config('born_label')
        if born_field:
            born_field = born_field.lower()
            try:
                born = int(remote.fields.get(contact_fields.get(born_field), 0))

                # support only positive django integer field valid values
                if born < 0 or born > 2147483647:
                    born = 0

            except ValueError:
                pass
            except TypeError:
                pass

        gender = ''
        gender_field = org.get_config('gender_label')
        female_label = org.get_config('female_label')
        male_label = org.get_config('male_label')

        if gender_field:
            gender_field = gender_field.lower()
            gender = remote.fields.get(contact_fields.get(gender_field), '')

            if gender and gender.lower() == female_label.lower():
                gender = self.model.FEMALE
            elif gender and gender.lower() == male_label.lower():
                gender = self.model.MALE
            else:
                gender = ''

        return {
            'org': org,
            'uuid': remote.uuid,
            'gender': gender,
            'born': born,
            'occupation': occupation,
            'registered_on': registered_on,
            'state': state,
            'district': district
        }

    def update_required(self, local, remote, local_kwargs):
        if not local_kwargs:
            return True

        update = local.gender != local_kwargs['gender'] or local.born != local_kwargs['born']
        update = update or local.occupation != local_kwargs['occupation']
        update = update or local.registered_on != local_kwargs['registered_on']
        update = update or local.state != local_kwargs['state'] or local.district != local_kwargs['district']
        return update

class RapidProBackend(BaseBackend):
    """
    RapidPro instance as a backend
    """
    @staticmethod
    def _get_client(org, api_version):
        return org.get_temba_client(api_version=api_version)

    def pull_fields(self, org):
        client = self._get_client(org, 2)
        incoming_objects = client.get_fields().all(retry_on_rate_exceed=True)

        return sync_local_to_set(org, FieldSyncer(), incoming_objects)

    def pull_boundaries(self, org):

        if org.get_config('is_global'):
            incoming_objects = Boundary.build_global_boundaries()
        else:
            client = self._get_client(org, 1)
            incoming_objects = client.get_boundaries()

        return sync_local_to_set(org, BoundarySyncer(), incoming_objects)

    def pull_contacts(self, org, modified_after, modified_before, progress_callback=None):
        client = self._get_client(org, 2)

        # all contacts created or modified in RapidPro in the time window
        active_query = client.get_contacts(after=modified_after, before=modified_before)
        fetches = active_query.iterfetches(retry_on_rate_exceed=True)

        # all contacts deleted in RapidPro in the same time window
        deleted_query = client.get_contacts(deleted=True, after=modified_after, before=modified_before)
        deleted_fetches = deleted_query.iterfetches(retry_on_rate_exceed=True)

        return sync_local_to_changes(org, ContactSyncer(), fetches, deleted_fetches, progress_callback)

    def pull_results(self, poll, modified_after, modified_before, progress_callback=None):
        org = poll.org
        client = self._get_client(org, 2)

        start = time.time()
        print "Start fetching runs for poll #%d on org #%d" % (poll.pk, org.pk)

        # ignore the TaskState time and use the time we stored in redis
        now = timezone.now()
        after = cache.get(PollResult.POLL_RESULTS_LAST_PULL_CACHE_KEY % (org.pk, poll.pk), None)

        poll_runs_query = client.get_runs(flow=poll.flow_uuid, responded=True, after=after, before=now)
        fetches = poll_runs_query.iterfetches(retry_on_rate_exceed=True)

        num_created = 0
        num_updated = 0
        num_ignored = 0
        num_synced = 0

        existing_poll_results = PollResult.objects.filter(flow=poll.flow_uuid, org=poll.org_id)

        poll_results_map = dict()
        for res in existing_poll_results:
            if res.contact not in poll_results_map:
                poll_results_map[res.contact] = {res.ruleset: res}
            else:
                poll_results_map[res.contact][res.ruleset] = res

        fetch_start = time.time()
        for fetch in fetches:

            print "RapidPro API fetch for poll #%d on org #%d %d - %d took %ds" % (poll.pk, org.pk, num_synced,
                                                                                num_synced + len(fetch),
                                                                                time.time() - fetch_start)

            local_sync_start = time.time()

            contact_uuids = [run.contact.uuid for run in fetch]
            contacts = Contact.objects.filter(org=org, uuid__in=contact_uuids)
            contacts_map = {c.uuid: c for c in contacts}

            new_poll_results = []

            for temba_run in fetch:
                flow_uuid = temba_run.flow.uuid
                contact_uuid = temba_run.contact.uuid
                completed = temba_run.exit_type == 'completed'

                contact_obj = contacts_map.get(contact_uuid, None)

                state = ''
                district = ''
                if contact_obj is not None:
                    state = contact_obj.state
                    district = contact_obj.district

                for temba_step in temba_run.steps:
                    ruleset_uuid = temba_step.node
                    category = temba_step.category
                    text = temba_step.text

                    existing_poll_result = poll_results_map.get(contact_uuid, dict()).get(ruleset_uuid, None)

                    if existing_poll_result is not None:

                        update_required = existing_poll_result.category != category or existing_poll_result.text != text
                        update_required = update_required or existing_poll_result.state != state
                        update_required = update_required or existing_poll_result.district != district
                        update_required = update_required or existing_poll_result.completed != completed

                        if update_required:
                            PollResult.objects.filter(pk=existing_poll_result.pk).update(category=category, text=text,
                                                                                state=state, district=district,
                                                                                completed=completed)

                            num_updated += 1
                        else:
                            num_ignored += 1

                    else:
                        new_poll_results.append(PollResult(org=org, flow=flow_uuid, ruleset=ruleset_uuid,
                                                           contact=contact_uuid, category=category, text=text,
                                                           state=state, date=temba_step.arrived_on,
                                                           district=district, completed=completed))

                        num_created += 1

            PollResult.objects.bulk_create(new_poll_results)

            num_synced += len(fetch)
            if progress_callback:
                progress_callback(num_synced)

            print "Local sync ops for poll #%d on org #%d %d - %d took %ds" % (poll.pk, org.pk, num_synced - len(fetch),
                                                                               num_synced,
                                                                               time.time() - local_sync_start)
            print "Total synced for poll #%d on org #%d %d runs in %ds" % (poll.pk, org.pk, num_synced,
                                                                           time.time() - start)
            fetch_start = time.time()
            print "=" * 40

        # update the time for this poll from which we fetch next time
        cache.set(PollResult.POLL_RESULTS_LAST_PULL_CACHE_KEY % (org.pk, poll.pk),
                  datetime_to_json_date(now.replace(tzinfo=pytz.utc)), None)

        # from django.db import connection as db_connection, reset_queries
        # slowest_queries = sorted(db_connection.queries, key=lambda q: q['time'], reverse=True)[:10]
        # for q in slowest_queries:
        #     print "=" * 60
        #     print "\n\n\n"
        #     print "%s -- %s" % (q['time'], q['sql'])
        # reset_queries()

        print "Finished pulling results for poll #%d on org #%d runs in %ds, " \
              "created %d, updated %d, ignored %d" % (poll.pk, org.pk, time.time() - start, num_created,
                                                      num_updated, num_ignored)
        return num_created, num_updated, num_ignored<|endoftext|>


<commit_before>'''
Created on 31 mars 2015

@author: Remi Cattiau
'''
import unittest
import os
import sys
import nxdrive
from nxdrive.engine.dao.sqlite import EngineDAO
import tempfile

class EngineDAOTest(unittest.TestCase):

    def _get_default_db(self, name='test_engine.db'):
        nxdrive_path = os.path.dirname(nxdrive.__file__)
        return os.path.join(nxdrive_path, 'tests', 'resources', name)

    def get_db_temp_file(self):
        tmp_db = tempfile.NamedTemporaryFile(suffix="test_db")
        if sys.platform == 'win32':
            tmp_db.close()
        return tmp_db

    def setUp(self):
        tmp_db = self.get_db_temp_file()
        db = open(self._get_default_db(), 'rb')
        with open(tmp_db.name, 'wb') as f:
            f.write(db.read())
        self._dao = EngineDAO(tmp_db.name)

    def tearDown(self):
        self._dao.dispose()

    def test_init_db(self):
        init_db = self.get_db_temp_file()
        dao = EngineDAO(init_db.name)
        # Test filters table
        self.assertEquals(0, len(dao.get_filters()))
        # Test state table
        self.assertEquals(0, len(dao.get_conflicts()))
        # Test configuration
        self.assertIsNone(dao.get_config("remote_user"))
        # Test RemoteScan table
        self.assertFalse(dao.is_path_scanned("/"))

    def test_migration_db_v1(self):
        init_db = self.get_db_temp_file()
        # Test empty db
        dao = EngineDAO(init_db.name)
        # Test a non empty db
        migrate_db = self.get_db_temp_file()
        db = open(self._get_default_db('test_engine_migration.db'), 'rb')
        with open(migrate_db.name, 'wb') as f:
            f.write(db.read())
        self._dao = EngineDAO(migrate_db.name)
        c = self._dao._get_read_connection().cursor()
        cols = c.execute("PRAGMA table_info('States')").fetchall()
        self.assertEquals(len(cols), 30)
        self.test_batch_folder_files()
        self.test_batch_upload_files()
        self.test_conflicts()
        self.test_errors()
        self.test_acquire_processors()
        self.test_configuration()

    def test_conflicts(self):
        self.assertEquals(self._dao.get_conflict_count(), 3)
        self.assertEquals(len(self._dao.get_conflicts()), 3)

    def test_errors(self):
        self.assertEquals(self._dao.get_error_count(), 1)
        self.assertEquals(self._dao.get_error_count(5), 0)
        self.assertEquals(len(self._dao.get_errors()), 1)
        row = self._dao.get_errors()[0]
        # Test reset error
        self._dao.reset_error(row)
        self.assertEquals(self._dao.get_error_count(), 0)
        row = self._dao.get_state_from_id(row.id)
        self.assertIsNone(row.last_error)
        self.assertIsNone(row.last_error_details)
        self.assertEqual(row.error_count, 0)
        # Test increase
        self._dao.increase_error(row, "Test")
        self.assertEquals(self._dao.get_error_count(), 0)
        self._dao.increase_error(row, "Test 2")
        self.assertEquals(self._dao.get_error_count(), 0)
        self.assertEquals(self._dao.get_error_count(1), 1)
        self._dao.increase_error(row, "Test 3")
        self.assertEquals(self._dao.get_error_count(), 0)
        self.assertEquals(self._dao.get_error_count(2), 1)
        # Synchronize with wrong version should fail
        self.assertFalse(self._dao.synchronize_state(row, row.version-1))
        self.assertEquals(self._dao.get_error_count(2), 1)
        # Synchronize should reset error
        self.assertTrue(self._dao.synchronize_state(row, row.version))
        self.assertEquals(self._dao.get_error_count(2), 0)

    def test_remote_scans(self):
        self.assertFalse(self._dao.is_path_scanned("/"))
        self._dao.add_path_scanned("/Test")
        self.assertTrue(self._dao.is_path_scanned("/Test"))
        self.assertFalse(self._dao.is_path_scanned("/Test2"))
        self._dao.clean_scanned()
        self.assertFalse(self._dao.is_path_scanned("/Test"))

    def test_last_sync(self):
        # Based only on file so not showing 2
        ids = [58, 8, 62, 61, 60]
        files = self._dao.get_last_files(5)
        self.assertEquals(len(files), 5)
        for i in range(5):
            self.assertEquals(files[i].id, ids[i])
        ids = [58, 62, 61, 60, 63]
        files = self._dao.get_last_files(5, "remote")
        self.assertEquals(len(files), 5)
        for i in range(5):
            self.assertEquals(files[i].id, ids[i])
        ids = [8, 11, 5]
        files = self._dao.get_last_files(5, "local")
        self.assertEquals(len(files), 3)
        for i in range(3):
            self.assertEquals(files[i].id, ids[i])

    def test_batch_folder_files(self):
        # Verify that the batch is ok
        ids = range(25, 47)
        index = 0
        state = self._dao.get_state_from_id(25) #ids[index])
        while index < len(ids)-1:
            index = index + 1
            state = self._dao.get_next_folder_file(state.remote_ref)
            self.assertEquals(state.id, ids[index])
        while index > 0:
            index = index - 1
            state = self._dao.get_previous_folder_file(state.remote_ref)
            self.assertEquals(state.id, ids[index])
        self.assertIsNone(self._dao.get_previous_folder_file(state.remote_ref))
        # Last file is 9
        state = self._dao.get_state_from_id(46)
        self.assertIsNone(self._dao.get_next_folder_file(state.remote_ref))

    def test_batch_upload_files(self):
        # Verify that the batch is ok
        ids = [58, 62, 61, 60, 63]
        index = 0
        state = self._dao.get_state_from_id(ids[index])
        while index < len(ids)-1:
            index = index + 1
            state = self._dao.get_next_upload_file(state.remote_ref)
            self.assertEquals(state.id, ids[index])
        while index > 0:
            index = index - 1
            state = self._dao.get_previous_upload_file(state.remote_ref)
            self.assertEquals(state.id, ids[index])
        self.assertIsNone(self._dao.get_previous_upload_file(state.remote_ref))
        # Last file is 9
        state = self._dao.get_state_from_id(9)
        self.assertIsNone(self._dao.get_next_upload_file(state.remote_ref))

    def test_reinit_processors(self):
        state = self._dao.get_state_from_id(1)
        self.assertEquals(state.processor, 0)

    def test_acquire_processors(self):
        self.assertTrue(self._dao.acquire_processor(666, 2))
        self.assertFalse(self._dao.acquire_processor(666, 2))
        self._dao.release_processor(666)
        self.assertTrue(self._dao.acquire_processor(666, 2))
        row = self._dao.get_state_from_id(2)
        # Check the auto-release
        self._dao.synchronize_state(row)
        self.assertTrue(self._dao.acquire_processor(666, 2))

    def test_configuration(self):
        result = self._dao.get_config("empty", "DefaultValue")
        self.assertEquals(result, "DefaultValue")
        result = self._dao.get_config("remote_user", "DefaultValue")
        self.assertEquals(result, "Administrator")
        self._dao.update_config("empty", "notAnymore")
        result = self._dao.get_config("empty", "DefaultValue")
        self.assertNotEquals(result, "DefaultValue")
        self._dao.update_config("remote_user", "Test")
        result = self._dao.get_config("remote_user", "DefaultValue")
        self.assertEquals(result, "Test")
        self._dao.update_config("empty", None)
        result = self._dao.get_config("empty", "DefaultValue")
        self.assertEquals(result, "DefaultValue")
        result = self._dao.get_config("empty")
        self.assertEquals(result, None)

    def test_filters(self):
        # Contains by default /fakeFilter/Test_Parent and /fakeFilter/Retest
        self.assertEquals(len(self._dao.get_filters()), 2)
        self._dao.remove_filter(u"/fakeFilter/Retest")
        self.assertEquals(len(self._dao.get_filters()), 1)
        self._dao.add_filter(u"/fakeFilter")
        # Should delete the subchild filter
        self.assertEquals(len(self._dao.get_filters()), 1)
        self._dao.add_filter(u"/otherFilter")
        self.assertEquals(len(self._dao.get_filters()), 2)
<commit_msg>NXDRIVE-290: Add back test of engine init on non existing file<commit_after>'''
Created on 31 mars 2015

@author: Remi Cattiau
'''
import unittest
import os
import sys
import nxdrive
from nxdrive.engine.dao.sqlite import EngineDAO
import tempfile

class EngineDAOTest(unittest.TestCase):

    def _get_default_db(self, name='test_engine.db'):
        nxdrive_path = os.path.dirname(nxdrive.__file__)
        return os.path.join(nxdrive_path, 'tests', 'resources', name)

    def get_db_temp_file(self):
        tmp_db = tempfile.NamedTemporaryFile(suffix="test_db")
        if sys.platform == 'win32':
            tmp_db.close()
        return tmp_db

    def setUp(self):
        tmp_db = self.get_db_temp_file()
        db = open(self._get_default_db(), 'rb')
        with open(tmp_db.name, 'wb') as f:
            f.write(db.read())
        self._dao = EngineDAO(tmp_db.name)

    def tearDown(self):
        self._dao.dispose()

    def test_init_db(self):
        init_db = self.get_db_temp_file()
        if sys.platform != 'win32':
            os.remove(init_db.name)
        dao = EngineDAO(init_db.name)
        # Test filters table
        self.assertEquals(0, len(dao.get_filters()))
        # Test state table
        self.assertEquals(0, len(dao.get_conflicts()))
        # Test configuration
        self.assertIsNone(dao.get_config("remote_user"))
        # Test RemoteScan table
        self.assertFalse(dao.is_path_scanned("/"))

    def test_migration_db_v1(self):
        init_db = self.get_db_temp_file()
        # Test empty db
        dao = EngineDAO(init_db.name)
        # Test a non empty db
        migrate_db = self.get_db_temp_file()
        db = open(self._get_default_db('test_engine_migration.db'), 'rb')
        with open(migrate_db.name, 'wb') as f:
            f.write(db.read())
        self._dao = EngineDAO(migrate_db.name)
        c = self._dao._get_read_connection().cursor()
        cols = c.execute("PRAGMA table_info('States')").fetchall()
        self.assertEquals(len(cols), 30)
        self.test_batch_folder_files()
        self.test_batch_upload_files()
        self.test_conflicts()
        self.test_errors()
        self.test_acquire_processors()
        self.test_configuration()

    def test_conflicts(self):
        self.assertEquals(self._dao.get_conflict_count(), 3)
        self.assertEquals(len(self._dao.get_conflicts()), 3)

    def test_errors(self):
        self.assertEquals(self._dao.get_error_count(), 1)
        self.assertEquals(self._dao.get_error_count(5), 0)
        self.assertEquals(len(self._dao.get_errors()), 1)
        row = self._dao.get_errors()[0]
        # Test reset error
        self._dao.reset_error(row)
        self.assertEquals(self._dao.get_error_count(), 0)
        row = self._dao.get_state_from_id(row.id)
        self.assertIsNone(row.last_error)
        self.assertIsNone(row.last_error_details)
        self.assertEqual(row.error_count, 0)
        # Test increase
        self._dao.increase_error(row, "Test")
        self.assertEquals(self._dao.get_error_count(), 0)
        self._dao.increase_error(row, "Test 2")
        self.assertEquals(self._dao.get_error_count(), 0)
        self.assertEquals(self._dao.get_error_count(1), 1)
        self._dao.increase_error(row, "Test 3")
        self.assertEquals(self._dao.get_error_count(), 0)
        self.assertEquals(self._dao.get_error_count(2), 1)
        # Synchronize with wrong version should fail
        self.assertFalse(self._dao.synchronize_state(row, row.version-1))
        self.assertEquals(self._dao.get_error_count(2), 1)
        # Synchronize should reset error
        self.assertTrue(self._dao.synchronize_state(row, row.version))
        self.assertEquals(self._dao.get_error_count(2), 0)

    def test_remote_scans(self):
        self.assertFalse(self._dao.is_path_scanned("/"))
        self._dao.add_path_scanned("/Test")
        self.assertTrue(self._dao.is_path_scanned("/Test"))
        self.assertFalse(self._dao.is_path_scanned("/Test2"))
        self._dao.clean_scanned()
        self.assertFalse(self._dao.is_path_scanned("/Test"))

    def test_last_sync(self):
        # Based only on file so not showing 2
        ids = [58, 8, 62, 61, 60]
        files = self._dao.get_last_files(5)
        self.assertEquals(len(files), 5)
        for i in range(5):
            self.assertEquals(files[i].id, ids[i])
        ids = [58, 62, 61, 60, 63]
        files = self._dao.get_last_files(5, "remote")
        self.assertEquals(len(files), 5)
        for i in range(5):
            self.assertEquals(files[i].id, ids[i])
        ids = [8, 11, 5]
        files = self._dao.get_last_files(5, "local")
        self.assertEquals(len(files), 3)
        for i in range(3):
            self.assertEquals(files[i].id, ids[i])

    def test_batch_folder_files(self):
        # Verify that the batch is ok
        ids = range(25, 47)
        index = 0
        state = self._dao.get_state_from_id(25) #ids[index])
        while index < len(ids)-1:
            index = index + 1
            state = self._dao.get_next_folder_file(state.remote_ref)
            self.assertEquals(state.id, ids[index])
        while index > 0:
            index = index - 1
            state = self._dao.get_previous_folder_file(state.remote_ref)
            self.assertEquals(state.id, ids[index])
        self.assertIsNone(self._dao.get_previous_folder_file(state.remote_ref))
        # Last file is 9
        state = self._dao.get_state_from_id(46)
        self.assertIsNone(self._dao.get_next_folder_file(state.remote_ref))

    def test_batch_upload_files(self):
        # Verify that the batch is ok
        ids = [58, 62, 61, 60, 63]
        index = 0
        state = self._dao.get_state_from_id(ids[index])
        while index < len(ids)-1:
            index = index + 1
            state = self._dao.get_next_upload_file(state.remote_ref)
            self.assertEquals(state.id, ids[index])
        while index > 0:
            index = index - 1
            state = self._dao.get_previous_upload_file(state.remote_ref)
            self.assertEquals(state.id, ids[index])
        self.assertIsNone(self._dao.get_previous_upload_file(state.remote_ref))
        # Last file is 9
        state = self._dao.get_state_from_id(9)
        self.assertIsNone(self._dao.get_next_upload_file(state.remote_ref))

    def test_reinit_processors(self):
        state = self._dao.get_state_from_id(1)
        self.assertEquals(state.processor, 0)

    def test_acquire_processors(self):
        self.assertTrue(self._dao.acquire_processor(666, 2))
        self.assertFalse(self._dao.acquire_processor(666, 2))
        self._dao.release_processor(666)
        self.assertTrue(self._dao.acquire_processor(666, 2))
        row = self._dao.get_state_from_id(2)
        # Check the auto-release
        self._dao.synchronize_state(row)
        self.assertTrue(self._dao.acquire_processor(666, 2))

    def test_configuration(self):
        result = self._dao.get_config("empty", "DefaultValue")
        self.assertEquals(result, "DefaultValue")
        result = self._dao.get_config("remote_user", "DefaultValue")
        self.assertEquals(result, "Administrator")
        self._dao.update_config("empty", "notAnymore")
        result = self._dao.get_config("empty", "DefaultValue")
        self.assertNotEquals(result, "DefaultValue")
        self._dao.update_config("remote_user", "Test")
        result = self._dao.get_config("remote_user", "DefaultValue")
        self.assertEquals(result, "Test")
        self._dao.update_config("empty", None)
        result = self._dao.get_config("empty", "DefaultValue")
        self.assertEquals(result, "DefaultValue")
        result = self._dao.get_config("empty")
        self.assertEquals(result, None)

    def test_filters(self):
        # Contains by default /fakeFilter/Test_Parent and /fakeFilter/Retest
        self.assertEquals(len(self._dao.get_filters()), 2)
        self._dao.remove_filter(u"/fakeFilter/Retest")
        self.assertEquals(len(self._dao.get_filters()), 1)
        self._dao.add_filter(u"/fakeFilter")
        # Should delete the subchild filter
        self.assertEquals(len(self._dao.get_filters()), 1)
        self._dao.add_filter(u"/otherFilter")
        self.assertEquals(len(self._dao.get_filters()), 2)
<|endoftext|>


<commit_before>import time
import pkg_resources
import asyncio
import gochans as gc
import client as cli
from client import close, error, bpwait, bpresume, sent, stop, rmv

udb = {}

def get_udb (keys):
    return cli.get_db(udb, keys)

def update_udb (keys, val):
    cli.update_db(udb, keys, val)

kwdone = cli.keyword('done')

dsdict = lambda dict, *keys: list((dict[key] for key in keys))

def usermsg (op, payload):
    print("UOP:", op, "\nUPAYLOAD", payload)

def dispatcher (ch, op, payload):
    ## print("DISPATCH:", op, payload)

    if op == cli.msg or op == 'msg':
        ws, appmsg = dsdict(payload, 'ws', 'data')
        mop = cli.get_msg_op(appmsg)
        mpload = cli.get_msg_payload(appmsg)
        if mop == kwdone or mop == 'done':
          ## print("close_connection and exit ...")
          closech = get_udb(['close', 'chan'])
          gc.go(closech.send, 'stop')
        else:
          update_udb([ws, 'lastrcv'], appmsg)
          update_udb([ws, 'rcvcnt'], get_udb([ws, 'rcvcnt'])+1)
          usermsg(mop, mpload)
    elif op == cli.sent:
        ws, msg = dsdict(payload, 'ws', cli.msg)
        print("CLIENT, Sent msg ", msg)
        update_udb([ws, 'lastsnt'], msg)
        update_udb([ws, 'sntcnt'], get_udb([ws, 'sntcnt'])+1)
    elif op == cli.open:
        ws = payload
        ##print("CLIENT :open/ws = ", ws)
        update_udb([ws], {'chan': ch, 'rcvcnt': 0, 'sntcnt': 0, 'errcnt': 0})
        update_udb(['com'], [ch, ws])
    elif op == close:
        ws, code, reason = dsdict(payload, 'ws', 'code', 'reason')
        print("CLIENT RMTclose/payload = ", payload)
        go(ch.send, {cli.op: stop,
                     cli.payload: {'ws': ws, 'cause': 'rmtclose'}})
    elif op == error:
        ws, err = dsdict(payload, 'ws', 'err')
        print("CLIENT :error/payload = ", payload)
        update_udb([ws, 'errcnt'], get_udb([ws, 'errcnt'])+1)
    elif op == bpwait:
        ws, msg, encode = dsdict(payload, 'ws', cli.msg, 'encode')
        print("CLIENT, Waiting to send msg ", msg)
        time.sleep(2)
        print("CLIENT, Trying resend ...")
        cli.send_msg(ws, msg, encode=encode)
    elif op == bpresume:
        print("CLIENT, BP Resume ", payload)
    elif op == stop:
        ws, cause = dsdict(payload, 'ws', 'cause')
        print("CLIENT, Stopping reads... Cause ", cause)
        cli.close_connection(ws)
        update_udb([ws], rmv)
    else:
        print("CLIENT :WTF/op = ", op, " payload = ", payload)

# 'ws://localhost:8765/ws'
def startit (url):
    close_chan = gc.Chan(size=2)
    update_udb(['close', 'chan'], close_chan)
    ch = cli.open_connection(url)
    cli.gorun(ch, dispatcher)

@asyncio.coroutine
def waitclose ():
  closech = get_udb(['close', 'chan'])
  x = yield from closech.recv()
  return x

import threading
import atexit
cloop = asyncio.get_event_loop()
atexit.register(cloop.close)
threading.Thread(name='closeloop-thread', target=cloop.run_forever).start()

def main():
  ## print("Hello from Aerobio Python")
  ## print(pkg_resources.resource_string(
    ##__name__, "resources/usage.txt").decode("utf-8").strip())
  ## print("connecting to 7070")
  startit('ws://localhost:7070/ws')
  ## closech = get_udb(['close', 'chan'])
  print("go(closech.recv)")
  ## time.sleep(.3)
  ## f = gc.go(closech.recv)
  ## x = f.result()
  f = asyncio.run_coroutine_threadsafe(waitclose(), cloop)
  x = f.result()
  print("closech result: ", x)
  ch, ws = get_udb(['com'])
  cli.close_connection(ws)

if __name__ == "__main__":
  main()

# from client import get_db, cli_db, msgsnt, msgrcv
# ex.startit('ws://localhost:8765/ws')
# ch,ws = ex.udb['com']
# for i in range(95): cli.send_msg(ws, {'op': "msg", 'payload': 'testing'})
# [ex.get_udb([ws, 'sntcnt']), ex.get_udb([ws, 'rcvcnt'])]
# [get_db(cli_db, [ws, msgsnt]), get_db(cli_db, [ws, msgrcv])]

## cd aerobio/
## zip -r ../aerobio.zip *
## cd ..
## echo "#\!/usr/bin/env python37" | cat - aerobio.zip > aerobiov2
<commit_msg>Move to trio based communications<commit_after>import os
import getopt, sys
import pkg_resources
import time
import trio
import client as cli
from client import close, error, bpwait, bpresume, sent, stop, rmv

udb = {}

def get_udb (keys):
    return cli.get_db(udb, keys)

def update_udb (keys, val):
    cli.update_db(udb, keys, val)

dsdict = lambda dict, *keys: list((dict[key] for key in keys))

def appmsg (ws, data):
    op, payload = dsdict(data, cli.op, cli.payload)
    if op == cli.keyword('validate'):
        print("")
        print(payload)
    elif op == cli.keyword('jobinfo'):
        print(payload)
    elif op == cli.keyword('register'):
        update_udb(['register'], payload)
    else:
        print("UOP:", op, "\nUPAYLOAD", payload)

def dispatcher (ch, op, payload):
    #print("DISPATCH:", op, payload)

    if op == cli.msg or op == 'msg':
        ws, data = dsdict(payload, 'ws', 'data')
        #print('CLIENT :msg/payload = ', payload)
        update_udb([ws, 'lastrcv'], data)
        update_udb([ws, 'rcvcnt'], get_udb([ws, 'rcvcnt'])+1)
        appmsg(ws, data)
    elif op == cli.sent:
        ws, msg = dsdict(payload, 'ws', cli.msg)
        #print("CLIENT, Sent msg ", msg)
        update_udb([ws, 'lastsnt'], msg)
        update_udb([ws, 'sntcnt'], get_udb([ws, 'sntcnt'])+1)
    elif op == cli.open:
        ws = payload
        #print("CLIENT :open/ws = ", ws)
        update_udb([ws], {'chan': ch, 'rcvcnt': 0, 'sntcnt': 0, 'errcnt': 0})
        update_udb(['com'], [ch, ws])
    elif op == close:
        ws, code, reason = dsdict(payload, 'ws', 'code', 'reason')
        print("CLIENT RMTclose/payload = ", payload)
    elif op == error:
        ws, err = dsdict(payload, 'ws', 'err')
        print("CLIENT :error/payload = ", payload)
        update_udb([ws, 'errcnt'], get_udb([ws, 'errcnt'])+1)
    elif op == bpwait:
        ws, msg, encode = dsdict(payload, 'ws', cli.msg, 'encode')
        print("CLIENT, Waiting to send msg ", msg)
        v = udb["bpwait"] if "bpwait" in udb else []
        v.append([ws, msg, encode])
        update_udb(["bpwait"], v)
        time.sleep(0.1)
    elif op == bpresume:
        print("CLIENT, BP Resume ", payload)
        update_udb(["resume"], payload)
    elif op == stop:
        ws, cause = dsdict(payload, 'ws', 'cause')
        print("CLIENT, Stopping reads... Cause ", cause)
        update_udb([ws], rmv)
    else:
        print("CLIENT :WTF/op = ", op, " payload = ", payload)

def getarg (arglist):
    arg = arglist[0]
    arglist = arglist[1:]
    return (arg, arglist)

def args2map ():
    user = os.environ['USER']
    argmap = {'user': user}
    fullArgs = sys.argv
    arglist = fullArgs[1:]
    (cmd,arglist) = getarg(arglist)
    argmap['cmd'] = cli.keyword(cmd)
    if cmd == 'run':
        (phase,arglist) = getarg(arglist)
        argmap['phase'] = phase
        (x, arglist) = getarg(arglist)
        if (x == 'replicates') or (x == 'combined'):
            argmap['modifier'] = x
        else:
            argmap['eid'] = x
        if not 'eid' in argmap:
            (eid, arglist) = getarg(arglist)
            argmap['eid'] = eid
        if not 'modifier' in argmap:
            argmap['modifier'] = 'replicates'
    elif (cmd == 'compare') or (cmd == 'xcompare') or (cmd == 'aggregate'):
        (compfile, arglist) = getarg(arglist)
        argmap['compfile'] = compfile
        (eid, arglist) = getarg(arglist)
        argmap['eid'] = eid
    return argmap

async def bpretry ():
    return "bpwait" in udb and udb["bpwait"]

async def resume ():
    while "resume" not in udb:
        await trio.sleep(0.1)
    return True

async def command (info):
    while not 'com' in udb:
        time.sleep(0.1)
    ws = info["ws"]
    argmap = info["appinfo"]["argmap"]
    await cli.send_msg(
        ws, {cli.op: argmap['cmd'], cli.keyword('data'): argmap})

def main():
    ## print("Hello from Aerobio Python")
    ## print(pkg_resources.resource_string(
    ##__name__, "resources/usage.txt").decode("utf-8").strip())
    ## print("connecting to 7070")
    argmap = args2map()
    #print("ArgMap:", argmap)
    url = 'ws://localhost:7070/ws'
    trio.run(cli.open_connection,
             url, dispatcher, command, {"argmap": argmap})

if __name__ == "__main__":
    main()

# from client import get_db, cli_db, msgsnt, msgrcv
# ex.startit('ws://localhost:8765/ws')
# ch,ws = ex.udb['com']
# for i in range(95): cli.send_msg(ws, {'op': "msg", 'payload': 'testing'})
# [ex.get_udb([ws, 'sntcnt']), ex.get_udb([ws, 'rcvcnt'])]
# [get_db(cli_db, [ws, msgsnt]), get_db(cli_db, [ws, msgrcv])]

## cd aerobio/
## zip -r ../aerobio.zip *
## cd ..
## echo "#\!/usr/bin/env python37" | cat - aerobio.zip > aerobiov2
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from spam_lists import (
    GoogleSafeBrowsing, HpHosts, GeneralizedURLTester, URLTesterChain,
    SPAMHAUS_DBL, SPAMHAUS_ZEN, SURBL_MULTI, SortedHostCollection
)
from wtforms.validators import ValidationError

from . import app

hp_hosts = HpHosts('url-shortener')
google_safe_browsing = GoogleSafeBrowsing(
    'url-shortener',
    '0.9',
    app.config['GOOGLE_SAFE_BROWSING_API_KEY']
)

spam_tester = GeneralizedURLTester(
    URLTesterChain(
        SPAMHAUS_DBL,
        SPAMHAUS_ZEN,
        SURBL_MULTI,
        hp_hosts,
        google_safe_browsing
    )
)

filename = app.config['HOST_BLACKLIST_FILE']
hosts = []
if filename is not None:
    with open(filename) as f:
        hosts = f.read().splitlines()

blacklist_tester = GeneralizedURLTester(
    SortedHostCollection(
        'blacklist',
        'blacklisted',
        hosts
    )
)

def get_msg_if_blacklisted_or_spam(url):
    """ Get a message if given URL has blacklisted host
    or is recognized as spam

    :param url: a URL value to be tested
    :returns: a string message if any of the testers recognize
    the URL as match, or None
    """
    msg_map = {
        blacklist_tester: 'The host of target URL is blacklisted.',
        spam_tester: 'The URL has been recognized as spam.'
    }

    for tester, msg in msg_map.items():
        if tester.any_match([url]):
            return msg

def not_blacklisted_nor_spam(form, field):
    """ Check if the data in the field is not
    blacklisted nor a spam

    This function is a custom WTForms field validator using
    get_msg_if_blacklisted_or_spam for validating data in the field.
    If the message returned for the data is not None, the function
    raises ValidationError with the message.

    :param form: a form whose field is to be validated
    :param field: a field containing data to be validated
    :raises ValidationError: if the function returns a message
    for the data
    """
    msg = get_msg_if_blacklisted_or_spam(field.data)
    if msg is not None:
        raise ValidationError(msg)

def sorted_host_list_from_file(name, classification, filename):
    """Create a sorted host list based on contents of a file

    :param name: name of a host list to be created
    :param classification: a string describing classification of all
    items stored by the host list
    :param filename: name of a file from which hosts fill be
    extracted. Each line of the file must contain one IP address
    or a hostname.
    :returns: a host list as an instance of SortedHostCollection
    """
    hosts = []
    with open(filename) as f:
        hosts = f.read().splitlines()
    host_list = SortedHostCollection(name, classification, [])
    for item in hosts:
        host_list.add(item)

    return host_list

class BlacklistValidator(object):
    """A URL spam detector using configurable blacklists

    :ivar _msg_map: a dictionary mapping blacklists used by an instance
    of the class to validation messages associated with them
    """

    def __init__(self, composite_blacklist, default_message):
        """ Initialize a new instance

        :param composite_blacklist: an object representing multiple
        blacklist used by an instance of this class

        It must have lookup_matching(urls) method accepting a sequence
        of URLs and returning an iterator returning other objects, each
        with a source property storing one of the blacklists used by
        the validator - the one containing the item that matches one
        of the URLs.

        It must also have a url_tester property with another property:
        url_testers. The last property represents a chain of blacklist
        objects used by the composite. This property must have
        insert(index, item) method, where :
        * index is a position at which a new blacklist object is
        inserted
        * object is a blacklist object to be inserted
        :param default_message: a default validation message to be provided
        when a URL matches a blacklist that does not have its specific
        validation message
        """
        self._composite_blacklist = composite_blacklist
        self._msg_map = {}
        self.default_message = default_message

    def prepend(self, blacklist, message=None):
        """Prepend an object representing a blacklist to
        the URL tester chain

        :param blacklist: represents a blacklist to be used to
        recognize spam URLs. This object must have
        lookup_matching(urls) method
        :param message: a custom, blacklist-specific validation message
        to be associated with the blacklist, or None if the default
        message is to be associated with the blacklist
        """
        self._composite_blacklist.url_tester.url_testers.insert(0, blacklist)
        if message is not None:
            self._msg_map[blacklist] = message

    def get_msg_if_blacklisted(self, url):
        """ Get a message if URL or one of its redirect addresses
        is blacklisted

        :param url: a URL address as a string
        :returns: a string message if the URL or its redirect addresses
        match content of any of the blacklists, or None
        """
        for match in self._composite_blacklist.lookup_matching([url]):
            return self._msg_map.get(match.source, self.default_message)

    def assert_not_blacklisted(self, form, field):
        """Assert the URL value from the field is not blacklisted

        This method is a custom WTForms field validator.

        :param form: a form whose field is to be validated
        :param field: a field containing data to be validated
        :raises ValidationError: if the function returns a message
        for the data
        """
        msg = self.get_msg_if_blacklisted(field.data)
        if msg is not None:
            raise ValidationError(msg)

url_validator = BlacklistValidator(
    GeneralizedURLTester(
        URLTesterChain(SURBL_MULTI, SPAMHAUS_ZEN, SPAMHAUS_DBL, hp_hosts)
    ),
    'The URL has been recognized as spam.'
)
<commit_msg>Add a before_app_run handler configuring URL validation<commit_after># -*- coding: utf-8 -*-
from spam_lists import (
    GoogleSafeBrowsing, HpHosts, GeneralizedURLTester, URLTesterChain,
    SPAMHAUS_DBL, SPAMHAUS_ZEN, SURBL_MULTI, SortedHostCollection
)
from wtforms.validators import ValidationError

from . import app, before_app_run, __version__, __title__

hp_hosts = HpHosts('url-shortener')
google_safe_browsing = GoogleSafeBrowsing(
    'url-shortener',
    '0.9',
    app.config['GOOGLE_SAFE_BROWSING_API_KEY']
)

spam_tester = GeneralizedURLTester(
    URLTesterChain(
        SPAMHAUS_DBL,
        SPAMHAUS_ZEN,
        SURBL_MULTI,
        hp_hosts,
        google_safe_browsing
    )
)

filename = app.config['HOST_BLACKLIST_FILE']
hosts = []
if filename is not None:
    with open(filename) as f:
        hosts = f.read().splitlines()

blacklist_tester = GeneralizedURLTester(
    SortedHostCollection(
        'blacklist',
        'blacklisted',
        hosts
    )
)

def get_msg_if_blacklisted_or_spam(url):
    """ Get a message if given URL has blacklisted host
    or is recognized as spam

    :param url: a URL value to be tested
    :returns: a string message if any of the testers recognize
    the URL as match, or None
    """
    msg_map = {
        blacklist_tester: 'The host of target URL is blacklisted.',
        spam_tester: 'The URL has been recognized as spam.'
    }

    for tester, msg in msg_map.items():
        if tester.any_match([url]):
            return msg

def not_blacklisted_nor_spam(form, field):
    """ Check if the data in the field is not
    blacklisted nor a spam

    This function is a custom WTForms field validator using
    get_msg_if_blacklisted_or_spam for validating data in the field.
    If the message returned for the data is not None, the function
    raises ValidationError with the message.

    :param form: a form whose field is to be validated
    :param field: a field containing data to be validated
    :raises ValidationError: if the function returns a message
    for the data
    """
    msg = get_msg_if_blacklisted_or_spam(field.data)
    if msg is not None:
        raise ValidationError(msg)

def sorted_host_list_from_file(name, classification, filename):
    """Create a sorted host list based on contents of a file

    :param name: name of a host list to be created
    :param classification: a string describing classification of all
    items stored by the host list
    :param filename: name of a file from which hosts fill be
    extracted. Each line of the file must contain one IP address
    or a hostname.
    :returns: a host list as an instance of SortedHostCollection
    """
    hosts = []
    with open(filename) as f:
        hosts = f.read().splitlines()
    host_list = SortedHostCollection(name, classification, [])
    for item in hosts:
        host_list.add(item)

    return host_list

class BlacklistValidator(object):
    """A URL spam detector using configurable blacklists

    :ivar _msg_map: a dictionary mapping blacklists used by an instance
    of the class to validation messages associated with them
    """

    def __init__(self, composite_blacklist, default_message):
        """ Initialize a new instance

        :param composite_blacklist: an object representing multiple
        blacklist used by an instance of this class

        It must have lookup_matching(urls) method accepting a sequence
        of URLs and returning an iterator returning other objects, each
        with a source property storing one of the blacklists used by
        the validator - the one containing the item that matches one
        of the URLs.

        It must also have a url_tester property with another property:
        url_testers. The last property represents a chain of blacklist
        objects used by the composite. This property must have
        insert(index, item) method, where :
        * index is a position at which a new blacklist object is
        inserted
        * object is a blacklist object to be inserted
        :param default_message: a default validation message to be provided
        when a URL matches a blacklist that does not have its specific
        validation message
        """
        self._composite_blacklist = composite_blacklist
        self._msg_map = {}
        self.default_message = default_message

    def prepend(self, blacklist, message=None):
        """Prepend an object representing a blacklist to
        the URL tester chain

        :param blacklist: represents a blacklist to be used to
        recognize spam URLs. This object must have
        lookup_matching(urls) method
        :param message: a custom, blacklist-specific validation message
        to be associated with the blacklist, or None if the default
        message is to be associated with the blacklist
        """
        self._composite_blacklist.url_tester.url_testers.insert(0, blacklist)
        if message is not None:
            self._msg_map[blacklist] = message

    def get_msg_if_blacklisted(self, url):
        """ Get a message if URL or one of its redirect addresses
        is blacklisted

        :param url: a URL address as a string
        :returns: a string message if the URL or its redirect addresses
        match content of any of the blacklists, or None
        """
        for match in self._composite_blacklist.lookup_matching([url]):
            return self._msg_map.get(match.source, self.default_message)

    def assert_not_blacklisted(self, form, field):
        """Assert the URL value from the field is not blacklisted

        This method is a custom WTForms field validator.

        :param form: a form whose field is to be validated
        :param field: a field containing data to be validated
        :raises ValidationError: if the function returns a message
        for the data
        """
        msg = self.get_msg_if_blacklisted(field.data)
        if msg is not None:
            raise ValidationError(msg)

url_validator = BlacklistValidator(
    GeneralizedURLTester(
        URLTesterChain(SURBL_MULTI, SPAMHAUS_ZEN, SPAMHAUS_DBL, hp_hosts)
    ),
    'The URL has been recognized as spam.'
)

@before_app_run.connect_via(app)
def configure_url_validator(sender, **kwargs):
    app.logger.info('Configuring URL validation...')

    gsb_api_key = sender.config['GOOGLE_SAFE_BROWSING_API_KEY']
    url_validator.prepend(
        GoogleSafeBrowsing(__title__, __version__, gsb_api_key)
    )
    app.logger.info('Google Safe Browsing API client loaded.')

    filename = sender.config['HOST_BLACKLIST_FILE']
    if filename is not None:
        url_validator.prepend(
            sorted_host_list_from_file('blacklist', 'blacklisted', filename),
            'The host of target URL is blacklisted.'
        )
        msg = 'Custom host blacklist loaded from {}.'.format(filename)
        app.logger.info(msg)
<|endoftext|>


<commit_before>import hashlib
import logging
from django.core.cache import caches

logger = logging.getLogger(__name__)

# stringify function args
def join_(*args):
    rv = ""
    for _string in args:
        rv += ' ' + _string
    return rv.lstrip()

# get cache key for storage
def cache_get_key(*args, **kwargs):
    serialise = []
    for arg in args:
        serialise.append(str(arg))
    for key, arg in kwargs.items():
        serialise.append(str(key))
        serialise.append(str(arg))
    key = hashlib.md5("".join(serialise).encode('utf-8')).hexdigest()
    return key

# decorator for caching functions
def cache_for(cache_label, time=None):
    def decorator(fn):
        def wrapper(*args, **kwargs):
            cache = caches[cache_label]
            key = cache_get_key(fn.__name__, *args, **kwargs)
            result = cache.get(key)
            if not result:
                result = fn(*args, **kwargs)
                cache.set(key, result, time)
                print('Cache {} set {}'.format(cache_label, join_(*args)))
            else:
                print('Cache {} hit {}'.format(cache_label, join_(*args)))
            return result

        return wrapper

    return decorator
<commit_msg>add default timeout option to force default django settings for timeout<commit_after>import hashlib
import logging
from django.core.cache import caches

logger = logging.getLogger(__name__)

# stringify function args
def join_(*args):
    rv = ""
    for _string in args:
        rv += ' ' + _string
    return rv.lstrip()

# get cache key for storage
def cache_get_key(*args, **kwargs):
    serialise = []
    for arg in args:
        serialise.append(str(arg))
    for key, arg in kwargs.items():
        serialise.append(str(key))
        serialise.append(str(arg))
    key = hashlib.md5("".join(serialise).encode('utf-8')).hexdigest()
    return key

# decorator for caching functions
def cache_for(cache_label, time=None, default_time=False):
    def decorator(fn):
        def wrapper(*args, **kwargs):
            cache = caches[cache_label]
            key = cache_get_key(fn.__name__, *args, **kwargs)
            result = cache.get(key)
            if not result:
                result = fn(*args, **kwargs)
                cache.set(key, result, time) if default_time is False else cache.set(key, result)
                print('Cache {} set {}'.format(cache_label, join_(*args)))
            else:
                print('Cache {} hit {}'.format(cache_label, join_(*args)))
            return result

        return wrapper

    return decorator
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
import datetime
import os
import sys
import zephyr
import random

if __name__ == '__main__':
    cur_file = os.path.abspath(__file__)
    django_dir = os.path.abspath(os.path.join(os.path.dirname(cur_file), '..'))
    sys.path.append(django_dir)
    os.environ['DJANGO_SETTINGS_MODULE'] = 'zscore.settings'

import django.contrib.auth.models
from django.core.exceptions import ValidationError

import sleep.models

def setup():
    zephyr.init()
    subs = zephyr.Subscriptions()
    subs.add(('message', '*', '%me%'))

def fetch_user_from_zgram(zgram):
    if not zgram.auth:
        print "  -> Rejecting: unauthenticated"
        raise LookupError("You aren't authenticated.")
    user, at, domain = zgram.sender.partition('@')
    if domain == 'ATHENA.MIT.EDU':
        email = "%s@mit.edu" % (user, )
        try:
            user = django.contrib.auth.models.User.objects.get(email=email)
            return user
        except django.contrib.auth.models.User.DoesNotExist:
            raise LookupError("No user with email %s" % (email, ))
    else:
        raise LookupError("Don't know about the domain %s" % (domain, ))

def build_reply(zgram):
    z = zephyr.ZNotice()
    z.cls = zgram.cls
    z.instance = zgram.instance
    z.recipient = zgram.sender
    z.opcode = 'auto'
    z.fields = ['zscore bot: http://zscore.mit.edu/', '']
    return z

def handle_zgram(zgram):
    reply = build_reply(zgram)
    try:
        user = fetch_user_from_zgram(zgram)
        if 'gnight' in zgram.message or 'goodnight' in zgram.message:
            success = sleep.models.PartialSleep.create_new_for_user(user)
            if success:
                msg = 'Sleep well!'
            else:
                msg = "Hmm, I can't seem to record you as asleep."
        elif 'awake' in zgram.message or "mornin'" in zgram.message:
            try:
                s = sleep.models.PartialSleep.finish_for_user(user)
                tmpl = "Good morning!\nYou slept from %s to %s.\n(That's %s.)"
                msg = tmpl % (s.start_time, s.end_time, s.length())
            except sleep.models.PartialSleep.DoesNotExist:
                msg = "Sorry, you don't seem to have been asleep."
            except ValidationError as e:
                msg = e.messages[0]
        elif 'help' in zgram.message:
            msg = "Hi! I'm the zscore zephyrbot.\n\nMessage me 'gnight' when you want to sleep\nand 'awake' when you wake up in the morning"
        elif 'meow' in zgram.message.lower():
            msg = random.choice(['meow!', "purrrr", "*barks*"])
        else:
            msg = "I'm sorry -- I don't understand."
    except LookupError as e:
        msg = "I'm sorry -- I can't find an account for you: %s" % (e.message, )
    reply.fields[1] = msg
    reply.send()

def main():
    setup()
    print "Waiting..."
    while True:
        zgram = zephyr.receive(True)
        if not zgram or zgram.opcode == 'PING':
            continue
        if zgram.opcode.lower() == 'kill':
            sys.exit(0)

        print '%s: [%s] -c %s -i "%s": %s -> %s' % (
            datetime.datetime.now(),
            zgram.opcode,
            zgram.cls, zgram.instance,
            zgram.sender, zgram.recipient,
        )
        if zgram.opcode.lower() == 'auto':
            print "  -> auto; ignoring"
            continue

        handle_zgram(zgram)

if __name__ == '__main__':
    main()
<commit_msg>Zephyrbot now listens on -c zscore<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
import datetime
import os
import sys
import zephyr
import random

if __name__ == '__main__':
    cur_file = os.path.abspath(__file__)
    django_dir = os.path.abspath(os.path.join(os.path.dirname(cur_file), '..'))
    sys.path.append(django_dir)
    os.environ['DJANGO_SETTINGS_MODULE'] = 'zscore.settings'

import django.contrib.auth.models
from django.core.exceptions import ValidationError

import sleep.models

def setup():
    zephyr.init()
    subs = zephyr.Subscriptions()
    subs.add(('message', '*', '%me%'), ('zscore', '*', '*')

def fetch_user_from_zgram(zgram):
    if not zgram.auth:
        print "  -> Rejecting: unauthenticated"
        raise LookupError("You aren't authenticated.")
    user, at, domain = zgram.sender.partition('@')
    if domain == 'ATHENA.MIT.EDU':
        email = "%s@mit.edu" % (user, )
        try:
            user = django.contrib.auth.models.User.objects.get(email=email)
            return user
        except django.contrib.auth.models.User.DoesNotExist:
            raise LookupError("No user with email %s" % (email, ))
    else:
        raise LookupError("Don't know about the domain %s" % (domain, ))

def build_reply(zgram):
    z = zephyr.ZNotice()
    z.cls = zgram.cls
    z.instance = zgram.instance
    z.recipient = zgram.sender
    z.opcode = 'auto'
    z.fields = ['zscore bot: http://zscore.mit.edu/', '']
    return z

def handle_zgram(zgram):
    reply = build_reply(zgram)
    if z.cls == 'zscore':
        pass
    else:
        try:
            user = fetch_user_from_zgram(zgram)
            if 'gnight' in zgram.message or 'goodnight' in zgram.message:
                success = sleep.models.PartialSleep.create_new_for_user(user)
                if success:
                    msg = 'Sleep well!'
                else:
                    msg = "Hmm, I can't seem to record you as asleep."
            elif 'awake' in zgram.message or "mornin'" in zgram.message:
                try:
                    s = sleep.models.PartialSleep.finish_for_user(user)
                    tmpl = "Good morning!\nYou slept from %s to %s.\n(That's %s.)"
                    msg = tmpl % (s.start_time, s.end_time, s.length())
                except sleep.models.PartialSleep.DoesNotExist:
                    msg = "Sorry, you don't seem to have been asleep."
                except ValidationError as e:
                    msg = e.messages[0]
            elif 'help' in zgram.message:
                msg = "Hi! I'm the zscore zephyrbot.\n\nMessage me 'gnight' when you want to sleep\nand 'awake' when you wake up in the morning"
            elif 'meow' in zgram.message.lower():
                msg = random.choice(['meow!', "purrrr", "*barks*"])
            else:
                msg = "I'm sorry -- I don't understand."
        except LookupError as e:
            msg = "I'm sorry -- I can't find an account for you: %s" % (e.message, )
    reply.fields[1] = msg
    reply.send()

def main():
    setup()
    print "Waiting..."
    while True:
        zgram = zephyr.receive(True)
        if not zgram or zgram.opcode == 'PING':
            continue
        if zgram.opcode.lower() == 'kill':
            sys.exit(0)

        print '%s: [%s] -c %s -i "%s": %s -> %s' % (
            datetime.datetime.now(),
            zgram.opcode,
            zgram.cls, zgram.instance,
            zgram.sender, zgram.recipient,
        )
        if zgram.opcode.lower() == 'auto':
            print "  -> auto; ignoring"
            continue

        handle_zgram(zgram)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
import datetime
import os
import sys
import zephyr
import random

if __name__ == '__main__':
    cur_file = os.path.abspath(__file__)
    django_dir = os.path.abspath(os.path.join(os.path.dirname(cur_file), '..'))
    sys.path.append(django_dir)
    os.environ['DJANGO_SETTINGS_MODULE'] = 'zscore.settings'

import django.contrib.auth.models
from django.core.exceptions import ValidationError

import sleep.models
import sleep.utils

def setup():
    zephyr.init()
    subs = zephyr.Subscriptions()
    subs.add(('message', '*', '%me%'))
    subs.add(('zscore', '*', '*'))

def fetch_user_from_zgram(zgram):
    if not zgram.auth:
        print "  -> Rejecting: unauthenticated"
        raise LookupError("You aren't authenticated.")
    user, at, domain = zgram.sender.partition('@')
    if domain == 'ATHENA.MIT.EDU':
        email = "%s@mit.edu" % (user, )
        try:
            user = django.contrib.auth.models.User.objects.get(email=email)
            return user
        except django.contrib.auth.models.User.DoesNotExist:
            raise LookupError("No user with email %s" % (email, ))
    else:
        raise LookupError("Don't know about the domain %s" % (domain, ))

def build_reply(zgram):
    z = zephyr.ZNotice()
    z.cls = zgram.cls
    z.instance = zgram.instance
    z.recipient = zgram.sender
    z.opcode = 'auto'
    z.fields = ['zscore bot: http://zscore.mit.edu/', '']
    return z

def handle_zgram(zgram):
    if zgram.cls == 'zscore':
        pass
    else:
        reply = build_reply(zgram)
        try:
            user = fetch_user_from_zgram(zgram)
            if 'gnight' in zgram.message or 'goodnight' in zgram.message:
                success = sleep.models.PartialSleep.create_new_for_user(user)
                if success:
                    msg = 'Sleep well!'
                else:
                    msg = "Hmm, I can't seem to record you as asleep."
            elif 'awake' in zgram.message or "mornin'" in zgram.message:
                try:
                    s = sleep.models.PartialSleep.finish_for_user(user)
                    tmpl = "Good morning!\nYou slept from %s to %s.\n(That's %s.)\nEdit: https://zscore.mit.edu/sleep/edit/%s/"
                    msg = tmpl % (s.start_time, s.end_time, s.length(), s.id)
                except sleep.models.PartialSleep.DoesNotExist:
                    msg = "Sorry, you don't seem to have been asleep."
                except ValidationError as e:
                    msg = e.messages[0]
            elif 'help' in zgram.message:
                msg = "Hi! I'm the zscore zephyrbot.\n\nMessage me 'gnight' when you want to sleep\nand 'awake' when you wake up in the morning.\n\nFor stats, message me 'stats'"
            elif 'meow' in zgram.message.lower():
                msg = random.choice(['meow!', "purrrr", "*barks*"])
            elif 'stats' in zgram.message:
                try:
                    sleeper = sleep.models.Sleeper.objects.get(pk = user.pk)
                    usermetrics = sleeper.sleeperprofile.metrics.all()
                    msglist = []
                    if 'global' in zgram.message or 'all-time' in zgram.message:
                        msglist.append("Your All-Time stats\n" + sleep.utils.zephyrDisplay(sleeper.movingStats(), usermetrics))
                    if 'decaying' in zgram.message or (len(msglist)==0):
                        msglist.append("Your Stats (exponential decay)\n" + sleep.utils.zephyrDisplay(sleeper.decayStats(), usermetrics))
                    msg = '\n\n'.join(msglist)
                except all as e:
                    print e
                    msg = 'Something went wrong! Zephyr -c zscore for help!'
            elif 'bug' in zgram.message or 'bug' in zgram.instance:
                msg = """Sorry! daemon/zscore does not accept bug requests\nor feature reports! Feel free to message us on -c zscore,\nemail us at zscore@mit.edu, or create a new issue at\nhttps://github.com/sleepers-anonymous/zscore/issues/new"""
            else:
                msg = "I'm sorry -- I don't understand. Message me 'help' for more information."
        except LookupError as e:
            msg = "I'm sorry -- I can't find an account for you: %s" % (e.message, )
        reply.fields[1] = msg
        reply.send()

def main():
    setup()
    print "Waiting..."
    while True:
        zgram = zephyr.receive(True)
        if not zgram or zgram.opcode == 'PING':
            continue
        if zgram.opcode.lower() == 'kill':
            sys.exit(0)

        print '%s: [%s] -c %s -i "%s": %s -> %s' % (
            datetime.datetime.now(),
            zgram.opcode,
            zgram.cls, zgram.instance,
            zgram.sender, zgram.recipient,
        )
        if zgram.opcode.lower() == 'auto':
            print "  -> auto; ignoring"
            continue

        handle_zgram(zgram)

if __name__ == '__main__':
    main()
<commit_msg>Add leaderboard command to zephyrbot<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
import datetime
import os
import sys
import zephyr
import random

if __name__ == '__main__':
    cur_file = os.path.abspath(__file__)
    django_dir = os.path.abspath(os.path.join(os.path.dirname(cur_file), '..'))
    sys.path.append(django_dir)
    os.environ['DJANGO_SETTINGS_MODULE'] = 'zscore.settings'

import django.contrib.auth.models
from django.core.exceptions import ValidationError

import sleep.models
import sleep.utils

def setup():
    zephyr.init()
    subs = zephyr.Subscriptions()
    subs.add(('message', '*', '%me%'))
    subs.add(('zscore', '*', '*'))

def fetch_user_from_zgram(zgram):
    if not zgram.auth:
        print "  -> Rejecting: unauthenticated"
        raise LookupError("You aren't authenticated.")
    user, at, domain = zgram.sender.partition('@')
    if domain == 'ATHENA.MIT.EDU':
        email = "%s@mit.edu" % (user, )
        try:
            user = django.contrib.auth.models.User.objects.get(email=email)
            return user
        except django.contrib.auth.models.User.DoesNotExist:
            raise LookupError("No user with email %s" % (email, ))
    else:
        raise LookupError("Don't know about the domain %s" % (domain, ))

def build_reply(zgram):
    z = zephyr.ZNotice()
    z.cls = zgram.cls
    z.instance = zgram.instance
    z.recipient = zgram.sender
    z.opcode = 'auto'
    z.fields = ['zscore bot: http://zscore.mit.edu/', '']
    return z

def handle_zgram(zgram):
    if zgram.cls == 'zscore':
        pass
    else:
        reply = build_reply(zgram)
        try:
            user = fetch_user_from_zgram(zgram)
            if 'gnight' in zgram.message or 'goodnight' in zgram.message:
                success = sleep.models.PartialSleep.create_new_for_user(user)
                if success:
                    msg = 'Sleep well!'
                else:
                    msg = "Hmm, I can't seem to record you as asleep."
            elif 'awake' in zgram.message or "mornin'" in zgram.message:
                try:
                    s = sleep.models.PartialSleep.finish_for_user(user)
                    tmpl = "Good morning!\nYou slept from %s to %s.\n(That's %s.)\nEdit: https://zscore.mit.edu/sleep/edit/%s/"
                    msg = tmpl % (s.start_time, s.end_time, s.length(), s.id)
                except sleep.models.PartialSleep.DoesNotExist:
                    msg = "Sorry, you don't seem to have been asleep."
                except ValidationError as e:
                    msg = e.messages[0]
            elif 'help' in zgram.message:
                msg = "Hi! I'm the zscore zephyrbot.\n\nMessage me 'gnight' when you want to sleep\nand 'awake' when you wake up in the morning.\n\nFor stats, message me 'stats'.\nFor the leaderboard, message me 'leaderboard'."
            elif 'meow' in zgram.message.lower():
                msg = random.choice(['meow!', "purrrr", "*barks*"])
            elif 'stats' in zgram.message:
                try:
                    sleeper = sleep.models.Sleeper.objects.get(pk = user.pk)
                    usermetrics = sleeper.sleeperprofile.metrics.all()
                    msglist = []
                    if 'global' in zgram.message or 'all-time' in zgram.message:
                        msglist.append("Your All-Time stats\n" + sleep.utils.zephyrDisplay(sleeper.movingStats(), usermetrics))
                    if 'decaying' in zgram.message or (len(msglist)==0):
                        msglist.append("Your Stats (exponential decay)\n" + sleep.utils.zephyrDisplay(sleeper.decayStats(), usermetrics))
                    msg = '\n\n'.join(msglist)
                except Exception as e:
                    print e
                    msg = 'Something went wrong! Zephyr -c zscore for help!'
            elif 'leaderboard' in zgram.message:
                try:
                    sleeper = sleep.models.Sleeper.objects.get(pk = user.pk)
                    usermetrics = sleeper.sleeperprofile.metrics.all()
                    ss = sleep.models.Sleeper.objects.sorted_sleepers(user = user)
                    top = [ s for s in ss if s['rank'] <= 10 or s['user'].pk == user.pk ]
                    msg = '\n'.join(str(s['rank']).rjust(3) + ' ' + s['user'].username[:12].ljust(12) + ' ' + str(s['zScore']) for s in top)
                except Exception as e:
                    print e
                    msg = 'Something went wrong! Zephyr -c zscore for help!'
            elif 'bug' in zgram.message or 'bug' in zgram.instance:
                msg = """Sorry! daemon/zscore does not accept bug requests\nor feature reports! Feel free to message us on -c zscore,\nemail us at zscore@mit.edu, or create a new issue at\nhttps://github.com/sleepers-anonymous/zscore/issues/new"""
            else:
                msg = "I'm sorry -- I don't understand. Message me 'help' for more information."
        except LookupError as e:
            msg = "I'm sorry -- I can't find an account for you: %s" % (e.message, )
        reply.fields[1] = msg
        reply.send()

def main():
    setup()
    print "Waiting..."
    while True:
        zgram = zephyr.receive(True)
        if not zgram or zgram.opcode == 'PING':
            continue
        if zgram.opcode.lower() == 'kill':
            sys.exit(0)

        print '%s: [%s] -c %s -i "%s": %s -> %s' % (
            datetime.datetime.now(),
            zgram.opcode,
            zgram.cls, zgram.instance,
            zgram.sender, zgram.recipient,
        )
        if zgram.opcode.lower() == 'auto':
            print "  -> auto; ignoring"
            continue

        handle_zgram(zgram)

if __name__ == '__main__':
    main()
<|endoftext|>