<commit_before>from datetime import timedelta, datetime
import itertools
import random

from django.core.management.base import BaseCommand
from django_countries import countries as Countries
from faker import Faker

from workshops.models import (
    Airport,
    Role,
    Tag,
    Badge,
    Lesson,
    Person,
    Award,
    Qualification,
    Organization,
    Event,
    Task,
)

class Command(BaseCommand):
    help = 'Add fake data to the database.'

    def add_arguments(self, parser):
        parser.add_argument(
            '--seed', action='store', default=None,
            help='Provide an initial seed for randomization mechanism.',
        )

    def fake_airports(self, faker, count=5):
        """Add some airports."""
        # we're not doing anything here, since:
        # 1. data migrations add some airports already
        # 2. we'll have more airports as fixtures as of #626
        pass

    def fake_roles(self, faker):
        """Provide fixed roles (before they end up in fixtures, see #626)."""
        roles = [
            ('organizer', 'Workshop organizer'),
            ('learner', 'Learner'),
            ('host', 'Workshop host'),
            ('instructor', 'Instructor'),
            ('helper', 'Helper'),
        ]
        for name, verbose_name in roles:
            Role.objects.create(name=name, verbose_name=verbose_name)

    def fake_tags(self, faker):
        """Provide fixed tags (before they end up in fixtures, see #626)."""
        tags = [
            ('SWC', 'Software Carpentry Workshop'),
            ('DC', 'Data Carpentry Workshop'),
            ('LC', 'Library Carpentry Workshop'),
            ('WiSE', 'Women in Science and Engineering'),
            ('TTT', 'Train the Trainers'),
        ]
        for tag, details in tags:
            Tag.objects.create(name=tag, details=details)

    def fake_badges(self, faker):
        """Provide fixed badges (before they end up in fixtures, see #626)."""
        # 4 badges are already in the migrations: swc-instructor,
        # dc-instructor, maintainer, and trainer
        badges = [
            ('creator', 'Creator',
             'Creating learning materials and other content'),
            ('member', 'Member', 'Software Carpentry Foundation member'),
            ('organizer', 'Organizer',
             'Organizing workshops and learning groups'),
        ]
        for name, title, criteria in badges:
            Badge.objects.create(name=name, title=title, criteria=criteria)

    def fake_instructors(self, faker, count=5, add_badge=True,
                         add_qualifications=True):
        """Add a few people with random instructor badge, random airport, and
        random qualification."""
        airports = list(Airport.objects.all())
        badges = list(Badge.objects.instructor_badges())
        lessons = list(Lesson.objects.all())
        for i in range(count):
            user_name = faker.user_name()
            emails = [faker.email(), faker.safe_email(), faker.free_email(),
                      faker.company_email()]
            gender = random.choice(Person.GENDER_CHOICES)[0]
            if gender == 'F':
                name = faker.first_name_female()
                last_name = faker.last_name_female()
            elif gender == 'M':
                name = faker.first_name_male()
                last_name = faker.last_name_male()
            else:
                name, last_name = faker.first_name(), faker.last_name()

            person = Person.objects.create(
                personal=name,
                family=last_name,
                email=random.choice(emails),
                gender=gender,
                may_contact=random.choice([True, False]),
                airport=random.choice(airports),
                github=user_name,
                twitter=user_name,
                url=faker.url(),
                username=user_name,
            )

            if add_badge:
                Award.objects.create(
                    person=person, badge=random.choice(badges),
                    awarded=faker.date_time_this_year(before_now=True,
                                                      after_now=False).date(),
                )
            if add_qualifications:
                for lesson in random.sample(lessons, 4):
                    Qualification.objects.create(person=person, lesson=lesson)

    def fake_noninstructors(self, faker, count=5):
        """Add a few people who aren't instructors."""
        return self.fake_instructors(
            faker=faker, count=count, add_badge=False,
            add_qualifications=False,
        )

    def fake_organizations(self, faker, count=5):
        """Add some organizations that host events."""
        countries = list(Countries)
        for i in range(count):
            Organization.objects.create(
                domain=faker.domain_name(),
                fullname=faker.company(),
                country=random.choice(countries)[0],
            )

    def fake_current_events(self, faker, count=5):
        """Ongoing and upcoming events."""
        twodays = timedelta(days=2)
        organizations = list(Organization.objects.exclude(domain='self-organized'))
        countries = list(Countries)

        for i in range(count):
            start = faker.date_time_this_year(before_now=False,
                                              after_now=True).date()
            Event.objects.create(
                slug='{:%Y-%m-%d}-{}'.format(start, faker.slug()),
                start=start,
                end=start + twodays,
                url=faker.url(),
                host=random.choice(organizations),
                # needed in order for event to be published
                country=random.choice(countries)[0],
                venue=faker.word().title(),
                address=faker.sentence(nb_words=4, variable_nb_words=True),
                latitude=random.uniform(-90, 90),
                longitude=random.uniform(0, 180),
            )

    def fake_uninvoiced_events(self, faker, count=5):
        """Preferably in the past, and with 'uninvoiced' status."""
        twodays = timedelta(days=2)
        countries = list(Countries)
        organizations = list(Organization.objects.exclude(domain='self-organized'))

        for i in range(count):
            start = faker.date_time_this_year(before_now=True,
                                              after_now=False).date()
            Event.objects.create(
                slug='{:%Y-%m-%d}-{}'.format(start, faker.slug()),
                start=start,
                end=start + twodays,
                url=faker.url(),
                host=random.choice(organizations),
                # needed in order for event to be published
                country=random.choice(countries)[0],
                venue=faker.word().title(),
                address=faker.sentence(nb_words=4, variable_nb_words=True),
                latitude=random.uniform(-90, 90),
                longitude=random.uniform(0, 180),
                # needed in order for event to be uninvoiced
                invoice_status='not-invoiced',
            )

    def fake_unpublished_events(self, faker, count=5):
        """Events with missing location data (which is required for publishing
        them)."""
        twodays = timedelta(days=2)
        organizations = list(Organization.objects.exclude(domain='self-organized'))

        for i in range(count):
            start = faker.date_time_this_year(before_now=True,
                                              after_now=True).date()
            Event.objects.create(
                slug='{:%Y-%m-%d}-{}'.format(start, faker.slug()),
                start=start,
                end=start + twodays,
                url=faker.url(),
                host=random.choice(organizations),
            )

    def fake_self_organized_events(self, faker, count=5):
        """Full-blown events with 'self-organized' host."""
        twodays = timedelta(days=2)
        self_organized = Organization.objects.get(domain='self-organized')
        countries = list(Countries)
        invoice_statuses = Event.INVOICED_CHOICES

        for i in range(count):
            start = faker.date_time_this_year(before_now=True,
                                              after_now=True).date()
            Event.objects.create(
                slug='{:%Y-%m-%d}-{}'.format(start, faker.slug()),
                start=start,
                end=start + twodays,
                url=faker.url(),
                host=self_organized,
                # needed in order for event to be published
                country=random.choice(countries)[0],
                venue=faker.word().title(),
                address=faker.sentence(nb_words=4, variable_nb_words=True),
                latitude=random.uniform(-90, 90),
                longitude=random.uniform(0, 180),
                # needed in order for event to be uninvoiced
                invoice_status=random.choice(invoice_statuses)[0],
            )

    def fake_tasks(self, faker, count=50):
        events = Event.objects.all()
        persons = Person.objects.all()
        roles = Role.objects.all()
        all_possible = itertools.product(events, persons, roles)

        for event, person, role in random.sample(list(all_possible), count):
            Task.objects.create(event=event, person=person, role=role)

    def fake_trainees(self, faker):
        host = Organization.objects.all()[0]
        tag = Tag.objects.get(name='TTT')
        learner = Role.objects.get(name='learner')
        swc_instructor = Badge.objects.get(name='swc-instructor')
        dc_instructor = Badge.objects.get(name='dc-instructor')

        # events
        first_event = Event.objects.create(host=host, slug='first-ttt-event')
        first_event.tags.add(tag)
        first_event.save()

        second_event = Event.objects.create(host=host, slug='second-ttt-event')
        second_event.tags.add(tag)
        second_event.save()

        third_event = Event.objects.create(host=host, slug='third-ttt-event')
        third_event.tags.add(tag)
        third_event.save()

        # persons
        bob = Person.objects.create(username='bob-trainee',
                                    personal='Bob', family='Smith')
        alice = Person.objects.create(username='alice-trainee',
                                      personal='Alice', family='Smith')
        john = Person.objects.create(username='john-trainee',
                                     personal='Alice', family='Smith')

        # tasks
        Task.objects.create(event=first_event, person=bob, role=learner)
        Task.objects.create(event=first_event, person=alice, role=learner)
        Task.objects.create(event=second_event, person=john, role=learner)

        # awards
        Award.objects.create(event=first_event, person=bob, badge=swc_instructor,
                             awarded=datetime(2016, 4, 1))
        Award.objects.create(event=third_event, person=bob, badge=dc_instructor,
                             awarded=datetime(2016, 4, 1))

    def handle(self, *args, **options):
        faker = Faker()

        seed = options['seed']
        if seed is not None:
            faker.seed(seed)

        self.fake_airports(faker)
        self.fake_roles(faker)
        self.fake_tags(faker)
        self.fake_badges(faker)
        self.fake_instructors(faker)
        self.fake_noninstructors(faker)
        self.fake_organizations(faker)
        self.fake_current_events(faker)
        self.fake_uninvoiced_events(faker)
        self.fake_unpublished_events(faker)
        self.fake_self_organized_events(faker)
        self.fake_tasks(faker)
        self.fake_trainees(faker)
<commit_msg>Populate title and URL of Task in dev database<commit_after>from datetime import timedelta, datetime
import itertools
import random

from django.core.management.base import BaseCommand
from django_countries import countries as Countries
from faker import Faker

from workshops.models import (
    Airport,
    Role,
    Tag,
    Badge,
    Lesson,
    Person,
    Award,
    Qualification,
    Organization,
    Event,
    Task,
)

class Command(BaseCommand):
    help = 'Add fake data to the database.'

    def add_arguments(self, parser):
        parser.add_argument(
            '--seed', action='store', default=None,
            help='Provide an initial seed for randomization mechanism.',
        )

    def fake_airports(self, faker, count=5):
        """Add some airports."""
        # we're not doing anything here, since:
        # 1. data migrations add some airports already
        # 2. we'll have more airports as fixtures as of #626
        pass

    def fake_roles(self, faker):
        """Provide fixed roles (before they end up in fixtures, see #626)."""
        roles = [
            ('organizer', 'Workshop organizer'),
            ('learner', 'Learner'),
            ('host', 'Workshop host'),
            ('instructor', 'Instructor'),
            ('helper', 'Helper'),
        ]
        for name, verbose_name in roles:
            Role.objects.create(name=name, verbose_name=verbose_name)

    def fake_tags(self, faker):
        """Provide fixed tags (before they end up in fixtures, see #626)."""
        tags = [
            ('SWC', 'Software Carpentry Workshop'),
            ('DC', 'Data Carpentry Workshop'),
            ('LC', 'Library Carpentry Workshop'),
            ('WiSE', 'Women in Science and Engineering'),
            ('TTT', 'Train the Trainers'),
        ]
        for tag, details in tags:
            Tag.objects.create(name=tag, details=details)

    def fake_badges(self, faker):
        """Provide fixed badges (before they end up in fixtures, see #626)."""
        # 4 badges are already in the migrations: swc-instructor,
        # dc-instructor, maintainer, and trainer
        badges = [
            ('creator', 'Creator',
             'Creating learning materials and other content'),
            ('member', 'Member', 'Software Carpentry Foundation member'),
            ('organizer', 'Organizer',
             'Organizing workshops and learning groups'),
        ]
        for name, title, criteria in badges:
            Badge.objects.create(name=name, title=title, criteria=criteria)

    def fake_instructors(self, faker, count=5, add_badge=True,
                         add_qualifications=True):
        """Add a few people with random instructor badge, random airport, and
        random qualification."""
        airports = list(Airport.objects.all())
        badges = list(Badge.objects.instructor_badges())
        lessons = list(Lesson.objects.all())
        for i in range(count):
            user_name = faker.user_name()
            emails = [faker.email(), faker.safe_email(), faker.free_email(),
                      faker.company_email()]
            gender = random.choice(Person.GENDER_CHOICES)[0]
            if gender == 'F':
                name = faker.first_name_female()
                last_name = faker.last_name_female()
            elif gender == 'M':
                name = faker.first_name_male()
                last_name = faker.last_name_male()
            else:
                name, last_name = faker.first_name(), faker.last_name()

            person = Person.objects.create(
                personal=name,
                family=last_name,
                email=random.choice(emails),
                gender=gender,
                may_contact=random.choice([True, False]),
                airport=random.choice(airports),
                github=user_name,
                twitter=user_name,
                url=faker.url(),
                username=user_name,
            )

            if add_badge:
                Award.objects.create(
                    person=person, badge=random.choice(badges),
                    awarded=faker.date_time_this_year(before_now=True,
                                                      after_now=False).date(),
                )
            if add_qualifications:
                for lesson in random.sample(lessons, 4):
                    Qualification.objects.create(person=person, lesson=lesson)

    def fake_noninstructors(self, faker, count=5):
        """Add a few people who aren't instructors."""
        return self.fake_instructors(
            faker=faker, count=count, add_badge=False,
            add_qualifications=False,
        )

    def fake_organizations(self, faker, count=5):
        """Add some organizations that host events."""
        countries = list(Countries)
        for i in range(count):
            Organization.objects.create(
                domain=faker.domain_name(),
                fullname=faker.company(),
                country=random.choice(countries)[0],
            )

    def fake_current_events(self, faker, count=5):
        """Ongoing and upcoming events."""
        twodays = timedelta(days=2)
        organizations = list(Organization.objects.exclude(domain='self-organized'))
        countries = list(Countries)

        for i in range(count):
            start = faker.date_time_this_year(before_now=False,
                                              after_now=True).date()
            Event.objects.create(
                slug='{:%Y-%m-%d}-{}'.format(start, faker.slug()),
                start=start,
                end=start + twodays,
                url=faker.url(),
                host=random.choice(organizations),
                # needed in order for event to be published
                country=random.choice(countries)[0],
                venue=faker.word().title(),
                address=faker.sentence(nb_words=4, variable_nb_words=True),
                latitude=random.uniform(-90, 90),
                longitude=random.uniform(0, 180),
            )

    def fake_uninvoiced_events(self, faker, count=5):
        """Preferably in the past, and with 'uninvoiced' status."""
        twodays = timedelta(days=2)
        countries = list(Countries)
        organizations = list(Organization.objects.exclude(domain='self-organized'))

        for i in range(count):
            start = faker.date_time_this_year(before_now=True,
                                              after_now=False).date()
            Event.objects.create(
                slug='{:%Y-%m-%d}-{}'.format(start, faker.slug()),
                start=start,
                end=start + twodays,
                url=faker.url(),
                host=random.choice(organizations),
                # needed in order for event to be published
                country=random.choice(countries)[0],
                venue=faker.word().title(),
                address=faker.sentence(nb_words=4, variable_nb_words=True),
                latitude=random.uniform(-90, 90),
                longitude=random.uniform(0, 180),
                # needed in order for event to be uninvoiced
                invoice_status='not-invoiced',
            )

    def fake_unpublished_events(self, faker, count=5):
        """Events with missing location data (which is required for publishing
        them)."""
        twodays = timedelta(days=2)
        organizations = list(Organization.objects.exclude(domain='self-organized'))

        for i in range(count):
            start = faker.date_time_this_year(before_now=True,
                                              after_now=True).date()
            Event.objects.create(
                slug='{:%Y-%m-%d}-{}'.format(start, faker.slug()),
                start=start,
                end=start + twodays,
                url=faker.url(),
                host=random.choice(organizations),
            )

    def fake_self_organized_events(self, faker, count=5):
        """Full-blown events with 'self-organized' host."""
        twodays = timedelta(days=2)
        self_organized = Organization.objects.get(domain='self-organized')
        countries = list(Countries)
        invoice_statuses = Event.INVOICED_CHOICES

        for i in range(count):
            start = faker.date_time_this_year(before_now=True,
                                              after_now=True).date()
            Event.objects.create(
                slug='{:%Y-%m-%d}-{}'.format(start, faker.slug()),
                start=start,
                end=start + twodays,
                url=faker.url(),
                host=self_organized,
                # needed in order for event to be published
                country=random.choice(countries)[0],
                venue=faker.word().title(),
                address=faker.sentence(nb_words=4, variable_nb_words=True),
                latitude=random.uniform(-90, 90),
                longitude=random.uniform(0, 180),
                # needed in order for event to be uninvoiced
                invoice_status=random.choice(invoice_statuses)[0],
            )

    def fake_tasks(self, faker, count=50):
        events = Event.objects.all()
        persons = Person.objects.all()
        roles = Role.objects.all()
        all_possible = itertools.product(events, persons, roles)

        for event, person, role in random.sample(list(all_possible), count):
            Task.objects.create(
                event=event,
                person=person,
                role=role,
                title=faker.sentence(nb_words=4, variable_nb_words=True),
                url=faker.url(),
            )

    def fake_trainees(self, faker):
        host = Organization.objects.all()[0]
        tag = Tag.objects.get(name='TTT')
        learner = Role.objects.get(name='learner')
        swc_instructor = Badge.objects.get(name='swc-instructor')
        dc_instructor = Badge.objects.get(name='dc-instructor')

        # events
        first_event = Event.objects.create(host=host, slug='first-ttt-event')
        first_event.tags.add(tag)
        first_event.save()

        second_event = Event.objects.create(host=host, slug='second-ttt-event')
        second_event.tags.add(tag)
        second_event.save()

        third_event = Event.objects.create(host=host, slug='third-ttt-event')
        third_event.tags.add(tag)
        third_event.save()

        # persons
        bob = Person.objects.create(username='bob-trainee',
                                    personal='Bob', family='Smith')
        alice = Person.objects.create(username='alice-trainee',
                                      personal='Alice', family='Smith')
        john = Person.objects.create(username='john-trainee',
                                     personal='Alice', family='Smith')

        # tasks
        Task.objects.create(event=first_event, person=bob, role=learner)
        Task.objects.create(event=first_event, person=alice, role=learner)
        Task.objects.create(event=second_event, person=john, role=learner)

        # awards
        Award.objects.create(event=first_event, person=bob, badge=swc_instructor,
                             awarded=datetime(2016, 4, 1))
        Award.objects.create(event=third_event, person=bob, badge=dc_instructor,
                             awarded=datetime(2016, 4, 1))

    def handle(self, *args, **options):
        faker = Faker()

        seed = options['seed']
        if seed is not None:
            faker.seed(seed)

        self.fake_airports(faker)
        self.fake_roles(faker)
        self.fake_tags(faker)
        self.fake_badges(faker)
        self.fake_instructors(faker)
        self.fake_noninstructors(faker)
        self.fake_organizations(faker)
        self.fake_current_events(faker)
        self.fake_uninvoiced_events(faker)
        self.fake_unpublished_events(faker)
        self.fake_self_organized_events(faker)
        self.fake_tasks(faker)
        self.fake_trainees(faker)
<|endoftext|>


<commit_before>from application import app
from .utils import add_to_daylist, validate_title, get_worklist, get_case_details
from flask import request, Response

import json

@app.route('/', methods=["GET"])
def index():
    return 'Cases API'

@app.route('/cases', methods=["GET","POST"])
def getCases():
    if request.method == 'GET':
        # team_id currently hardcoded to 1578, a team with 6 cases on UACT, future story will exist to correctly ascertain this value
        team_id = '1578'
        data = get_worklist(team_id)
        #If the cases array is empty
        if not data["cases"]:
            return Response("No cases found for {0} for team".format(team_id), 404)
        else:
            return json.dumps(data)
    else: #POST will trigger this leg
        #Get case information from POST body
        case_data = request.get_json()
        title_number = case_data["titleNumber"]

        title_validation_code = validate_title(title_number)
        if title_validation_code == "1":

            application_reference = add_to_daylist(title_number)

        else:
            application_reference = ""

        resp = Response('{"submissionRef" : "' + case_data["submissionRef"] + '", "applicationReference" : "' + application_reference + '", "TitleValidationCode" : "' + title_validation_code + '"}', status=200, mimetype='application/json')
        return resp

@app.route('/cases/<caseid>', methods=["GET"])
def getCase(caseid):
    data = get_case_details(caseid)

    if data:
        return json.dumps(data)
    else:
        return Response("No case found for {0}".format(caseid), 404)
<commit_msg>changed team id<commit_after>from application import app
from .utils import add_to_daylist, validate_title, get_worklist, get_case_details
from flask import request, Response

import json

@app.route('/', methods=["GET"])
def index():
    return 'Cases API'

@app.route('/cases', methods=["GET","POST"])
def getCases():
    if request.method == 'GET':
        # team_id currently hardcoded to 1578, a team with 6 cases on UACT, future story will exist to correctly ascertain this value
        team_id = '20007'
        data = get_worklist(team_id)
        #If the cases array is empty
        if not data["cases"]:
            return Response("No cases found for {0} for team".format(team_id), 404)
        else:
            return json.dumps(data)
    else: #POST will trigger this leg
        #Get case information from POST body
        case_data = request.get_json()
        title_number = case_data["titleNumber"]

        title_validation_code = validate_title(title_number)
        if title_validation_code == "1":

            application_reference = add_to_daylist(title_number)

        else:
            application_reference = ""

        resp = Response('{"submissionRef" : "' + case_data["submissionRef"] + '", "applicationReference" : "' + application_reference + '", "TitleValidationCode" : "' + title_validation_code + '"}', status=200, mimetype='application/json')
        return resp

@app.route('/cases/<caseid>', methods=["GET"])
def getCase(caseid):
    data = get_case_details(caseid)

    if data:
        return json.dumps(data)
    else:
        return Response("No case found for {0}".format(caseid), 404)
<|endoftext|>


<commit_before>from __future__ import print_function

from django.core.management import BaseCommand

from dimagi.utils.chunked import chunked

from corehq.apps.es import CaseES
from corehq.apps.hqcase.utils import bulk_update_cases
from corehq.apps.receiverwrapper.exceptions import LocalSubmissionError
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
from corehq.form_processor.models import CommCareCaseSQL, CommCareCaseIndexSQL
from corehq.util.log import with_progress_bar

class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument(
            'domain',
            help="File path for log file",
        )
        parser.add_argument(
            'log_file',
            help="File path for log file",
        )

    def handle(self, domain, log_file, **options):
        total_cases = CaseES().domain(domain).case_type('household').is_closed().count()
        case_accessor = CaseAccessors(domain)
        failed_updates = []
        with open(log_file, "w") as fh:
            fh.write('--------Successful Form Ids----------')
            for cases in chunked(with_progress_bar(self._get_cases_to_process(domain), total_cases), 100):
                related_cases = {case.case_id for case in case_accessor.get_all_reverse_indices_info(list(cases))
                                 if case.relationship == CommCareCaseIndexSQL.CHILD}
                case_tupes = [(case_id, {}, True) for case_id in related_cases]
                try:
                    xform, cases = bulk_update_cases(domain, case_tupes)
                    fh.write(xform.form_id)
                except LocalSubmissionError as e:
                    print(unicode(e))
                    failed_updates.extend(related_cases)
            fh.write('--------Failed Cases--------------')
            for case_id in failed_updates:
                fh.write(case_id)

    def _get_cases_to_process(self, domain):
        from corehq.sql_db.util import get_db_aliases_for_partitioned_query
        dbs = get_db_aliases_for_partitioned_query()
        for db in dbs:
            cases = CommCareCaseSQL.objects.using(db).filter(domain=domain, type='household', closed=True)
            for case in cases:
                yield case.case_id
<commit_msg>fixing logging<commit_after>from __future__ import print_function

from django.core.management import BaseCommand

from dimagi.utils.chunked import chunked

from corehq.apps.es import CaseES
from corehq.apps.hqcase.utils import bulk_update_cases
from corehq.apps.receiverwrapper.exceptions import LocalSubmissionError
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
from corehq.form_processor.models import CommCareCaseSQL, CommCareCaseIndexSQL
from corehq.util.log import with_progress_bar

class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument(
            'domain',
            help="File path for log file",
        )
        parser.add_argument(
            'log_file',
            help="File path for log file",
        )

    def handle(self, domain, log_file, **options):
        total_cases = CaseES().domain(domain).case_type('household').is_closed().count()
        case_accessor = CaseAccessors(domain)
        failed_updates = []
        with open(log_file, "w") as fh:
            fh.write('--------Successful Form Ids----------\n')
            for cases in chunked(with_progress_bar(self._get_cases_to_process(domain), total_cases), 100):
                related_cases = {case.case_id for case in case_accessor.get_all_reverse_indices_info(list(cases))
                                 if case.relationship == CommCareCaseIndexSQL.CHILD}
                case_tupes = [(case_id, {}, True) for case_id in related_cases]
                try:
                    xform, cases = bulk_update_cases(domain, case_tupes)
                    fh.write(xform.form_id + '\n')
                except LocalSubmissionError as e:
                    print(unicode(e))
                    failed_updates.extend(related_cases)
            fh.write('--------Failed Cases--------------\n')
            for case_id in failed_updates:
                fh.write(case_id)

    def _get_cases_to_process(self, domain):
        from corehq.sql_db.util import get_db_aliases_for_partitioned_query
        dbs = get_db_aliases_for_partitioned_query()
        for db in dbs:
            cases = CommCareCaseSQL.objects.using(db).filter(domain=domain, type='household', closed=True)
            for case in cases:
                yield case.case_id
<|endoftext|>


<commit_before>
"""
Implementation of the command-line I{flake8} tool.
"""
import sys
import os

from flake8.util import skip_file
from flake8 import pep8
from flake8 import pyflakes
from flake8 import mccabe

def check_file(path, complexity=10):

    warnings = pyflakes.checkPath(path)
    warnings += pep8.input_file(path)
    warnings += mccabe.get_module_complexity(path, complexity)
    return warnings

def check_code(code, complexity=10):
    warnings = pyflakes.check(code, '<stdin>')
    warnings += mccabe.get_code_complexity(code, complexity)
    return warnings

def _get_python_files(paths):
    for path in paths:
        if os.path.isdir(path):
            for dirpath, dirnames, filenames in os.walk(path):
                for filename in filenames:
                    if not filename.endswith('.py'):
                        continue
                    fullpath = os.path.join(dirpath, filename)
                    if not skip_file(fullpath):
                        yield fullpath

        else:
            if not skip_file(path):
                yield path

def main():
    pep8.process_options()
    warnings = 0
    args = sys.argv[1:]
    if args:
        for path in _get_python_files(args):
            warnings += check_file(path)
    else:
        stdin = sys.stdin.read()
        warnings += check_code(stdin)

    raise SystemExit(warnings > 0)

def _get_files(repo, **kwargs):
    for rev in xrange(repo[kwargs['node']], len(repo)):
        for file_ in repo[rev].files():
            if not file_.endswith('.py'):
                continue
            if skip_file(file_):
                continue
            yield file_

def hg_hook(ui, repo, **kwargs):
    pep8.process_options()
    warnings = 0
    for file_ in _get_files(repo, **kwargs):
        warnings += check_file(file_)

    strict = ui.config('flake8', 'strict')
    if strict is None:
        strict = True

    if strict.lower() in ('1', 'true'):
        return warnings

    return 0

if __name__ == '__main__':
    main()
<commit_msg>make short options work<commit_after>
"""
Implementation of the command-line I{flake8} tool.
"""
import sys
import os

from flake8.util import skip_file
from flake8 import pep8
from flake8 import pyflakes
from flake8 import mccabe

def check_file(path, complexity=10):

    warnings = pyflakes.checkPath(path)
    warnings += pep8.input_file(path)
    warnings += mccabe.get_module_complexity(path, complexity)
    return warnings

def check_code(code, complexity=10):
    warnings = pyflakes.check(code, '<stdin>')
    warnings += mccabe.get_code_complexity(code, complexity)
    return warnings

def _get_python_files(paths):
    for path in paths:
        if os.path.isdir(path):
            for dirpath, dirnames, filenames in os.walk(path):
                for filename in filenames:
                    if not filename.endswith('.py'):
                        continue
                    fullpath = os.path.join(dirpath, filename)
                    if not skip_file(fullpath):
                        yield fullpath

        else:
            if not skip_file(path):
                yield path

def main():
    options, args = pep8.process_options()
    warnings = 0
    if args:
        for path in _get_python_files(args):
            warnings += check_file(path)
    else:
        stdin = sys.stdin.read()
        warnings += check_code(stdin)

    raise SystemExit(warnings > 0)

def _get_files(repo, **kwargs):
    for rev in xrange(repo[kwargs['node']], len(repo)):
        for file_ in repo[rev].files():
            if not file_.endswith('.py'):
                continue
            if skip_file(file_):
                continue
            yield file_

def hg_hook(ui, repo, **kwargs):
    pep8.process_options()
    warnings = 0
    for file_ in _get_files(repo, **kwargs):
        warnings += check_file(file_)

    strict = ui.config('flake8', 'strict')
    if strict is None:
        strict = True

    if strict.lower() in ('1', 'true'):
        return warnings

    return 0

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
 mbed CMSIS-DAP debugger
 Copyright (c) 2006-2013 ARM Limited

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

from interface import Interface
import logging, os

try:
    import usb.core
    import usb.util
except:
    if os.name == "posix" and not os.uname()[0] == 'Darwin':
        logging.error("PyUSB is required on a Linux Machine")
    isAvailable = False
else:
    isAvailable = True

class PyUSB(Interface):
    """
    This class provides basic functions to access
    a USB HID device using pyusb:
        - write/read an endpoint
    """

    vid = 0
    pid = 0

    isAvailable = isAvailable

    def __init__(self):
        super(PyUSB, self).__init__()
        self.ep_out = None
        self.ep_in = None
        self.dev = None

    @staticmethod
    def getAllConnectedInterface(vid, pid):
        """
        returns all the connected devices which matches PyUSB.vid/PyUSB.pid.
        returns an array of PyUSB (Interface) objects
        """
        # find all devices matching the vid/pid specified
        all_devices = usb.core.find(find_all=True, idVendor=vid, idProduct=pid)

        if not all_devices:
            logging.debug("No device connected")
            return None

        boards = []

        # iterate on all devices found
        for board in all_devices:
            intf_number = 0
            found = False

            # get active config
            config = board.get_active_configuration()

            # iterate on all interfaces:
            #    - if we found a HID interface -> CMSIS-DAP
            for interface in config:
                if interface.bInterfaceClass == 0x03:
                    intf_number = interface.bInterfaceNumber
                    found = True
                    break

            if found == False:
                continue

            try:
                if board.is_kernel_driver_active(intf_number) is True:
                    board.detach_kernel_driver(intf_number)
            except Exception as e:
                print e
                pass

            intf = usb.util.find_descriptor(config, bInterfaceNumber = intf_number)
            ep_out = usb.util.find_descriptor(intf,
                                              # match the first OUT endpoint
                                              custom_match = \
                                              lambda e: \
                                              usb.util.endpoint_direction(e.bEndpointAddress) == \
                                              usb.util.ENDPOINT_OUT
                                              )
            ep_in = usb.util.find_descriptor(intf,
                                             # match the first IN endpoint
                                             custom_match = \
                                             lambda e: \
                                             usb.util.endpoint_direction(e.bEndpointAddress) == \
                                             usb.util.ENDPOINT_IN
                                             )
            product_name = usb.util.get_string(board, 256, 2)
            vendor_name = usb.util.get_string(board, 256, 1)
            if ep_out is None or ep_in is None:
                logging.error('Endpoints not found')
                return None

            new_board = PyUSB()
            new_board.ep_in = ep_in
            new_board.ep_out = ep_out
            new_board.dev = board
            new_board.vid = vid
            new_board.pid = pid
            new_board.product_name = product_name
            new_board.vendor_name = vendor_name
            boards.append(new_board)

        return boards

    def write(self, data):
        """
        write data on the OUT endpoint associated to the HID interface
        """
        if self.ep_out is None:
            raise ValueError('EP_OUT endpoint is NULL')

        self.ep_out.write(data)
        #logging.debug('sent: %s', data)
        return

    def read(self, timeout = -1):
        """
        read data on the IN endpoint associated to the HID interface
        """
        if self.ep_in is None:
            raise ValueError('EP_IN endpoint is NULL')

        data = self.ep_in.read(self.ep_in.wMaxPacketSize, timeout)
        #logging.debug('received: %s', data)
        return data
<commit_msg>Pyusb backend fix for using EP0<commit_after>"""
 mbed CMSIS-DAP debugger
 Copyright (c) 2006-2013 ARM Limited

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

from interface import Interface
import logging, os

try:
    import usb.core
    import usb.util
except:
    if os.name == "posix" and not os.uname()[0] == 'Darwin':
        logging.error("PyUSB is required on a Linux Machine")
    isAvailable = False
else:
    isAvailable = True

class PyUSB(Interface):
    """
    This class provides basic functions to access
    a USB HID device using pyusb:
        - write/read an endpoint
    """

    vid = 0
    pid = 0

    isAvailable = isAvailable

    def __init__(self):
        super(PyUSB, self).__init__()
        self.ep_out = None
        self.ep_in = None
        self.dev = None

    @staticmethod
    def getAllConnectedInterface(vid, pid):
        """
        returns all the connected devices which matches PyUSB.vid/PyUSB.pid.
        returns an array of PyUSB (Interface) objects
        """
        # find all devices matching the vid/pid specified
        all_devices = usb.core.find(find_all=True, idVendor=vid, idProduct=pid)

        if not all_devices:
            logging.debug("No device connected")
            return None

        boards = []

        # iterate on all devices found
        for board in all_devices:
            intf_number = 0
            found = False

            # get active config
            config = board.get_active_configuration()

            # iterate on all interfaces:
            #    - if we found a HID interface -> CMSIS-DAP
            for interface in config:
                if interface.bInterfaceClass == 0x03:
                    intf_number = interface.bInterfaceNumber
                    found = True
                    break

            if found == False:
                continue

            try:
                if board.is_kernel_driver_active(intf_number) is True:
                    board.detach_kernel_driver(intf_number)
            except Exception as e:
                print e
                pass

            intf = usb.util.find_descriptor(config, bInterfaceNumber = intf_number)
            ep_out = usb.util.find_descriptor(intf,
                                              # match the first OUT endpoint
                                              custom_match = \
                                              lambda e: \
                                              usb.util.endpoint_direction(e.bEndpointAddress) == \
                                              usb.util.ENDPOINT_OUT
                                              )
            ep_in = usb.util.find_descriptor(intf,
                                             # match the first IN endpoint
                                             custom_match = \
                                             lambda e: \
                                             usb.util.endpoint_direction(e.bEndpointAddress) == \
                                             usb.util.ENDPOINT_IN
                                             )
            product_name = usb.util.get_string(board, 256, 2)
            vendor_name = usb.util.get_string(board, 256, 1)
            """If there is no EP for OUT then we can use CTRL EP"""
            if ep_in is None: #ep_out is None or
                logging.error('Endpoints not found')
                return None

            new_board = PyUSB()
            new_board.ep_in = ep_in
            new_board.ep_out = ep_out
            new_board.dev = board
            new_board.vid = vid
            new_board.pid = pid
            new_board.product_name = product_name
            new_board.vendor_name = vendor_name
            boards.append(new_board)

        return boards

    def write(self, data):
        """
        write data on the OUT endpoint associated to the HID interface
        """
        if self.ep_out is None:
            self.dev.ctrl_transfer(0x21,0x9,0x200,0x3,data)
            return
            #raise ValueError('EP_OUT endpoint is NULL')

        self.ep_out.write(data)
        #logging.debug('sent: %s', data)
        return

    def read(self, timeout = -1):
        """
        read data on the IN endpoint associated to the HID interface
        """
        if self.ep_in is None:
            raise ValueError('EP_IN endpoint is NULL')

        data = self.ep_in.read(self.ep_in.wMaxPacketSize, timeout)
        #logging.debug('received: %s', data)
        return data
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-

import os
import time
import socket
import urllib, urlparse
import hashlib
import threading
import Queue
from cStringIO import StringIO
import base64
from defs import *
from protocol import parse_frame, make_frame
from utils import r_select

class _Client(object):

    def __init__(self, url, proxies=None):
        self.url = url
        self.uri = urlparse.urlparse(url)
        self.scheme = self.uri.scheme.lower()
        self.proxies = proxies
        self.is_ssl = self.scheme == 'wss'
        self.sock = None
        self.f = None
        self.evt_abort = threading.Event()
        self.q_in = Queue.Queue()
        self.key = ''
        self.status = None
        self.headers = Headers()

    def _parse_response(self, resp_header_raw):
        lines = resp_header_raw.split(crlf)
        h = lines[0].split(' ', 2)
        status = (int(h[1]), h[2])
        rows = [line.split(':')[:2] for line in lines if ':' in line]
        headers = Headers()
        for _k, _v in rows:
            k = _k.strip().lower()
            v = _v.strip()
            if k in headers:
                v0 = headers[k]
                if isinstance(v0, list):
                    v0.append(v)
                else:
                    headers[k] = [v0, v]
            else:
                headers[k] = v
        return status, headers

    def _check_handshake(self, resp_headers):
        connection = resp_headers.get('connection', None)
        if 'upgrade' not in resp_headers.get_lower_list('connection'):
            return False
        if 'websocket' not in resp_headers.get_lower_list('upgrade'):
            return False
        if 'sec-websocket-accept' not in resp_headers:
            return False
        key_hash = '%s%s' % (self.key, ws_uid)
        key_hash = base64.b64encode(hashlib.sha1(key_hash).digest())
        _accept = resp_headers['sec-websocket-accept']
        if key_hash != _accept:
            return False
        return True

    def handshake(self, timeout=20.0):
        t0 = time.time()
        uri = self.uri
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            sock.connect((uri.hostname, (uri.port or 80)))
        except socket.error:
            raise WsError(u'failed to connect to host.')
        self.sock = sock
        origin = '%s://%s' % (uri.scheme, uri.netloc)
        key = base64.b64encode(os.urandom(8))
        self.key = key
        headers = (('host', uri.netloc),
                   ('user-agent', 'hd-cluster/libs/ws.client'),
                   ('connection', 'upgrade'),
                   ('upgrade', 'websocket'),
                   ('origin', origin),
                   ('sec-websocket-version', 13),
                   ('sec-websocket-key', key),)
        sock.send('GET %s HTTP/1.1%s' % (uri.path, crlf))
        headers_str = crlf.join(['%s: %s' % (k,v) for k,v in headers])
        sock.send(headers_str)
        sock.send(crlf + crlf)
        buff = StringIO()
        while not self.evt_abort.is_set():
            r = r_select([sock], timeout=0.5)
            if not r:
                if t0 + timeout < time.time():
                    return False
                continue
            data = r[0].recv(1024)
            if not data:
                continue
            buff.write(data)
            if crlf + crlf not in buff.getvalue():
                continue
            resp_raw = buff.getvalue()
            resp_header_raw = resp_raw.split(crlf+crlf)[0]
            status, resp_headers = self._parse_response(resp_header_raw)
            self.status = status
            self.resp_headers = resp_headers
            if self.status[0] != 101:
                raise HTTPError(*self.status)
            handshake_ok = self._check_handshake(resp_headers)
            if not handshake_ok:
                return False
            data = resp_raw[len(resp_header_raw + crlf + crlf):]
            if data:
                try:
                    parse_frame(data)
                except EOF:
                    pass
            self.f = sock.makefile()
            return True

    def recv(self, timeout=5.0, allow_fragments=True):
        _op, _buff = None, None
        while not self.evt_abort.is_set():
            frame = self._recv_next(timeout=timeout)
            if frame:
                fin, op, payload = frame
                if not allow_fragments:
                    if fin and not _buff:
                        return frame
                    if not fin:
                        if not _buff:
                            _op = op
                            _buff = StringIO()
                    _buff.write(payload)
                    if fin:
                        return fin, _op, _buff.getvalue()
                    else:
                        continue
            return frame

    def _recv_next(self, timeout=5.0):
        _op, _buff = None, None
        t0 = time.time()
        while t0 + timeout >= time.time() and not self.evt_abort.is_set():
            if not self.f:
                time.sleep(0.1)
                continue
            r = r_select([self.f], timeout=0.1)
            if not r:
                continue
            f = r[0]
            try:
                frame = parse_frame(f)
                return frame
            except (IOError, AttributeError, socket.error, WsIOError):
                self.close()
                # raise WsCommunicationError()

    def __iter__(self):
        if not self.f:
            return
        while not self.evt_abort.is_set():
            item = self._recv_next()
            if item:
                yield item

    def send(self, data, fin=True, op=OP_TEXT, mask=True):
        if self.evt_abort.is_set() or not self.f:
            raise WsError('websocket was closed.')
        sub_f_size = MAX_FRAME_SIZE
        size = len(data)
        if size > sub_f_size:
            cur = 0
            while True:
                part = data[cur: cur + sub_f_size]
                if not part:
                    break
                _fin = False
                if cur + len(part) >= size:
                    _fin = fin
                _op = op
                if cur > 0:
                    _op = 0
                frame = make_frame(_fin, _op, part, mask=mask)
                self.f.write(frame)
                cur += len(part)
            self.f.flush()
        # if fin and (size > sub_f_size):
        #     cur = 0
        #     while True:
        #         part = data[cur: cur + sub_f_size]
        #         if not part:
        #             break
        #         _fin = 0
        #         if cur + len(part) >= size:
        #             _fin = 1
        #         _op = op
        #         if cur > 0:
        #             _op = 0
        #         frame = make_frame(_fin, _op, part, mask=mask)
        #         self.f.write(frame)
        #         cur += len(part)
        #     self.f.flush()
        else:
            frame = make_frame(fin, op, data, mask=mask)
            self.f.write(frame)
            self.f.flush()

    def close(self):
        if not self.evt_abort.is_set():
            self.evt_abort.set()
        if self.f:
            self.f.close()
            self.f = None
        if self.sock:
            self.sock.close()
            self.sock = None

    def __enter__(self):
        return self

    def __exit__(self, *args, **kargs):
        self.close()

def ws_connect(*args, **kargs):
    return _Client(*args, **kargs)

if __name__ == '__main__':
    with ws_connect('ws://50.gz2.yj.hp:8082/ws') as c:
        if c.handshake():
            print 'handshake: ok'
            c.send('{"op":"LIST_NOTES","principal":"anonymous","ticket":"anonymous"}')
            for msg in c:
                print msg
                break
    # ws_connect('ws://localhost:10080/hub/notebooks/notebook/2BA4MWGBT/_.ws')
<commit_msg>rename _Client<commit_after>#-*- coding: utf-8 -*-

import os
import time
import socket
import urllib, urlparse
import hashlib
import threading
import Queue
from cStringIO import StringIO
import base64
from defs import *
from protocol import parse_frame, make_frame
from utils import r_select

class Client(object):

    def __init__(self, url, proxies=None):
        self.url = url
        self.uri = urlparse.urlparse(url)
        self.scheme = self.uri.scheme.lower()
        self.proxies = proxies
        self.is_ssl = self.scheme == 'wss'
        self.sock = None
        self.f = None
        self.evt_abort = threading.Event()
        self.q_in = Queue.Queue()
        self.key = ''
        self.status = None
        self.headers = Headers()

    def _parse_response(self, resp_header_raw):
        lines = resp_header_raw.split(crlf)
        h = lines[0].split(' ', 2)
        status = (int(h[1]), h[2])
        rows = [line.split(':')[:2] for line in lines if ':' in line]
        headers = Headers()
        for _k, _v in rows:
            k = _k.strip().lower()
            v = _v.strip()
            if k in headers:
                v0 = headers[k]
                if isinstance(v0, list):
                    v0.append(v)
                else:
                    headers[k] = [v0, v]
            else:
                headers[k] = v
        return status, headers

    def _check_handshake(self, resp_headers):
        connection = resp_headers.get('connection', None)
        if 'upgrade' not in resp_headers.get_lower_list('connection'):
            return False
        if 'websocket' not in resp_headers.get_lower_list('upgrade'):
            return False
        if 'sec-websocket-accept' not in resp_headers:
            return False
        key_hash = '%s%s' % (self.key, ws_uid)
        key_hash = base64.b64encode(hashlib.sha1(key_hash).digest())
        _accept = resp_headers['sec-websocket-accept']
        if key_hash != _accept:
            return False
        return True

    def handshake(self, timeout=20.0):
        t0 = time.time()
        uri = self.uri
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            sock.connect((uri.hostname, (uri.port or 80)))
        except socket.error:
            raise WsError(u'failed to connect to host.')
        self.sock = sock
        origin = '%s://%s' % (uri.scheme, uri.netloc)
        key = base64.b64encode(os.urandom(8))
        self.key = key
        headers = (('host', uri.netloc),
                   ('user-agent', 'hd-cluster/libs/ws.client'),
                   ('connection', 'upgrade'),
                   ('upgrade', 'websocket'),
                   ('origin', origin),
                   ('sec-websocket-version', 13),
                   ('sec-websocket-key', key),)
        sock.send('GET %s HTTP/1.1%s' % (uri.path, crlf))
        headers_str = crlf.join(['%s: %s' % (k,v) for k,v in headers])
        sock.send(headers_str)
        sock.send(crlf + crlf)
        buff = StringIO()
        while not self.evt_abort.is_set():
            r = r_select([sock], timeout=0.5)
            if not r:
                if t0 + timeout < time.time():
                    return False
                continue
            data = r[0].recv(1024)
            if not data:
                continue
            buff.write(data)
            if crlf + crlf not in buff.getvalue():
                continue
            resp_raw = buff.getvalue()
            resp_header_raw = resp_raw.split(crlf+crlf)[0]
            status, resp_headers = self._parse_response(resp_header_raw)
            self.status = status
            self.resp_headers = resp_headers
            if self.status[0] != 101:
                raise HTTPError(*self.status)
            handshake_ok = self._check_handshake(resp_headers)
            if not handshake_ok:
                return False
            data = resp_raw[len(resp_header_raw + crlf + crlf):]
            if data:
                try:
                    parse_frame(data)
                except EOF:
                    pass
            self.f = sock.makefile()
            return True

    def recv(self, timeout=5.0, allow_fragments=True):
        _op, _buff = None, None
        while not self.evt_abort.is_set():
            frame = self._recv_next(timeout=timeout)
            if frame:
                fin, op, payload = frame
                if not allow_fragments:
                    if fin and not _buff:
                        return frame
                    if not fin:
                        if not _buff:
                            _op = op
                            _buff = StringIO()
                    _buff.write(payload)
                    if fin:
                        return fin, _op, _buff.getvalue()
                    else:
                        continue
            return frame

    def _recv_next(self, timeout=5.0):
        _op, _buff = None, None
        t0 = time.time()
        while t0 + timeout >= time.time() and not self.evt_abort.is_set():
            if not self.f:
                time.sleep(0.1)
                continue
            r = r_select([self.f], timeout=0.1)
            if not r:
                continue
            f = r[0]
            try:
                frame = parse_frame(f)
                return frame
            except (IOError, AttributeError, socket.error, WsIOError):
                self.close()
                # raise WsCommunicationError()

    def __iter__(self):
        if not self.f:
            return
        while not self.evt_abort.is_set():
            item = self._recv_next()
            if item:
                yield item

    def send(self, data, fin=True, op=OP_TEXT, mask=True):
        if self.evt_abort.is_set() or not self.f:
            raise WsError('websocket was closed.')
        sub_f_size = MAX_FRAME_SIZE
        size = len(data)
        if size > sub_f_size:
            cur = 0
            while True:
                part = data[cur: cur + sub_f_size]
                if not part:
                    break
                _fin = False
                if cur + len(part) >= size:
                    _fin = fin
                _op = op
                if cur > 0:
                    _op = 0
                frame = make_frame(_fin, _op, part, mask=mask)
                self.f.write(frame)
                cur += len(part)
            self.f.flush()
        # if fin and (size > sub_f_size):
        #     cur = 0
        #     while True:
        #         part = data[cur: cur + sub_f_size]
        #         if not part:
        #             break
        #         _fin = 0
        #         if cur + len(part) >= size:
        #             _fin = 1
        #         _op = op
        #         if cur > 0:
        #             _op = 0
        #         frame = make_frame(_fin, _op, part, mask=mask)
        #         self.f.write(frame)
        #         cur += len(part)
        #     self.f.flush()
        else:
            frame = make_frame(fin, op, data, mask=mask)
            self.f.write(frame)
            self.f.flush()

    def close(self):
        if not self.evt_abort.is_set():
            self.evt_abort.set()
        if self.f:
            self.f.close()
            self.f = None
        if self.sock:
            self.sock.close()
            self.sock = None

    def __enter__(self):
        return self

    def __exit__(self, *args, **kargs):
        self.close()

def ws_connect(*args, **kargs):
    return Client(*args, **kargs)

if __name__ == '__main__':
    with ws_connect('ws://50.gz2.yj.hp:8082/ws') as c:
        if c.handshake():
            print 'handshake: ok'
            c.send('{"op":"LIST_NOTES","principal":"anonymous","ticket":"anonymous"}')
            for msg in c:
                print msg
                break
    # ws_connect('ws://localhost:10080/hub/notebooks/notebook/2BA4MWGBT/_.ws')
<|endoftext|>


<commit_before>#! /usr/bin/python
from __future__ import print_function, division
import serial # for pulling data from Current Cost
import xml.etree.ElementTree as ET # for XML parsing
import time # for printing UNIX timecode
import urllib2 # for sending data to Pachube
import json # for assembling JSON data for Pachube
import sys # for printing errors to stderr
import os # to find names of existing files
import argparse
import threading
import signal

"""
TODO: Cope with fact that we can't trust the serial ports;
  we probably want to use radioIDs... it'd be nice to
  find a way to track individual monitors though.
  We should also take advantage of the fact that all
  monitors record aggregate data. We could merge these together.
"""

"""
TODO: handle mappings
"""

######################################
#     GLOBALS                        #
######################################

abort = False # Make this True to halt all threads
directory = None # The directory to write data to

#####################################
#     SENSOR CLASS                  #
#####################################
class Sensor(object):

    def __init__(self):
        self.timeInfo = TimeInfo()
        self.radioIDs = {} # list of all radioIDs seen by this sensor
        self.radioID  = None # current radioID

    def update(self, watts, radioID):
        self.timeInfo.update()
        self.watts = watts

        self.radioID = radioID
        if radioID in self.radioIDs.keys():
            self.radioIDs[radioID] += 1
        else:
            self.radioIDs[radioID] = 0

    def __str__(self):
        return '{:>7d}{}{:>9} {}'.format(self.watts, self.timeInfo, self.radioID, self.radioIDs)

    def writeData(self):
        return '{:d}\t{}\t{}\n'.format(int(round(self.timeInfo.lastSeen)), self.watts, self.radioID)

#####################################
#     CURRENT COST CLASS            #
#####################################
class CurrentCost(threading.Thread):

    id = None # The ID for this Current Cost.  e.g. "A" or "B" or "C" etc.

    def __init__(self, port ):
        threading.Thread.__init__(self)
        global abort
        try:
            self.port = port
            self.serial = None
            self._openPort()
            self._getInfo()
            self.sensors   = {}
        except (OSError, serial.SerialException):
            abort = True
            raise

    def _openPort(self):
        if self.serial != None and self.serial.isOpen():
            print("Closing serial port {}\n".format(self.port), file=sys.stderr)
            try:
                self.serial.close()
            except:
                pass

        print("Opening serial port ", self.port, file=sys.stderr)

        try:
            self.serial = serial.Serial(self.port, 57600)
            self.serial.flushInput()
        except OSError, e:
            print("Serial port " + self.port + " unavailable.  Is another process using it?", str(e), sep="\n", file=sys.stderr)
            raise
        except serial.SerialException, e:
            print("serial.SerialException:", str(e), "Is the correct USB port specified in config.xml?\n", sep="\n", file=sys.stderr)
            raise

    def run(self):
        global abort
        try:
            while abort == False:
                self.update()
        except:
            abort = True
            raise

    def readXML(self, data):
        """Reads a line from the serial port and returns an ElementTree"""
        RETRIES = 10
        for i in range(RETRIES):
            try:
                line = self.serial.readline()
                tree = ET.XML(line)

                # Check if this is histogram data from the current cost (which we're not interested in)
                # (This could also be done by checking the size of 'line' - this would probably be faster although
                #  possibly the size of a "histogram" is variable)
                if tree.findtext('hist') != None:
                    continue

                # Check if all the elements we're looking for exist in this XML
                success = True
                for key in data.keys():
                    data[key] = tree.findtext(key)
                    if data[key] == None:
                        success = False
                        print("Key \'{}\' not found in XML:\n{}".format(key, line), file=sys.stderr)
                        break

                if success:
                    return data
                else:
                    continue

            except OSError, e: # catch errors raised by serial.readline
                print("Serial port " + self.port + " unavailable.  Is another process using it?", str(e), sep="\n", file=sys.stderr)
            except serial.SerialException, e:
                print("SerialException on port {}:".format(self.port), str(e), "Has the device been unplugged?\n", sep="\n", file=sys.stderr)
            except ET.ParseError, e: # Catch XML errors (occasionally the current cost outputs malformed XML)
                print("XML error: ", str(e), line, sep="\n", file=sys.stderr)
            except ValueError, e: # Attempting to use a port that is not open
                print("ValueError: ", str(e), sep="\n", file=sys.stderr)

            # We should only execute the following after an exception
            # Retry...
            time.sleep(1)
            print("retrying... retry number {} of {}\n".format(i, RETRIES), file=sys.stderr)

            # Try to flush the serial port.
            try:
                self.serial.flushInput()
            except: # Ignore errors.  We're going to retry anyway.
                pass

            # Try to re-open the port.
            try:
                self._openPort()
            except: # Ignore errors.  We're going to retry anyway.
                pass

        # If we get to here then we have failed after every retry
        global abort
        abort = True
        raise Exception("readXML failed after {} retries".format(RETRIES))

    def _getInfo(self):
        """Get DSB and version number from Current Cost monitor"""
        data = {'dsb': None, 'src': None}
        data           = self.readXML( data )
        self.dsb       = data['dsb']
        self.CCversion = data['src']

    def writeToDisk(self, sensorNum):
        filename =  directory + "channel_" + self.id + str(sensorNum) + ".dat"
        try:
            filehandle = open(filename, 'a+')
        except Exception, e:
            print("Failed to open file {}.\n".format(filename), str(e), sep="\n", file=sys.stderr)
            raise

        filehandle.write( self.sensors[sensorNum].writeData() )
        filehandle.close()

    def update(self):
        """Read sensor data from serial port."""

        # For Current Cost XML details, see currentcost.com/cc128/xml.htm
        data = {'id': None, 'sensor': None, 'ch1/watts': None}

        data     = self.readXML( data )
        radioID  = int( data['id']        )
        sensor   = int( data['sensor']    )
        watts    = int( data['ch1/watts'] )

        if sensor not in self.sensors.keys():
            self.sensors[sensor] = Sensor()

        self.sensors[sensor].update(watts, radioID)

        self.writeToDisk( sensor )

    def __str__(self):
        string  = "----------------\n"
        string += "monitorID = {}\n".format(self.id)
        string += "port      = {}\n".format(self.port)
        string += "DSB       = {}\n".format(self.dsb)
        string += "Version   = {}\n\n".format(self.CCversion)
        string += "               -----PERIOD STATS (secs)--\n"
        string += "SENSOR  WATTS  MEAN    MAX    MIN CURRENT COUNT  RADIOID RADIOIDs\n\n"
        keys = self.sensors.keys()
        keys.sort()
        for key in keys:
            sensor  = self.sensors[key]
            string += '{:>5}{}\n'.format(key, str(sensor))

        string += "\n"
        return string

class TimeInfo(object):
    """Class for recording simple statistics about time. Useful during IAM testing."""

    def __init__(self):
        self.count    = -1
        self.lastSeen = 0

    def update(self):
        unixTime = time.time()
        self.count += 1
        self.current = unixTime - self.lastSeen

        if self.count == 0: # this is the first time we've run
            self.current = None
            self.mean = None
            self.max  = None
            self.min  = None
        elif self.count == 1:
            self.mean = self.current
            self.max  = self.current
            self.min  = self.current
        else:
            self.mean = ((float(self.mean) * (self.count - 1)) + self.current) / self.count
            if self.current > self.max: self.max = self.current
            if self.current < self.min: self.min = self.current

        self.lastSeen = unixTime

    def __str__(self):
        if self.count < 1:
            return "     -       -      -      -      {:d}".format(self.count)
        else:
            return '{:>7.2f}{:>7.1f}{:>7.1f}{:>7.1f}{:>7d}'.format(self.mean, self.max, self.min, self.current, self.count)

class Manager(object):
    """
    Singleton. Used to manage multiple CurrentCost objects.
    """

    def __init__(self, monitors):
        self.monitors = monitors

    def run(self, noDisplay=True):
        # Start each monitor thread
        for monitor in self.monitors:
            monitor.start()

        # Use this main thread of control to continually
        # print out info
        if noDisplay:
            print("Press CTRL+C to stop.\n")
            signal.pause()
        else:
            while abort == False:
                os.system('clear')
                print(str(self))
                self.checkDuplicateRadioIDs()
                print("\nPress CTRL+C to stop.\n")
                time.sleep(1)

        self.stop()

    def stop(self):
        global abort
        abort = True

        print("Stopping...")

        # Don't exit the main thread until our worker threads have all quit
        for monitor in self.monitors:
            print("Waiting for monitor {} to stop...\n".format(monitor.id))
            monitor.join()

        print("Done.\n")

    def checkDuplicateRadioIDs(self):
        """
        Look across all monitors to check for duplicated radioIDs
        """

        # make a list of radioIDs
        radioIDs = set([])
        duplicateRadioIDs = set([])
        for monitor in self.monitors:
            for sensorNum, sensorData in monitor.sensors.iteritems():
                for radioID in sensorData.radioIDs.keys():
                    if radioID in radioIDs:
                        duplicateRadioIDs = set.union(duplicateRadioIDs, [radioID])
                    else:
                        radioIDs = set.union(radioIDs, [radioID])

        if len(duplicateRadioIDs) > 0:
            print("\nDuplicate radio IDs: {}\n".format(duplicateRadioIDs))

    def __str__(self):
        string = ""
        for monitor in self.monitors:
            string += str(monitor)

        return string

#########################################
#      LOAD CONFIG                      #
#########################################

def loadConfig():
    configTree    = ET.parse("config.xml") # load config from config file
    global directory
    directory     = configTree.findtext("directory") # File to save data to
    monitorsETree = configTree.findall("monitor")

    monitors = []

    for monitor in monitorsETree:
        currentCost    = CurrentCost( monitor.findtext("serialport") )
        currentCost.id = monitor.attrib['id']

        monitorMapping = monitor.findtext("mapping")
        mapping = monitorMapping.split("\n")
        mappingMatrix = []
        for line in mapping:
            strippedLine = line.strip()
            if strippedLine != '':
                mappingMatrix.append([i.strip() for i in strippedLine.split(',')])

        currentCost.mapping = mappingMatrix

        monitors.append( currentCost )

    return monitors

#########################################
#      HANDLE SIGINTs                   #
# So we do the right thing with CTRL+C  #
#########################################

def signalHandler(signal, frame):
    print("\nSIGINT received.")
    global abort
    abort = True

########################################
#  PROCESS COMMAND LINE ARGUMENTS      #
########################################

if __name__ == "__main__":
    # Process command line args
    parser = argparse.ArgumentParser(description='Log data from multiple Current Cost IAMs.')
    parser.add_argument('--noDisplay', dest='noDisplay', action='store_const',
                        const=True, default=False, help='Do not display info to std out. Useful for use with nohup command.')
    args = parser.parse_args()

    # load config
    monitors = loadConfig()

    # register SIGINT handler
    print("setting signal handler")
    signal.signal(signal.SIGINT, signalHandler)

    manager = Manager(monitors)

    print(args.noDisplay)

    try:
        manager.run(args.noDisplay)
    except:
        manager.stop()
        raise
<commit_msg>Added a --printXML command line option to just print XML.  Also created a new CurrentCost.readLine() and CurrentCost.resetSerial() functions which are broken out from code previously in readXML().<commit_after>#! /usr/bin/python
from __future__ import print_function, division
import serial # for pulling data from Current Cost
import xml.etree.ElementTree as ET # for XML parsing
import time # for printing UNIX timecode
import urllib2 # for sending data to Pachube
import json # for assembling JSON data for Pachube
import sys # for printing errors to stderr
import os # to find names of existing files
import argparse
import threading
import signal

"""
TODO: Cope with fact that we can't trust the serial ports;
  we probably want to use radioIDs... it'd be nice to
  find a way to track individual monitors though.
  We should also take advantage of the fact that all
  monitors record aggregate data. We could merge these together.
"""

"""
TODO: handle mappings
"""

######################################
#     GLOBALS                        #
######################################

abort = False # Make this True to halt all threads
directory = None # The directory to write data to

#####################################
#     SENSOR CLASS                  #
#####################################
class Sensor(object):

    def __init__(self):
        self.timeInfo = TimeInfo()
        self.radioIDs = {} # list of all radioIDs seen by this sensor
        self.radioID  = None # current radioID

    def update(self, watts, radioID):
        self.timeInfo.update()
        self.watts = watts

        self.radioID = radioID
        if radioID in self.radioIDs.keys():
            self.radioIDs[radioID] += 1
        else:
            self.radioIDs[radioID] = 0

    def __str__(self):
        return '{:>7d}{}{:>9} {}'.format(self.watts, self.timeInfo, self.radioID, self.radioIDs)

    def writeData(self):
        return '{:d}\t{}\t{}\n'.format(int(round(self.timeInfo.lastSeen)), self.watts, self.radioID)

#####################################
#     CURRENT COST CLASS            #
#####################################
class CurrentCost(threading.Thread):

    id = None # The ID for this Current Cost.  e.g. "A" or "B" or "C" etc.

    def __init__(self, port):
        threading.Thread.__init__(self)
        global abort
        try:
            self.printXML = False
            self.port = port
            self.serial = None
            self._openPort()
            self._getInfo()
            self.sensors   = {}
        except (OSError, serial.SerialException):
            abort = True
            raise

    def _openPort(self):
        if self.serial != None and self.serial.isOpen():
            print("Closing serial port {}\n".format(self.port), file=sys.stderr)
            try:
                self.serial.close()
            except:
                pass

        print("Opening serial port ", self.port, file=sys.stderr)

        try:
            self.serial = serial.Serial(self.port, 57600)
            self.serial.flushInput()
        except OSError, e:
            print("Serial port " + self.port + " unavailable.  Is another process using it?", str(e), sep="\n", file=sys.stderr)
            raise
        except serial.SerialException, e:
            print("serial.SerialException:", str(e), "Is the correct USB port specified in config.xml?\n", sep="\n", file=sys.stderr)
            raise

    def run(self):
        global abort
        try:
            if self.printXML == True: # Just print XML to the screen
                while abort == False:
                    print( str(self.port), self.readLine(), sep="\n" )
            else:
                while abort == False:
                    self.update()
        except:
            abort = True
            raise

    def readLine(self):
        """Simply read a line from the serial port. Blocking.  On error, print useful message and raise the error."""
        try:
            line = self.serial.readline()
        except OSError, e: # catch errors raised by serial.readline
            print("Serial port " + self.port + " unavailable.  Is another process using it?", str(e), sep="\n", file=sys.stderr)
            raise
        except serial.SerialException, e:
            print("SerialException on port {}:".format(self.port), str(e), "Has the device been unplugged?\n", sep="\n", file=sys.stderr)
            raise
        except ValueError, e: # Attempting to use a port that is not open
            print("ValueError: ", str(e), sep="\n", file=sys.stderr)
            raise

        return line

    def resetSerial(self, i, RETRIES):
        """ Reset the serial port"""
        time.sleep(1)
        print("retrying... resetSerial number {} of {}\n".format(i, RETRIES), file=sys.stderr)

        # Try to flush the serial port.
        try:
            self.serial.flushInput()
        except: # Ignore errors.  We're going to retry anyway.
            pass

        # Try to re-open the port.
        try:
            self._openPort()
        except: # Ignore errors.  We're going to retry anyway.
            pass

    def readXML(self, data):
        """Reads a line from the serial port and returns an ElementTree"""
        RETRIES = 10
        for i in range(RETRIES):
            try:
                line = self.readLine()
                tree = ET.XML(line)

                # Check if this is histogram data from the current cost (which we're not interested in)
                # (This could also be done by checking the size of 'line' - this would probably be faster although
                #  possibly the size of a "histogram" is variable)
                if tree.findtext('hist') != None:
                    continue

                # Check if all the elements we're looking for exist in this XML
                success = True
                for key in data.keys():
                    data[key] = tree.findtext(key)
                    if data[key] == None:
                        success = False
                        print("Key \'{}\' not found in XML:\n{}".format(key, line), file=sys.stderr)
                        break

                if success:
                    return data
                else:
                    continue

            except (OSError, serial.SerialException, ValueError): # raised by readLine()
                self.resetSerial(i, RETRIES)
            except ET.ParseError, e: # Catch XML errors (occasionally the current cost outputs malformed XML)
                print("XML error: ", str(e), line, sep="\n", file=sys.stderr)
                self.resetSerial(i, RETRIES)

        # If we get to here then we have failed after every retry
        global abort
        abort = True
        raise Exception("readXML failed after {} retries".format(RETRIES))

    def _getInfo(self):
        """Get DSB and version number from Current Cost monitor"""
        data = {'dsb': None, 'src': None}
        data           = self.readXML( data )
        self.dsb       = data['dsb']
        self.CCversion = data['src']

    def writeToDisk(self, sensorNum):
        filename =  directory + "channel_" + self.id + str(sensorNum) + ".dat"
        try:
            filehandle = open(filename, 'a+')
        except Exception, e:
            print("Failed to open file {}.\n".format(filename), str(e), sep="\n", file=sys.stderr)
            raise

        filehandle.write( self.sensors[sensorNum].writeData() )
        filehandle.close()

    def update(self):
        """Read sensor data from serial port."""

        # For Current Cost XML details, see currentcost.com/cc128/xml.htm
        data = {'id': None, 'sensor': None, 'ch1/watts': None}

        data     = self.readXML( data )
        radioID  = int( data['id']        )
        sensor   = int( data['sensor']    )
        watts    = int( data['ch1/watts'] )

        if sensor not in self.sensors.keys():
            self.sensors[sensor] = Sensor()

        self.sensors[sensor].update(watts, radioID)

        self.writeToDisk( sensor )

    def __str__(self):
        string  = "----------------\n"
        string += "monitorID = {}\n".format(self.id)
        string += "port      = {}\n".format(self.port)
        string += "DSB       = {}\n".format(self.dsb)
        string += "Version   = {}\n\n".format(self.CCversion)
        string += "               -----PERIOD STATS (secs)--\n"
        string += "SENSOR  WATTS  MEAN    MAX    MIN CURRENT COUNT  RADIOID RADIOIDs\n\n"
        keys = self.sensors.keys()
        keys.sort()
        for key in keys:
            sensor  = self.sensors[key]
            string += '{:>5}{}\n'.format(key, str(sensor))

        string += "\n"
        return string

class TimeInfo(object):
    """Class for recording simple statistics about time. Useful during IAM testing."""

    def __init__(self):
        self.count    = -1
        self.lastSeen = 0

    def update(self):
        unixTime = time.time()
        self.count += 1
        self.current = unixTime - self.lastSeen

        if self.count == 0: # this is the first time we've run
            self.current = None
            self.mean = None
            self.max  = None
            self.min  = None
        elif self.count == 1:
            self.mean = self.current
            self.max  = self.current
            self.min  = self.current
        else:
            self.mean = ((float(self.mean) * (self.count - 1)) + self.current) / self.count
            if self.current > self.max: self.max = self.current
            if self.current < self.min: self.min = self.current

        self.lastSeen = unixTime

    def __str__(self):
        if self.count < 1:
            return "     -       -      -      -      {:d}".format(self.count)
        else:
            return '{:>7.2f}{:>7.1f}{:>7.1f}{:>7.1f}{:>7d}'.format(self.mean, self.max, self.min, self.current, self.count)

class Manager(object):
    """
    Singleton. Used to manage multiple CurrentCost objects.
    """

    def __init__(self, monitors, args):
        self.monitors = monitors
        self.args     = args

    def run(self):
        # Start each monitor thread
        for monitor in self.monitors:
            monitor.printXML = self.args.printXML
            monitor.start()

        # Use this main thread of control to continually
        # print out info
        if self.args.noDisplay or self.args.printXML:
            print("Press CTRL+C to stop.\n")
            signal.pause()
        else:
            while abort == False:
                os.system('clear')
                print(str(self))
                self.checkDuplicateRadioIDs()
                print("\nPress CTRL+C to stop.\n")
                time.sleep(1)

        self.stop()

    def stop(self):
        global abort
        abort = True

        print("Stopping...")

        # Don't exit the main thread until our worker threads have all quit
        for monitor in self.monitors:
            print("Waiting for monitor {} to stop...\n".format(monitor.id))
            monitor.join()

        print("Done.\n")

    def checkDuplicateRadioIDs(self):
        """
        Look across all monitors to check for duplicated radioIDs
        """

        # make a list of radioIDs
        radioIDs = set([])
        duplicateRadioIDs = set([])
        for monitor in self.monitors:
            for sensorNum, sensorData in monitor.sensors.iteritems():
                for radioID in sensorData.radioIDs.keys():
                    if radioID in radioIDs:
                        duplicateRadioIDs = set.union(duplicateRadioIDs, [radioID])
                    else:
                        radioIDs = set.union(radioIDs, [radioID])

        if len(duplicateRadioIDs) > 0:
            print("\nDuplicate radio IDs: {}\n".format(duplicateRadioIDs))

    def __str__(self):
        string = ""
        for monitor in self.monitors:
            string += str(monitor)

        return string

#########################################
#      LOAD CONFIG                      #
#########################################

def loadConfig():
    configTree    = ET.parse("config.xml") # load config from config file
    global directory
    directory     = configTree.findtext("directory") # File to save data to
    monitorsETree = configTree.findall("monitor")

    monitors = []

    for monitor in monitorsETree:
        currentCost    = CurrentCost( monitor.findtext("serialport") )
        currentCost.id = monitor.attrib['id']

        monitorMapping = monitor.findtext("mapping")
        mapping = monitorMapping.split("\n")
        mappingMatrix = []
        for line in mapping:
            strippedLine = line.strip()
            if strippedLine != '':
                mappingMatrix.append([i.strip() for i in strippedLine.split(',')])

        currentCost.mapping = mappingMatrix

        monitors.append( currentCost )

    return monitors

#########################################
#      HANDLE SIGINTs                   #
# So we do the right thing with CTRL+C  #
#########################################

def signalHandler(signal, frame):
    print("\nSIGINT received.")
    global abort
    abort = True

########################################
#  PROCESS COMMAND LINE ARGUMENTS      #
########################################

if __name__ == "__main__":
    # Process command line args
    parser = argparse.ArgumentParser(description='Log data from multiple Current Cost IAMs.')
    parser.add_argument('--noDisplay', dest='noDisplay', action='store_const',
                        const=True, default=False, help='Do not display info to std out. Useful for use with nohup command.')
    parser.add_argument('--printXML', dest='printXML', action='store_const',
                        const=True, default=False, help='Just dump XML from the monitor(s) to std out. Do not log data.')

    args = parser.parse_args()

    # load config
    monitors = loadConfig()

    # register SIGINT handler
    print("setting signal handler")
    signal.signal(signal.SIGINT, signalHandler)

    manager = Manager(monitors, args)

    try:
        manager.run()
    except:
        manager.stop()
        raise
<|endoftext|>