<commit_before>from django.dispatch import receiver
from django.db.models.signals import *
from farms.models import *
from irrigator_pro.settings import DEBUG

def minNone( *args ):
    args = filter( lambda x: x is not None, args)
    if args:
        return min(args)
    else:
        return None

## From
## http://stackoverflow.com/questions/15624817/have-loaddata-ignore-or-disable-post-save-signals

from functools import wraps
def disable_for_loaddata(signal_handler):
    """
    Decorator that turns off signal handlers when loading fixture data.
    """

    @wraps(signal_handler)
    def wrapper(*args, **kwargs):
        if kwargs.get('raw', None):
            return
        signal_handler(*args, **kwargs)
    return wrapper

## These signal handlers records the (earliest) relevant date of any
## created/changed/deleted object upon which calculation of
## WaterRegister entries depend.

@receiver(pre_save, sender=WaterHistory)
@receiver(pre_delete, sender=WaterHistory)
@disable_for_loaddata
def handler_WaterHistory(sender, instance, **kwargs):
    if instance.id: # save changes to existing object
        new_instance = instance
        old_instance = WaterHistory.objects.get(pk=instance.id)

        # content items changed: date, rain, irrigation
        content_changed = old_instance.date       != new_instance.date or \
                          old_instance.rain       != new_instance.rain or \
                          old_instance.irrigation != new_instance.irrigation

        if True: #content_changed:
            for field in old_instance.field_list.all():
                field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                                 old_instance.date)
                field.save()

        # field list changed
        removed_fields = set( old_instance.field_list.all() ) - \
                         set( new_instance.field_list.all() )

        added_fields   = set( new_instance.field_list.all() ) - \
                         set( old_instance.field_list.all() )

        for field in removed_fields:
            field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                             old_earliest_probereading_date)
            field.save()

        for field in added_fields:
            field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                             new_earliest_probereading_date)
            field.save()
    else:
        try:
            for field in instance.field_list.all():
                field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                                 instance.date)
                field.save()
        except ValueError:
            pass

@receiver(pre_save, sender=ProbeReading)
@receiver(pre_delete, sender=ProbeReading)
@disable_for_loaddata
def handler_ProbeReading(sender, instance, **kwargs):
    if instance.id: # save changes to existing object
        new_instance = instance
        old_instance = ProbeReading.objects.get(pk=instance.id)
        old_radio_id = old_instance.radio_id
        old_reading_datetime = old_instance.reading_datetime

        old_probe = Probe.objects.get(radio_id=old_radio_id,
                                      crop_season__season_start_date__lte=old_reading_datetime.date(),
                                      crop_season__season_end_date__gte=old_reading_datetime.date())

        for field in old_probe.field_list.all():
            field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                             old_instance.reading_datetime.date() )
            field.save()

    this_radio_id = instance.radio_id
    this_reading_datetime = instance.reading_datetime

    new_probe = Probe.objects.get(radio_id=this_radio_id,
                                  crop_season__season_start_date__lte=this_reading_datetime.date(),
                                  crop_season__season_end_date__gte=this_reading_datetime.date())

    for field in new_probe.field_list.all():
        field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                         instance.reading_datetime.date() )
        field.save()

@receiver(pre_save,   sender=CropSeasonEvent)
@receiver(pre_delete, sender=CropSeasonEvent)
@disable_for_loaddata
def handler_CropSeasonEvent(sender, instance, **kwargs):
    if instance.id: # save changes to existing object
        new_instance = instance
        old_instance = CropSeasonEvent.objects.get(pk=instance.id)
        old_instance.field.earliest_changed_dependency_date = old_instance.date
        old_instance.field.save()

    instance.field.earliest_changed_dependency_date = instance.date
    instance.field.save()

@receiver(pre_save, sender=CropSeason)
@receiver(pre_delete, sender=CropSeason)
@disable_for_loaddata
def handler_CropSeason(sender, instance, **kwargs):
    if instance.id: # save changes to existing object
        new_instance = instance
        old_instance = CropSeason.objects.get(pk=instance.id)

        old_date = None
        new_date = None
        if old_instance.season_start_date != new_instance.season_start_date:
            old_date = minNone(old_date, old_instance.season_start_date)
            new_date = minNone(new_date, new_instance.season_start_date)

        if old_instance.season_end_date != new_instance.season_end_date:
            old_date = minNone(old_date, old_instance.season_end_date)
            new_date = minNone(new_date, new_instance.season_end_date)

        if old_instance.crop != new_instance.crop:
            old_date = old_instance.season_start_date
            new_date = new_instance.season_start_date

        if old_date is not None:
            for field in old_instance.field_list.all():
                field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                                 old_date)
                field.save()

        if new_date is not None:
            for field in new_instance.field_list.all():
                field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                                 new_date)
                field.save()

        removed_fields = set( old_instance.field_list.all() ) - \
                         set( new_instance.field_list.all() )

        added_fields   = set( new_instance.field_list.all() ) - \
                         set( old_instance.field_list.all() )

        for field in removed_fields:
            field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                             old_instance.season_start_date)
            field.save()

        for field in added_fields:
            field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                             new_instance.season_start_date)
            field.save()

    else:
        pass

@receiver(post_save, sender=CropSeason)
@disable_for_loaddata
def handler_CropSeason_postsave(sender, instance, created, **kwargs):
    if created == True:
        for field in instance.field_list.all():
            field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                             instance.season_start_date)
            field.save()

@receiver(pre_save,   sender=Probe)
@receiver(pre_delete, sender=Probe)
@disable_for_loaddata
def handler_Probe(sender, instance,  **kwargs):
    if instance.id:  # save changes to existing object
        new_instance = instance
        old_instance = Probe.objects.get(pk=instance.id)

        old_radio_id = old_instance.radio_id
        old_season_start_date = old_instance.crop_season.season_start_date
        old_season_end_date   = old_instance.crop_season.season_end_date
        old_probereadings     = ProbeReading.objects.filter(radio_id=old_radio_id,
                                                             reading_datetime__range=(old_season_start_date,
                                                                                old_season_end_date)
                                                             )
        if old_probereadings:
            old_earliest_probereading_date = old_probereadings.earliest('reading_datetime').reading_datetime.date();
        else:
            old_earliest_probereading_date = None;

        new_radio_id = new_instance.radio_id
        new_season_start_date = new_instance.crop_season.season_start_date
        new_season_end_date   = new_instance.crop_season.season_end_date
        new_probereadings     = ProbeReading.objects.filter(radio_id=new_radio_id,
                                                             reading_datetime__range=(new_season_start_date,
                                                                                new_season_end_date)
                                                             )
        if new_probereadings:
            new_earliest_probereading_date = new_probereadings.earliest('reading_datetime').reading_datetime.date();
        else:
            new_earliest_probereading_date = None;

        if old_radio_id != new_radio_id:  # changed radioid
            if old_instance.id and old_instance.field_list:
                for field in old_instance.field_list.all():
                    field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                                     old_earliest_probereading_date)
                    field.save()

            if new_instance.id and new_instance.field_list:
                for field in new_instance.field_list.all():
                    field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                                     new_earliest_probereading_date)
                    field.save()

        removed_fields = set( old_instance.field_list.all() ) - \
                         set( new_instance.field_list.all() )

        added_fields   = set( new_instance.field_list.all() ) - \
                         set( old_instance.field_list.all() )

        for field in removed_fields:
            field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                             old_earliest_probereading_date)
            field.save()

        for field in added_fields:
            field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                             new_earliest_probereading_date)
            field.save()
    else: # new object or delete object
        radio_id = instance.radio_id
        season_start_date = instance.crop_season.season_start_date
        season_end_date   = instance.crop_season.season_end_date
        probereadings     = ProbeReading.objects.filter(radio_id=radio_id,
                                                             reading_datetime__range=(season_start_date,
                                                                                season_end_date)
                                                             )
        if probereadings:
            earliest_probereading_date = probereadings.earliest('reading_datetime').reading_datetime.date();
        else:
            earliest_probereading_date = None;

        if instance.id and instance.field_list:
            for field in instance.field_list.all():
                field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                                 earliest_probereading_date)
                field.save()
<commit_msg>In signal.py, don't store probe_reading.reading_datetime for comparison, store probe_reading.reading_datetime.date().<commit_after>from django.dispatch import receiver
from django.db.models.signals import *
from farms.models import *
from irrigator_pro.settings import DEBUG

def minNone( *args ):
    args = filter( lambda x: x is not None, args)
    if args:
        return min(args)
    else:
        return None

## From
## http://stackoverflow.com/questions/15624817/have-loaddata-ignore-or-disable-post-save-signals

from functools import wraps
def disable_for_loaddata(signal_handler):
    """
    Decorator that turns off signal handlers when loading fixture data.
    """

    @wraps(signal_handler)
    def wrapper(*args, **kwargs):
        if kwargs.get('raw', None):
            return
        signal_handler(*args, **kwargs)
    return wrapper

## These signal handlers records the (earliest) relevant date of any
## created/changed/deleted object upon which calculation of
## WaterRegister entries depend.

@receiver(pre_save, sender=WaterHistory)
@receiver(pre_delete, sender=WaterHistory)
@disable_for_loaddata
def handler_WaterHistory(sender, instance, **kwargs):
    if instance.id: # save changes to existing object
        new_instance = instance
        old_instance = WaterHistory.objects.get(pk=instance.id)

        # content items changed: date, rain, irrigation
        content_changed = old_instance.date       != new_instance.date or \
                          old_instance.rain       != new_instance.rain or \
                          old_instance.irrigation != new_instance.irrigation

        if True: #content_changed:
            for field in old_instance.field_list.all():
                field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                                 old_instance.date)
                field.save()

        # field list changed
        removed_fields = set( old_instance.field_list.all() ) - \
                         set( new_instance.field_list.all() )

        added_fields   = set( new_instance.field_list.all() ) - \
                         set( old_instance.field_list.all() )

        for field in removed_fields:
            field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                             old_earliest_probereading_date)
            field.save()

        for field in added_fields:
            field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                             new_earliest_probereading_date)
            field.save()
    else:
        try:
            for field in instance.field_list.all():
                field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                                 instance.date)
                field.save()
        except ValueError:
            pass

@receiver(pre_save, sender=ProbeReading)
@receiver(pre_delete, sender=ProbeReading)
@disable_for_loaddata
def handler_ProbeReading(sender, instance, **kwargs):
    if instance.id: # save changes to existing object
        new_instance = instance
        old_instance = ProbeReading.objects.get(pk=instance.id)
        old_radio_id = old_instance.radio_id
        old_reading_date = old_instance.reading_datetime.date()

        old_probe = Probe.objects.get(radio_id=old_radio_id,
                                      crop_season__season_start_date__lte=old_reading_date,
                                      crop_season__season_end_date__gte=old_reading_date)

        for field in old_probe.field_list.all():
            field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                             old_instance.reading_datetime.date() )
            field.save()

    this_radio_id = instance.radio_id
    this_reading_date = instance.reading_datetime.date()

    new_probe = Probe.objects.get(radio_id=this_radio_id,
                                  crop_season__season_start_date__lte=this_reading_date,
                                  crop_season__season_end_date__gte=this_reading_date)

    for field in new_probe.field_list.all():
        field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                         instance.reading_datetime.date() )
        field.save()

@receiver(pre_save,   sender=CropSeasonEvent)
@receiver(pre_delete, sender=CropSeasonEvent)
@disable_for_loaddata
def handler_CropSeasonEvent(sender, instance, **kwargs):
    if instance.id: # save changes to existing object
        new_instance = instance
        old_instance = CropSeasonEvent.objects.get(pk=instance.id)
        old_instance.field.earliest_changed_dependency_date = old_instance.date
        old_instance.field.save()

    instance.field.earliest_changed_dependency_date = instance.date
    instance.field.save()

@receiver(pre_save, sender=CropSeason)
@receiver(pre_delete, sender=CropSeason)
@disable_for_loaddata
def handler_CropSeason(sender, instance, **kwargs):
    if instance.id: # save changes to existing object
        new_instance = instance
        old_instance = CropSeason.objects.get(pk=instance.id)

        old_date = None
        new_date = None
        if old_instance.season_start_date != new_instance.season_start_date:
            old_date = minNone(old_date, old_instance.season_start_date)
            new_date = minNone(new_date, new_instance.season_start_date)

        if old_instance.season_end_date != new_instance.season_end_date:
            old_date = minNone(old_date, old_instance.season_end_date)
            new_date = minNone(new_date, new_instance.season_end_date)

        if old_instance.crop != new_instance.crop:
            old_date = old_instance.season_start_date
            new_date = new_instance.season_start_date

        if old_date is not None:
            for field in old_instance.field_list.all():
                field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                                 old_date)
                field.save()

        if new_date is not None:
            for field in new_instance.field_list.all():
                field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                                 new_date)
                field.save()

        removed_fields = set( old_instance.field_list.all() ) - \
                         set( new_instance.field_list.all() )

        added_fields   = set( new_instance.field_list.all() ) - \
                         set( old_instance.field_list.all() )

        for field in removed_fields:
            field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                             old_instance.season_start_date)
            field.save()

        for field in added_fields:
            field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                             new_instance.season_start_date)
            field.save()

    else:
        pass

@receiver(post_save, sender=CropSeason)
@disable_for_loaddata
def handler_CropSeason_postsave(sender, instance, created, **kwargs):
    if created == True:
        for field in instance.field_list.all():
            field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                             instance.season_start_date)
            field.save()

@receiver(pre_save,   sender=Probe)
@receiver(pre_delete, sender=Probe)
@disable_for_loaddata
def handler_Probe(sender, instance,  **kwargs):
    if instance.id:  # save changes to existing object
        new_instance = instance
        old_instance = Probe.objects.get(pk=instance.id)

        old_radio_id = old_instance.radio_id
        old_season_start_date = old_instance.crop_season.season_start_date
        old_season_end_date   = old_instance.crop_season.season_end_date
        old_probereadings     = ProbeReading.objects.filter(radio_id=old_radio_id,
                                                             reading_datetime__range=(old_season_start_date,
                                                                                old_season_end_date)
                                                             )
        if old_probereadings:
            old_earliest_probereading_date = old_probereadings.earliest('reading_datetime').reading_datetime.date();
        else:
            old_earliest_probereading_date = None;

        new_radio_id = new_instance.radio_id
        new_season_start_date = new_instance.crop_season.season_start_date
        new_season_end_date   = new_instance.crop_season.season_end_date
        new_probereadings     = ProbeReading.objects.filter(radio_id=new_radio_id,
                                                             reading_datetime__range=(new_season_start_date,
                                                                                new_season_end_date)
                                                             )
        if new_probereadings:
            new_earliest_probereading_date = new_probereadings.earliest('reading_datetime').reading_datetime.date();
        else:
            new_earliest_probereading_date = None;

        if old_radio_id != new_radio_id:  # changed radioid
            if old_instance.id and old_instance.field_list:
                for field in old_instance.field_list.all():
                    field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                                     old_earliest_probereading_date)
                    field.save()

            if new_instance.id and new_instance.field_list:
                for field in new_instance.field_list.all():
                    field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                                     new_earliest_probereading_date)
                    field.save()

        removed_fields = set( old_instance.field_list.all() ) - \
                         set( new_instance.field_list.all() )

        added_fields   = set( new_instance.field_list.all() ) - \
                         set( old_instance.field_list.all() )

        for field in removed_fields:
            field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                             old_earliest_probereading_date)
            field.save()

        for field in added_fields:
            field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                             new_earliest_probereading_date)
            field.save()
    else: # new object or delete object
        radio_id = instance.radio_id
        season_start_date = instance.crop_season.season_start_date
        season_end_date   = instance.crop_season.season_end_date
        probereadings     = ProbeReading.objects.filter(radio_id=radio_id,
                                                             reading_datetime__range=(season_start_date,
                                                                                season_end_date)
                                                             )
        if probereadings:
            earliest_probereading_date = probereadings.earliest('reading_datetime').reading_datetime.date();
        else:
            earliest_probereading_date = None;

        if instance.id and instance.field_list:
            for field in instance.field_list.all():
                field.earliest_changed_dependency_date = minNone(field.earliest_changed_dependency_date,
                                                                 earliest_probereading_date)
                field.save()
<|endoftext|>


<commit_before>import sys
import time
import multiprocessing
from distutils.version import LooseVersion, StrictVersion

import marathon
from behave import given, when, then

from itest_utils import get_marathon_connection_string
sys.path.append('../')

@given('a working marathon instance')
def working_marathon(context):
    """Adds a working marathon client as context.client for the purposes of
    interacting with it in the test."""
    if not hasattr(context, 'client'):
        marathon_connection_string = "http://%s" % \
            get_marathon_connection_string()
        context.client = marathon.MarathonClient(marathon_connection_string)

@then(u'we get the marathon instance\'s info')
def get_marathon_info(context):
    assert context.client.get_info()

@when(u'we create a trivial new app')
def create_trivial_new_app(context):
    context.client.create_app('test-trivial-app', marathon.MarathonApp(
        cmd='sleep 3600', mem=16, cpus=0.1, instances=5))

@then(u'we should be able to kill the tasks')
def kill_a_task(context):
    time.sleep(5)
    app = context.client.get_app('test-trivial-app')
    tasks = app.tasks
    context.client.kill_task(
        app_id='test-trivial-app', task_id=tasks[0].id, scale=True)

@when(u'we create a complex new app')
def create_complex_new_app_with_unicode(context):
    app_config = {
        'container': {
            'type': 'DOCKER',
            'docker': {
                'portMappings':
                    [{'protocol': 'tcp',
                      'containerPort': 8888,
                      'hostPort': 0}],
                'image': u'localhost/fake_docker_url',
                'network': 'BRIDGE',
                'parameters': [{'key': 'add-host', 'value': 'google-public-dns-a.google.com:8.8.8.8'}],
            },
            'volumes':
                [{'hostPath': u'/etc/stuff',
                 'containerPath': u'/etc/stuff',
                  'mode': 'RO'}],
        },
        'instances': 1,
        'mem': 30,
        'args': [],
        'backoff_factor': 2,
        'cpus': 0.25,
        'uris': ['file:///root/.dockercfg'],
        'backoff_seconds': 1,
        'constraints': None,
        'cmd': u'/bin/true',
        'health_checks': [
            {
                'protocol': 'HTTP',
                'path': '/health',
                'gracePeriodSeconds': 3,
                'intervalSeconds': 10,
                'portIndex': 0,
                'timeoutSeconds': 10,
                'maxConsecutiveFailures': 3
            },
        ],
    }
    context.client.create_app(
        'test-complex-app', marathon.MarathonApp(**app_config))

@then(u'we should see the {which} app running via the marathon api')
def see_complext_app_running(context, which):
    print(context.client.list_apps())
    assert context.client.get_app('test-%s-app' % which)

@when(u'we wait the {which} app deployment finish')
def wait_deployment_finish(context, which):
    while True:
        time.sleep(1)
        app = context.client.get_app('test-%s-app' % which, embed_tasks=True)
        if not app.deployments:
            break

@then(u'we should be able to kill the #{to_kill} tasks of the {which} app')
def kill_tasks(context, to_kill, which):
    app_tasks = context.client.get_app(
        'test-%s-app' % which, embed_tasks=True).tasks

    index_to_kill = eval("[" + to_kill + "]")
    task_to_kill = [app_tasks[index].id for index in index_to_kill]

    context.client.kill_given_tasks(task_to_kill)

@then(u'we should be able to list tasks of the {which} app')
def list_tasks(context, which):
    app = context.client.get_app('test-%s-app' % which)
    tasks = context.client.list_tasks('test-%s-app' % which)
    assert len(tasks) == app.instances

def listen_for_events(client, events):
    for msg in client.event_stream():
        events.append(msg)

@when(u'marathon version is greater than {version}')
def marathon_version_chech(context, version):
    info = context.client.get_info()
    if StrictVersion(info.version) < StrictVersion(version):
        context.scenario.skip(reason='Marathon version is too low for this scenario')

@when(u'we start listening for events')
def start_listening_stream(context):
    manager = multiprocessing.Manager()
    mlist = manager.list()
    context.manager = manager
    context.events = mlist
    p = multiprocessing.Process(target=listen_for_events, args=(context.client, mlist))
    p.start()
    context.p = p

@then(u'we should see list of events')
def stop_listening_stream(context):
    time.sleep(10)
    context.p.terminate()

    # event list should contain 5 status_update_event with taskStatus == TASK_RUNNING
    filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_RUNNING"]
    assert len(filtered_events) == 5

    # and 1 status_update_event with taskStatus == TASK_KILLED
    filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_KILLED"]
    assert len(filtered_events) == 1

    # and 1 deployment_step_success events with target instances == 5
    filtered_events = [e for e in context.events if e.event_type == "deployment_step_success" and e.plan.target.apps[0]['instances'] == 5]
    assert len(filtered_events) == 1

    # and 1 deployment_step_success events with target instances == 4
    filtered_events = [e for e in context.events if e.event_type == "deployment_step_success" and e.plan.target.apps[0]['instances'] == 4]
    assert len(filtered_events) == 1
<commit_msg>Reorganize test slightly<commit_after>import sys
import time
import multiprocessing
from distutils.version import LooseVersion, StrictVersion

import marathon
from behave import given, when, then

from itest_utils import get_marathon_connection_string
sys.path.append('../')

@given('a working marathon instance')
def working_marathon(context):
    """Adds a working marathon client as context.client for the purposes of
    interacting with it in the test."""
    if not hasattr(context, 'client'):
        marathon_connection_string = "http://%s" % \
            get_marathon_connection_string()
        context.client = marathon.MarathonClient(marathon_connection_string)

@then(u'we get the marathon instance\'s info')
def get_marathon_info(context):
    assert context.client.get_info()

@when(u'we create a trivial new app')
def create_trivial_new_app(context):
    context.client.create_app('test-trivial-app', marathon.MarathonApp(
        cmd='sleep 3600', mem=16, cpus=0.1, instances=5))

@then(u'we should be able to kill the tasks')
def kill_a_task(context):
    time.sleep(5)
    app = context.client.get_app('test-trivial-app')
    tasks = app.tasks
    context.client.kill_task(
        app_id='test-trivial-app', task_id=tasks[0].id, scale=True)

@when(u'we create a complex new app')
def create_complex_new_app_with_unicode(context):
    app_config = {
        'container': {
            'type': 'DOCKER',
            'docker': {
                'portMappings':
                    [{'protocol': 'tcp',
                      'containerPort': 8888,
                      'hostPort': 0}],
                'image': u'localhost/fake_docker_url',
                'network': 'BRIDGE',
                'parameters': [{'key': 'add-host', 'value': 'google-public-dns-a.google.com:8.8.8.8'}],
            },
            'volumes':
                [{'hostPath': u'/etc/stuff',
                 'containerPath': u'/etc/stuff',
                  'mode': 'RO'}],
        },
        'instances': 1,
        'mem': 30,
        'args': [],
        'backoff_factor': 2,
        'cpus': 0.25,
        'uris': ['file:///root/.dockercfg'],
        'backoff_seconds': 1,
        'constraints': None,
        'cmd': u'/bin/true',
        'health_checks': [
            {
                'protocol': 'HTTP',
                'path': '/health',
                'gracePeriodSeconds': 3,
                'intervalSeconds': 10,
                'portIndex': 0,
                'timeoutSeconds': 10,
                'maxConsecutiveFailures': 3
            },
        ],
    }
    context.client.create_app(
        'test-complex-app', marathon.MarathonApp(**app_config))

@then(u'we should see the {which} app running via the marathon api')
def see_complext_app_running(context, which):
    print(context.client.list_apps())
    assert context.client.get_app('test-%s-app' % which)

@when(u'we wait the {which} app deployment finish')
def wait_deployment_finish(context, which):
    while True:
        time.sleep(1)
        app = context.client.get_app('test-%s-app' % which, embed_tasks=True)
        if not app.deployments:
            break

@then(u'we should be able to kill the #{to_kill} tasks of the {which} app')
def kill_tasks(context, to_kill, which):
    app_tasks = context.client.get_app(
        'test-%s-app' % which, embed_tasks=True).tasks

    index_to_kill = eval("[" + to_kill + "]")
    task_to_kill = [app_tasks[index].id for index in index_to_kill]

    context.client.kill_given_tasks(task_to_kill)

@then(u'we should be able to list tasks of the {which} app')
def list_tasks(context, which):
    app = context.client.get_app('test-%s-app' % which)
    tasks = context.client.list_tasks('test-%s-app' % which)
    assert len(tasks) == app.instances

def listen_for_events(client, events):
    for msg in client.event_stream():
        events.append(msg)

@when(u'marathon version is greater than {version}')
def marathon_version_chech(context, version):
    info = context.client.get_info()
    if StrictVersion(info.version) < StrictVersion(version):
        context.scenario.skip(reason='Marathon version is too low for this scenario')

@when(u'we start listening for events')
def start_listening_stream(context):
    manager = multiprocessing.Manager()
    mlist = manager.list()
    context.manager = manager
    context.events = mlist
    p = multiprocessing.Process(target=listen_for_events, args=(context.client, mlist))
    p.start()
    context.p = p

@then(u'we should see list of events')
def stop_listening_stream(context):
    time.sleep(10)
    context.p.terminate()

    # event list should contain 5 status_update_event with taskStatus == TASK_RUNNING
    filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_RUNNING"]
    assert len(filtered_events) == 5

    # and 1 deployment_step_success events with target instances == 5
    filtered_events = [e for e in context.events if e.event_type == "deployment_step_success" and e.plan.target.apps[0]['instances'] == 5]
    assert len(filtered_events) == 1

    # and 1 status_update_event with taskStatus == TASK_KILLED
    filtered_events = [e for e in context.events if e.event_type == "status_update_event" and e.task_status == "TASK_KILLED"]
    assert len(filtered_events) == 1

    # and 1 deployment_step_success events with target instances == 4
    filtered_events = [e for e in context.events if e.event_type == "deployment_step_success" and e.plan.target.apps[0]['instances'] == 4]
    assert len(filtered_events) == 1
<|endoftext|>


<commit_before>from plugin import plugin, require
import requests
from bs4 import BeautifulSoup

valid_genres = [
        "comedy", "sci-fi", "horror",
        "romance", "action", "thriller",
        "drama", "mystery", "crime",
        "animation", "adventure", "fantasy",
        "comedy,romance", "action,comedy",
        "superhero"
        ]

@require(network=True)
@plugin("topmedia")
class topmedia():
    """
    Plugin to extract most popular movies and TV shows from IMDB by genre
    """

    def __call__(self, jarvis, medium):
        valid_mediums = ["tv", "movies", "television", "movie", "cinema"]
        medium = medium.lower()

        if medium not in valid_mediums:
            jarvis.say("Please run the command with a valid medium.\nExample: topmedia tv")
        else:
            for i in range(len(valid_genres)):
                jarvis.say(str(i+1) + "." + " " + valid_genres[i])
            # .lower() to take care of accidental capitalisations
            # ? Might be useful to allow the user to choose a genre by number.
            # ? For example, if a user enters 1, we direct the user to top comedy i.e. valid_genres[0] movies
            user_genre = jarvis.input("Please choose a genre from one of the genres above:\n").lower()
            if user_genre not in valid_genres:
                jarvis.say("The genre you have entered is not valid.\nPlease try again.")
            else:    # the medium is already validated, so it's either tv or movie
                if medium == "tv" or medium == "television":
                    self.top_250_extractor(jarvis, user_genre, "tv")
                else:
                    self.top_250_extractor(jarvis, user_genre, "movie")

    def top_250_extractor(self,jarvis, genre, medium):
        if medium == "tv":
            title_type = "tvSeries"
        else:
            title_type = "movie"
        # Each page on IMDB contains 50 movies. So, start = 1 implies movies ranked 1-50.
        for start in [1,51,101,151,201]:
            url = ("https://www.imdb.com/search/title/?genres="
                + genre
                + "&start="
                + str(start)
                + "&explore=title_type,genres&title_type="
                + title_type)
            r = requests.get(url)
            soup = BeautifulSoup(r.content, "html.parser")
            table = soup.find("div", attrs = {"class":"lister-list"})
            i = 0
            for row2 in table.findAll("h3", attrs = {"class":"lister-item-header"}):
                #Printing rank and name of the movie/show.
                jarvis.say(str(start + i) + ". " + row2.a.text)    # The a tag has the name of the movie/show
                i += 1<commit_msg>Enforce PEP8 on top_media.py<commit_after>from plugin import plugin, require
import requests
from bs4 import BeautifulSoup

valid_genres = [
    "comedy", "sci-fi", "horror",
    "romance", "action", "thriller",
    "drama", "mystery", "crime",
    "animation", "adventure", "fantasy",
    "comedy,romance", "action,comedy",
    "superhero"
]

@require(network=True)
@plugin("topmedia")
class topmedia():
    """
    Plugin to extract most popular movies and TV shows from IMDB by genre
    """

    def __call__(self, jarvis, medium):
        valid_mediums = ["tv", "movies", "television", "movie", "cinema"]
        medium = medium.lower()

        if medium not in valid_mediums:
            jarvis.say("Please run the command with a valid medium.\nExample: topmedia tv")
        else:
            for i in range(len(valid_genres)):
                jarvis.say(str(i + 1) + "." + " " + valid_genres[i])
            # .lower() to take care of accidental capitalisations
            # ? Might be useful to allow the user to choose a genre by number.
            # ? For example, if a user enters 1, we direct the user to top comedy i.e. valid_genres[0] movies
            user_genre = jarvis.input("Please choose a genre from one of the genres above:\n").lower()
            if user_genre not in valid_genres:
                jarvis.say("The genre you have entered is not valid.\nPlease try again.")
            else:  # the medium is already validated, so it's either tv or movie
                if medium == "tv" or medium == "television":
                    self.top_250_extractor(jarvis, user_genre, "tv")
                else:
                    self.top_250_extractor(jarvis, user_genre, "movie")

    def top_250_extractor(self, jarvis, genre, medium):
        if medium == "tv":
            title_type = "tvSeries"
        else:
            title_type = "movie"
            # Each page on IMDB contains 50 movies. So, start = 1 implies movies ranked 1-50.
        for start in [1, 51, 101, 151, 201]:
            url = ("https://www.imdb.com/search/title/?genres="
                   + genre
                   + "&start="
                   + str(start)
                   + "&explore=title_type,genres&title_type="
                   + title_type)
            r = requests.get(url)
            soup = BeautifulSoup(r.content, "html.parser")
            table = soup.find("div", attrs={"class": "lister-list"})
            i = 0
            for row2 in table.findAll("h3", attrs={"class": "lister-item-header"}):
                # Printing rank and name of the movie/show.
                jarvis.say(str(start + i) + ". " + row2.a.text)  # The a tag has the name of the movie/show
                i += 1
<|endoftext|>


<commit_before>'''
Created on Mar 2, 2016

@author: loongchan
'''
from pykml import parser
from pykml.parser import Schema
from geopy.distance import vincenty
import utm, os.path, csv
#from nis import cat

'''
Tool to convert from UTM to lat/long
'''
def utm_to_latlong(easting, northing, zone_number = 10, zone_letter = 'U'):
    try:
        return utm.to_latlon(easting, northing, zone_number, zone_letter)
    except:
        print "You might need to install utm.. 'pip install utm'"

def latlong_to_utm(lat, long):
    try:
        return utm.from_latlon(lat, long)
    except:
        print "You might need to install utm.. 'pip install utm'"

def load_kml_file(filename, schema_type = "kml22gx.xsd"):
    with open(filename) as fh:
        schema_gx = Schema(schema_type)
        doc = parser.parse(fh)

        if (schema_gx.validate(doc)):
            return doc.getroot()
        else:
            print "Invalid kml format for file: "+str(filename)
            return None

def load_kml_string(kml_string, schema_type = "kml22gx.xsd"):
    schema_gx = Schema(schema_type)
    doc = parser.fromstring(kml_string, schema_gx)

    if (schema_gx.validate(doc)):
        return doc
    else:
        print "Invalid kml format for string passed in"
        return None

def kml_to_csv(kml_file, use_headers=True, output_filename = 'output.csv'):
    #if file, then load it, else it's a string
    root = None
    if os.path.isfile(kml_file):
        root =load_kml_file(kml_file)
    else:
        root = load_kml_string(kml_file)

    # now we start parsing the xml... isn't there a better way???
    with open(output_filename, 'wb') as fh:
        csv_out = csv.writer(fh)

        #get headers if we want it
        if use_headers:
            header_line = list();
            for child in root.Document.Folder.Placemark.ExtendedData.iter():
                if 'name' in dict(child.attrib):
                    header_line.append(child.attrib['name'])
            header_line.append('LAT')
            header_line.append('LONG')
            csv_out.writerow(header_line)

        #we save all data AND location via lat/long
        for child in root.Document.Folder.iter():
            if 'Placemark' in child.tag:
                line_str = list()
                for gchild in child.iter():
                    if 'ExtendedData' in gchild.tag:
                        for ggchild in child.iter():
                            if 'SimpleData' in ggchild.tag:
                                line_str.append(ggchild.text)
                    if 'Point' in gchild.tag:
                        coords_split = str(gchild.coordinates).split(',')
                        line_str.append(coords_split[1])
                        line_str.append(coords_split[0])

                csv_out.writerow(line_str)

def number_of_graffiti_points(latitude, longitude, radius1=50, radius2=100, graffiti_fh=None):
    graffiti_file = '../data/graffiti/graffiti.csv'

    # check if graffiti_fh is NOT empty
    open_myself = False
    if graffiti_fh is None:
        graffiti_fh = open(graffiti_file, 'rb')
        open_myself = True

    graffiti_fh.seek(0)
    graffiti_reader = csv.DictReader(graffiti_fh)

    # yeah, we check the point against all 8k+ rows..yeah, yeah....
    count1 = 0
    count2 = 0
    for row in graffiti_reader:
        latlong_diff = vincenty((latitude, longitude), (row['LAT'],row['LONG']))

        #set counts based on radius1 and radius2
        if latlong_diff.m < radius1:
            count1 = count1 + int(row['COUNT'])

        if latlong_diff < radius2:
            count2 = count2 + int(row['COUNT'])

    #now we got the counts, we need to clean up and return
    if open_myself:
        graffiti_fh.close()

    return (count1, count2)

# takes lat/long and returns distance from nearest (Adult, Men, Women/Families, Youth)
def number_of_homeless_shelters_at(latitude, longitude, homeless_fh=None):
    homeless_file = '../data/homeless_shelters/doc.csv'

    # check if graffiti_fh is NOT empty
    open_myself = False
    if homeless_fh is None:
        homeless_fh = open(homeless_file, 'rb')
        open_myself = True

    homeless_fh.seek(0)
    homeless_reader = csv.DictReader(homeless_fh)

    # yeah, we check the point against all 8k+ rows..yeah, yeah....
    keys = ['adults', 'men', 'women/families', 'youth']
    retval = dict((x, float('Inf')) for x in keys)
    for row in homeless_reader:
        latlong_diff = vincenty((latitude, longitude), (row['LAT'],row['LONG']))

        #we need to consider the "category" of which there is 4.
        cat = row['CATEGORY'].split(' ')[0].lower()
        if retval[cat] > latlong_diff.m:
            retval[cat] = latlong_diff.m

    # now we get the tuple from dict in order, since for feature vector we care about that kind of stuff
    final_retval = tuple()
    for c in keys:
        final_retval = final_retval + (retval[c],)

    #now we got the counts, we need to clean up and return
    if open_myself:
        homeless_fh.close()

    return final_retval

def number_of_street_lights_at(latitude, longitude, light_fh=None, radius1=50):
    light_file = '../data/street_lightings/street_lighting_poles.csv'

    # check if graffiti_fh is NOT empty
    open_myself = False
    if light_fh is None:
        light_fh = open(light_file, 'rb')
        open_myself = True

    light_fh.seek(0)
    light_reader = csv.DictReader(light_fh)

    # yeah, we check the point against all 8k+ rows..yeah, yeah....
    count1 = 0
    for row in light_reader:
        latlong_diff = vincenty((latitude, longitude), (row['LAT'],row['LONG']))

        #set counts based on radius1
        if latlong_diff.m < radius1:
            count1 = count1 + 1

    #now we got the counts, we need to clean up and return
    if open_myself:
        light_fh.close()

    return (count1,)

<commit_msg>Fixed error when checking if difference is less than radius2 -probably won't use this function due to being too slow<commit_after>'''
Created on Mar 2, 2016

@author: loongchan
'''
from pykml import parser
from pykml.parser import Schema
from geopy.distance import vincenty
import utm, os.path, csv
#from nis import cat

'''
Tool to convert from UTM to lat/long
'''
def utm_to_latlong(easting, northing, zone_number = 10, zone_letter = 'U'):
    try:
        return utm.to_latlon(easting, northing, zone_number, zone_letter)
    except:
        print "You might need to install utm.. 'pip install utm'"

def latlong_to_utm(lat, long):
    try:
        return utm.from_latlon(lat, long)
    except:
        print "You might need to install utm.. 'pip install utm'"

def load_kml_file(filename, schema_type = "kml22gx.xsd"):
    with open(filename) as fh:
        schema_gx = Schema(schema_type)
        doc = parser.parse(fh)

        if (schema_gx.validate(doc)):
            return doc.getroot()
        else:
            print "Invalid kml format for file: "+str(filename)
            return None

def load_kml_string(kml_string, schema_type = "kml22gx.xsd"):
    schema_gx = Schema(schema_type)
    doc = parser.fromstring(kml_string, schema_gx)

    if (schema_gx.validate(doc)):
        return doc
    else:
        print "Invalid kml format for string passed in"
        return None

def kml_to_csv(kml_file, use_headers=True, output_filename = 'output.csv'):
    #if file, then load it, else it's a string
    root = None
    if os.path.isfile(kml_file):
        root =load_kml_file(kml_file)
    else:
        root = load_kml_string(kml_file)

    # now we start parsing the xml... isn't there a better way???
    with open(output_filename, 'wb') as fh:
        csv_out = csv.writer(fh)

        #get headers if we want it
        if use_headers:
            header_line = list();
            for child in root.Document.Folder.Placemark.ExtendedData.iter():
                if 'name' in dict(child.attrib):
                    header_line.append(child.attrib['name'])
            header_line.append('LAT')
            header_line.append('LONG')
            csv_out.writerow(header_line)

        #we save all data AND location via lat/long
        for child in root.Document.Folder.iter():
            if 'Placemark' in child.tag:
                line_str = list()
                for gchild in child.iter():
                    if 'ExtendedData' in gchild.tag:
                        for ggchild in child.iter():
                            if 'SimpleData' in ggchild.tag:
                                line_str.append(ggchild.text)
                    if 'Point' in gchild.tag:
                        coords_split = str(gchild.coordinates).split(',')
                        line_str.append(coords_split[1])
                        line_str.append(coords_split[0])

                csv_out.writerow(line_str)

def number_of_graffiti_points(latitude, longitude, radius1=50, radius2=100, graffiti_fh=None):
    graffiti_file = '../data/graffiti/graffiti.csv'

    # check if graffiti_fh is NOT empty
    open_myself = False
    if graffiti_fh is None:
        graffiti_fh = open(graffiti_file, 'rb')
        open_myself = True

    graffiti_fh.seek(0)
    graffiti_reader = csv.DictReader(graffiti_fh)

    # yeah, we check the point against all 8k+ rows..yeah, yeah....
    count1 = 0
    count2 = 0
    for row in graffiti_reader:
        latlong_diff = vincenty((latitude, longitude), (row['LAT'],row['LONG']))

        #set counts based on radius1 and radius2
        if latlong_diff.m < radius1:
            count1 = count1 + int(row['COUNT'])

        if latlong_diff.m < radius2:
            count2 = count2 + int(row['COUNT'])

    #now we got the counts, we need to clean up and return
    if open_myself:
        graffiti_fh.close()

    return (count1, count2)

# takes lat/long and returns distance from nearest (Adult, Men, Women/Families, Youth)
def number_of_homeless_shelters_at(latitude, longitude, homeless_fh=None):
    homeless_file = '../data/homeless_shelters/doc.csv'

    # check if graffiti_fh is NOT empty
    open_myself = False
    if homeless_fh is None:
        homeless_fh = open(homeless_file, 'rb')
        open_myself = True

    homeless_fh.seek(0)
    homeless_reader = csv.DictReader(homeless_fh)

    # yeah, we check the point against all 8k+ rows..yeah, yeah....
    keys = ['adults', 'men', 'women/families', 'youth']
    retval = dict((x, float('Inf')) for x in keys)
    for row in homeless_reader:
        latlong_diff = vincenty((latitude, longitude), (row['LAT'],row['LONG']))

        #we need to consider the "category" of which there is 4.
        cat = row['CATEGORY'].split(' ')[0].lower()
        if retval[cat] > latlong_diff.m:
            retval[cat] = latlong_diff.m

    # now we get the tuple from dict in order, since for feature vector we care about that kind of stuff
    final_retval = tuple()
    for c in keys:
        final_retval = final_retval + (retval[c],)

    #now we got the counts, we need to clean up and return
    if open_myself:
        homeless_fh.close()

    return final_retval

def number_of_street_lights_at(latitude, longitude, light_fh=None, radius1=50):
    light_file = '../data/street_lightings/street_lighting_poles.csv'

    # check if graffiti_fh is NOT empty
    open_myself = False
    if light_fh is None:
        light_fh = open(light_file, 'rb')
        open_myself = True

    light_fh.seek(0)
    light_reader = csv.DictReader(light_fh)

    # yeah, we check the point against all 8k+ rows..yeah, yeah....
    count1 = 0
    for row in light_reader:
        latlong_diff = vincenty((latitude, longitude), (row['LAT'],row['LONG']))

        #set counts based on radius1
        if latlong_diff.m < radius1:
            count1 = count1 + 1

    #now we got the counts, we need to clean up and return
    if open_myself:
        light_fh.close()

    return (count1,)

<|endoftext|>


<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# this is the command line interface.

import sys, os, getopt
from zohmg.utils import fail

# add all bundled eggs to sys.path
eggpath='/usr/local/lib/zohmg/egg'
for (dir, dirnames, files) in os.walk(eggpath):
    for file in files:
        suffix = file.split(".")[-1]
        if suffix == "egg":
            sys.path.append(dir+"/"+file)

# TODO: read version from $somewhere.
version = '0.2.3-dev'

def usage(reason = None):
    zohmg = os.path.basename(sys.argv[0])
    if reason:
        print "error: " + reason
    print "zohmg " + version
    print "usage:"
    print zohmg + " create <dir>"
    print zohmg + " setup"
    print zohmg + " import <mapper> <hdfs-input-dir> [--local] [--lzo]"
    print zohmg + " server [--host=<host>] [--port=<port>]"
    print zohmg + " reset"
    print zohmg + " help"

def print_version():
    print "zohmg version " + version

def print_help():
    print "Need help?"
    print
    print "There are a few documents in /usr/local/share/zohmg/doc that might be of some help,"
    print "and there's an IRC channel -- #zohmg on freenode -- where you can ask questions."

# command line entry-point.
def zohmg():
    try:
        # read the first argument.
        cmd = sys.argv[1]
    except:
        # there was no first argument.
        usage()
        sys.exit(0)

    if   cmd == 'create' : create()
    elif cmd == 'setup'  : setup()
    elif cmd == 'import' : process()
    elif cmd == 'server' : server()
    elif cmd == 'reset'  : reset()
    elif cmd in ['version', '--version']: print_version()
    elif cmd in ['help',    '--help']:    print_help()
    else: usage()

def create():
    from zohmg.create import Create
    try:
        path = sys.argv[2]
    except:
        usage("create needs an argument.")
        sys.exit(1)

    Create(path)

def setup():
    refuse_to_act_in_nonzohmg_directory()
    from zohmg.setup import Setup
    Setup().go()

# import.
def process():
    refuse_to_act_in_nonzohmg_directory()
    from zohmg.process import Process
    try:
        # check for two arguments,
        mapper = sys.argv[2]
        # (works only with relative paths for now.)
        mapperpath = os.path.abspath(".")+"/"+mapper
        inputdir = sys.argv[3]
        dumbo_args = sys.argv[4:]
    except:
        usage("import needs two arguments.")
        sys.exit(1)

    Process().go(mapperpath, inputdir, dumbo_args)

def server():
    refuse_to_act_in_nonzohmg_directory()
    import zohmg.server
    host, port = zohmg.server.defaults()

    try:
        opts, args = getopt.getopt(sys.argv[2:], "h:p:", ["host=", "port="])
    except getopt.GetoptError, err:
        print str(err)
        usage()
        sys.exit(2)

    for o, a in opts:
        if o in ("-h", "--host"):
            host=a
        elif o in ("-p", "--port"):
            port=a
        else:
            assert False, "unhandled option"

    project_dir = os.path.abspath("")
    zohmg.server.start(project_dir, host=host, port=port)

def reset():
    refuse_to_act_in_nonzohmg_directory()
    from zohmg.reset import Reset
    Reset().please()

# exits if 'zohmg' was run in a directory without the special .zohmg-file.
def refuse_to_act_in_nonzohmg_directory():
    cwd = os.getcwd()
    if not os.path.exists(cwd+"/.zohmg"):
        msg = "error: This is not a proper zohmg project."
        fail(msg)
<commit_msg>dfs or not.<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# this is the command line interface.

import sys, os, getopt
from zohmg.utils import fail

# add all bundled eggs to sys.path
eggpath='/usr/local/lib/zohmg/egg'
for (dir, dirnames, files) in os.walk(eggpath):
    for file in files:
        suffix = file.split(".")[-1]
        if suffix == "egg":
            sys.path.append(dir+"/"+file)

# TODO: read version from $somewhere.
version = '0.2.3-dev'

def usage(reason = None):
    zohmg = os.path.basename(sys.argv[0])
    if reason:
        print "error: " + reason
    print "zohmg " + version
    print "usage:"
    print zohmg + " create <dir>"
    print zohmg + " setup"
    print zohmg + " import <mapper> <input-dir> [--local] [--lzo]"
    print zohmg + " server [--host=<host>] [--port=<port>]"
    print zohmg + " reset"
    print zohmg + " help"

def print_version():
    print "zohmg version " + version

def print_help():
    print "Need help?"
    print
    print "There are a few documents in /usr/local/share/zohmg/doc that might be of some help,"
    print "and there's an IRC channel -- #zohmg on freenode -- where you can ask questions."

# command line entry-point.
def zohmg():
    try:
        # read the first argument.
        cmd = sys.argv[1]
    except:
        # there was no first argument.
        usage()
        sys.exit(0)

    if   cmd == 'create' : create()
    elif cmd == 'setup'  : setup()
    elif cmd == 'import' : process()
    elif cmd == 'server' : server()
    elif cmd == 'reset'  : reset()
    elif cmd in ['version', '--version']: print_version()
    elif cmd in ['help',    '--help']:    print_help()
    else: usage()

def create():
    from zohmg.create import Create
    try:
        path = sys.argv[2]
    except:
        usage("create needs an argument.")
        sys.exit(1)

    Create(path)

def setup():
    refuse_to_act_in_nonzohmg_directory()
    from zohmg.setup import Setup
    Setup().go()

# import.
def process():
    refuse_to_act_in_nonzohmg_directory()
    from zohmg.process import Process
    try:
        # check for two arguments,
        mapper = sys.argv[2]
        # (works only with relative paths for now.)
        mapperpath = os.path.abspath(".")+"/"+mapper
        inputdir = sys.argv[3]
        dumbo_args = sys.argv[4:]
    except:
        usage("import needs two arguments.")
        sys.exit(1)

    Process().go(mapperpath, inputdir, dumbo_args)

def server():
    refuse_to_act_in_nonzohmg_directory()
    import zohmg.server
    host, port = zohmg.server.defaults()

    try:
        opts, args = getopt.getopt(sys.argv[2:], "h:p:", ["host=", "port="])
    except getopt.GetoptError, err:
        print str(err)
        usage()
        sys.exit(2)

    for o, a in opts:
        if o in ("-h", "--host"):
            host=a
        elif o in ("-p", "--port"):
            port=a
        else:
            assert False, "unhandled option"

    project_dir = os.path.abspath("")
    zohmg.server.start(project_dir, host=host, port=port)

def reset():
    refuse_to_act_in_nonzohmg_directory()
    from zohmg.reset import Reset
    Reset().please()

# exits if 'zohmg' was run in a directory without the special .zohmg-file.
def refuse_to_act_in_nonzohmg_directory():
    cwd = os.getcwd()
    if not os.path.exists(cwd+"/.zohmg"):
        msg = "error: This is not a proper zohmg project."
        fail(msg)
<|endoftext|>


<commit_before>import json
import struct
import socket
import socketserver
import io
import nacl.utils
from nacl.encoding import HexEncoder
from nacl.public import PrivateKey, PublicKey, Box
from nacl.exceptions import CryptoError

MAX_MESSAGE_SIZE = 1024 * 1024

class SecureRpcException(Exception):
    pass

def read_from_socket(sock, n):
    buffer = io.BytesIO()
    while n > 0:
        b = sock.recv(n)
        if not b:
            break
        buffer.write(b)
        n = n - len(b)
    return buffer.getvalue()

def read_message(sock, private_key):
    size = struct.unpack('!L', read_from_socket(sock, 4))[0]
    if size > MAX_MESSAGE_SIZE:
        raise SecureRpcException('transport error: invalid message size %s' % size)
    public_key_sender = read_from_socket(sock, 32)
    if len(public_key_sender) != 32:
        raise SecureRpcException('transport error: public key not full read')
    public_key_sender = PublicKey(public_key_sender)
    ciphertext = read_from_socket(sock, size)
    if len(ciphertext) != size:
        raise SecureRpcException('transport error: message not fully read')
    try:
        box = Box(private_key, public_key_sender)
        plaintext = box.decrypt(ciphertext)
        anti_replay_nonce, plaintext = plaintext[:16], plaintext[16:]
        message = json.loads(plaintext.decode('utf-8'))
    except ValueError as e:
        raise SecureRpcException('decode error: invalid json')
    except CryptoError as e:
        raise SecureRpcException('decrypt error: %s' % e)
    return public_key_sender, anti_replay_nonce, message

def send_message(sock, message, anti_replay_nonce, private_key_sender, public_key_recipient):
    plaintext = anti_replay_nonce + json.dumps(message).encode('utf-8')
    box = Box(private_key_sender, public_key_recipient)
    ciphertext = box.encrypt(plaintext, nacl.utils.random(Box.NONCE_SIZE))
    public_key_sender = private_key_sender.public_key.encode()
    sock.sendall(struct.pack('!L', len(ciphertext)) + public_key_sender + ciphertext)

class SecureRpcRequest(object):
    def __init__(self, public_key, remote_addr):
        self.public_key = public_key
        self.remote_addr = remote_addr

def secure_rpc_serve(host, port, server_private_key, server_proxy):
    private_key = PrivateKey(server_private_key, HexEncoder)
    class MyTCPHandler(socketserver.BaseRequestHandler):
        def handle(self):
            public_key, nonce, message = read_message(self.request, private_key)
            public_key_hex = public_key.encode(HexEncoder)
            request = SecureRpcRequest(public_key_hex, self.request.getpeername()[0])
            try:
                response = getattr(server_proxy, message['method'])(*([request] + message['params']))
            except Exception as e:
                response = {'__error__': str(e)}
            send_message(self.request, response, nonce, private_key, public_key)
    socketserver.TCPServer.allow_reuse_address = True
    server = socketserver.TCPServer((host, port), MyTCPHandler)
    server.serve_forever()

class SecureRpcClient(object):
    def __init__(self, host, port, server_public_key, client_private_key):
        self.host = host
        self.port = port
        self.server_public_key = PublicKey(server_public_key, HexEncoder)
        self.client_private_key = PrivateKey(client_private_key, HexEncoder)

    def invoke(self, method, params, timeout=60):
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.connect((self.host, self.port))
        s.settimeout(timeout)
        request = {'method': method, 'params': params}
        nonce = nacl.utils.random(16)
        send_message(s, request, nonce, self.client_private_key, self.server_public_key)
        sender_public_key, response_nonce, response = read_message(s, self.client_private_key)
        if sender_public_key.encode(HexEncoder) != self.server_public_key.encode(HexEncoder):
            raise SecureRpcException('reply authentication error')
        if nonce != response_nonce:
            raise SecureRpcException('reply integrity error: replay suspected')
        if isinstance(response, dict) and '__error__' in response:
            raise SecureRpcException(response['__error__'])
        return response
<commit_msg>make sure public keys are strings; don't proxy private methods<commit_after>import json
import struct
import socket
import socketserver
import io
import nacl.utils
from nacl.encoding import HexEncoder
from nacl.public import PrivateKey, PublicKey, Box
from nacl.exceptions import CryptoError

MAX_MESSAGE_SIZE = 1024 * 1024

class SecureRpcException(Exception):
    pass

def read_from_socket(sock, n):
    buffer = io.BytesIO()
    while n > 0:
        b = sock.recv(n)
        if not b:
            break
        buffer.write(b)
        n = n - len(b)
    return buffer.getvalue()

def read_message(sock, private_key):
    size = struct.unpack('!L', read_from_socket(sock, 4))[0]
    if size > MAX_MESSAGE_SIZE:
        raise SecureRpcException('transport error: invalid message size %s' % size)
    public_key_sender = read_from_socket(sock, 32)
    if len(public_key_sender) != 32:
        raise SecureRpcException('transport error: public key not full read')
    public_key_sender = PublicKey(public_key_sender)
    ciphertext = read_from_socket(sock, size)
    if len(ciphertext) != size:
        raise SecureRpcException('transport error: message not fully read')
    try:
        box = Box(private_key, public_key_sender)
        plaintext = box.decrypt(ciphertext)
        anti_replay_nonce, plaintext = plaintext[:16], plaintext[16:]
        message = json.loads(plaintext.decode('utf-8'))
    except ValueError as e:
        raise SecureRpcException('decode error: invalid json')
    except CryptoError as e:
        raise SecureRpcException('decrypt error: %s' % e)
    return public_key_sender, anti_replay_nonce, message

def send_message(sock, message, anti_replay_nonce, private_key_sender, public_key_recipient):
    plaintext = anti_replay_nonce + json.dumps(message).encode('utf-8')
    box = Box(private_key_sender, public_key_recipient)
    ciphertext = box.encrypt(plaintext, nacl.utils.random(Box.NONCE_SIZE))
    public_key_sender = private_key_sender.public_key.encode()
    sock.sendall(struct.pack('!L', len(ciphertext)) + public_key_sender + ciphertext)

class SecureRpcRequest(object):
    def __init__(self, public_key, remote_addr):
        self.public_key = public_key
        self.remote_addr = remote_addr

def secure_rpc_serve(host, port, server_private_key, server_proxy):
    private_key = PrivateKey(server_private_key, HexEncoder)
    class MyTCPHandler(socketserver.BaseRequestHandler):
        def handle(self):
            public_key, nonce, message = read_message(self.request, private_key)
            public_key_hex = str(public_key.encode(HexEncoder), 'utf-8')
            request = SecureRpcRequest(public_key_hex, self.request.getpeername()[0])
            try:
                if message['method'].startswith('_'):
                    raise Exception('access denied')
                method = getattr(server_proxy, message['method'])
                response = method(*([request] + message['params']))
            except Exception as e:
                response = {'__error__': str(e)}
            send_message(self.request, response, nonce, private_key, public_key)
    socketserver.TCPServer.allow_reuse_address = True
    server = socketserver.TCPServer((host, port), MyTCPHandler)
    server.serve_forever()

class SecureRpcClient(object):
    def __init__(self, host, port, server_public_key, client_private_key):
        self.host = host
        self.port = port
        self.server_public_key = PublicKey(server_public_key, HexEncoder)
        self.client_private_key = PrivateKey(client_private_key, HexEncoder)

    def invoke(self, method, params, timeout=60):
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.connect((self.host, self.port))
        s.settimeout(timeout)
        request = {'method': method, 'params': params}
        nonce = nacl.utils.random(16)
        send_message(s, request, nonce, self.client_private_key, self.server_public_key)
        sender_public_key, response_nonce, response = read_message(s, self.client_private_key)
        if sender_public_key.encode(HexEncoder) != self.server_public_key.encode(HexEncoder):
            raise SecureRpcException('reply authentication error')
        if nonce != response_nonce:
            raise SecureRpcException('reply integrity error: replay suspected')
        if isinstance(response, dict) and '__error__' in response:
            raise SecureRpcException(response['__error__'])
        return response
<|endoftext|>


<commit_before># Copyright (c) 2006-2008 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Steve Reinhardt

# Simple test script
#
# "m5 test.py"

import os
import optparse
import sys
from os.path import join as joinpath

import m5
from m5.defines import buildEnv
from m5.objects import *
from m5.util import addToPath, panic

if buildEnv['FULL_SYSTEM']:
    panic("This script requires syscall emulation mode (*_SE).")

addToPath('../common')

import Simulation
from cpu2000 import *

# Get paths we might need.  It's expected this file is in m5/configs/example.
config_path = os.path.dirname(os.path.abspath(__file__))
config_root = os.path.dirname(config_path)
m5_root = os.path.dirname(config_root)

parser = optparse.OptionParser()

# Benchmark options
parser.add_option("-c", "--cmd",
    default=joinpath(m5_root, "tests/test-progs/hello/bin/alpha/linux/hello"),
    help="The binary to run in syscall emulation mode.")
parser.add_option("-o", "--options", default="",
    help='The options to pass to the binary, use " " around the entire string')
parser.add_option("-i", "--input", default="", help="Read stdin from a file.")
parser.add_option("--output", default="", help="Redirect stdout to a file.")
parser.add_option("--errout", default="", help="Redirect stderr to a file.")
parser.add_option("--ruby-debug", action="store_true")
parser.add_option("--ruby-debug-file", default="", help="Ruby debug out file (stdout if blank)")

execfile(os.path.join(config_root, "common", "Options.py"))

(options, args) = parser.parse_args()

if args:
    print "Error: script doesn't take any positional arguments"
    sys.exit(1)

if options.bench:
    try:
        if buildEnv['TARGET_ISA'] != 'alpha':
            print >>sys.stderr, "Simpoints code only works for Alpha ISA at this time"
            sys.exit(1)
        exec("workload = %s('alpha', 'tru64', 'ref')" % options.bench)
        process = workload.makeLiveProcess()
    except:
        print >>sys.stderr, "Unable to find workload for %s" % options.bench
        sys.exit(1)
else:
    process = LiveProcess()
    process.executable = options.cmd
    process.cmd = [options.cmd] + options.options.split()

if options.input != "":
    process.input = options.input
if options.output != "":
    process.output = options.output
if options.errout != "":
    process.errout = options.errout

if options.detailed:
    #check for SMT workload
    workloads = options.cmd.split(';')
    if len(workloads) > 1:
        process = []
        smt_idx = 0
        inputs = []
        outputs = []
        errouts = []

        if options.input != "":
            inputs = options.input.split(';')
        if options.output != "":
            outputs = options.output.split(';')
        if options.errout != "":
            errouts = options.errout.split(';')

        for wrkld in workloads:
            smt_process = LiveProcess()
            smt_process.executable = wrkld
            smt_process.cmd = wrkld + " " + options.options
            if inputs and inputs[smt_idx]:
                smt_process.input = inputs[smt_idx]
            if outputs and outputs[smt_idx]:
                smt_process.output = outputs[smt_idx]
            if errouts and errouts[smt_idx]:
                smt_process.errout = errouts[smt_idx]
            process += [smt_process, ]
            smt_idx += 1

(CPUClass, test_mem_mode, FutureClass) = Simulation.setCPUClass(options)

CPUClass.clock = '1GHz'

np = options.num_cpus

rubymem = RubyMemory(
  range = AddrRange("512MB"),
  clock = "1GHz",
  num_cpus = np,
  config_file = "ruby.config",
  stats_file = "m5out/ruby.stats"
)

if options.ruby_debug == True:
  rubymem.debug = True
  rubymem.debug_file = options.ruby_debug_file

system = System(cpu = [CPUClass(cpu_id=i) for i in xrange(np)],
                physmem = rubymem)

if options.l2cache:
    print "Error: -l2cache incompatible with ruby, must configure it ruby-style"
    sys.exit(1)

if options.caches:
    print "Error: -caches incompatible with ruby, must configure it ruby-style"
    sys.exit(1)

for i in xrange(np):
    system.cpu[i].connectMemPorts(system.physmem)

    '''process = LiveProcess()
    process.executable = options.cmd
    process.cmd = [options.cmd, str(i)]
    '''
    system.cpu[i].workload = process

    if options.fastmem:
        system.cpu[i].physmem_port = system.physmem.port

root = Root(system = system)

Simulation.run(options, root, system, FutureClass)
<commit_msg>ruby: ruby_se.py updated to new config system<commit_after># Copyright (c) 2006-2008 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Steve Reinhardt

# Simple test script
#
# "m5 test.py"

import os
import optparse
import sys
from os.path import join as joinpath

import m5
from m5.defines import buildEnv
from m5.objects import *
from m5.util import addToPath, panic

if buildEnv['FULL_SYSTEM']:
    panic("This script requires syscall emulation mode (*_SE).")

addToPath('../common')
addToPath('../ruby')

import Ruby

import Simulation
from cpu2000 import *

# Get paths we might need.  It's expected this file is in m5/configs/example.
config_path = os.path.dirname(os.path.abspath(__file__))
config_root = os.path.dirname(config_path)
m5_root = os.path.dirname(config_root)

parser = optparse.OptionParser()

# Benchmark options
parser.add_option("-c", "--cmd",
    default=joinpath(m5_root, "tests/test-progs/hello/bin/alpha/linux/hello"),
    help="The binary to run in syscall emulation mode.")
parser.add_option("-o", "--options", default="",
    help='The options to pass to the binary, use " " around the entire string')
parser.add_option("-i", "--input", default="", help="Read stdin from a file.")
parser.add_option("--output", default="", help="Redirect stdout to a file.")
parser.add_option("--errout", default="", help="Redirect stderr to a file.")

# cache parameters
parser.add_option("--l1d_size", type="string", default="32kB")
parser.add_option("--l1i_size", type="string", default="32kB")
parser.add_option("--l2_size", type="string", default="1MB")
parser.add_option("--l1d_assoc", type="int", default=2)
parser.add_option("--l1i_assoc", type="int", default=2)
parser.add_option("--l2_assoc", type="int", default=16)

execfile(os.path.join(config_root, "common", "Options.py"))

(options, args) = parser.parse_args()

if args:
    print "Error: script doesn't take any positional arguments"
    sys.exit(1)

if options.bench:
    try:
        if buildEnv['TARGET_ISA'] != 'alpha':
            print >>sys.stderr, "Simpoints code only works for Alpha ISA at this time"
            sys.exit(1)
        exec("workload = %s('alpha', 'tru64', 'ref')" % options.bench)
        process = workload.makeLiveProcess()
    except:
        print >>sys.stderr, "Unable to find workload for %s" % options.bench
        sys.exit(1)
else:
    process = LiveProcess()
    process.executable = options.cmd
    process.cmd = [options.cmd] + options.options.split()

if options.input != "":
    process.input = options.input
if options.output != "":
    process.output = options.output
if options.errout != "":
    process.errout = options.errout

if options.detailed:
    #check for SMT workload
    workloads = options.cmd.split(';')
    if len(workloads) > 1:
        process = []
        smt_idx = 0
        inputs = []
        outputs = []
        errouts = []

        if options.input != "":
            inputs = options.input.split(';')
        if options.output != "":
            outputs = options.output.split(';')
        if options.errout != "":
            errouts = options.errout.split(';')

        for wrkld in workloads:
            smt_process = LiveProcess()
            smt_process.executable = wrkld
            smt_process.cmd = wrkld + " " + options.options
            if inputs and inputs[smt_idx]:
                smt_process.input = inputs[smt_idx]
            if outputs and outputs[smt_idx]:
                smt_process.output = outputs[smt_idx]
            if errouts and errouts[smt_idx]:
                smt_process.errout = errouts[smt_idx]
            process += [smt_process, ]
            smt_idx += 1

(CPUClass, test_mem_mode, FutureClass) = Simulation.setCPUClass(options)

#
# Currently ruby only works in timing mode
#
assert(options.timing)
assert(test_mem_mode == 'timing')
assert(FutureClass == None)

CPUClass.clock = '1GHz'

np = options.num_cpus

system = System(cpu = [CPUClass(cpu_id=i) for i in xrange(np)],
                physmem = PhysicalMemory())

system.ruby = Ruby.create_system(options, system.physmem)

assert(options.num_cpus == len(system.ruby.cpu_ruby_ports))

for (i, cpu) in enumerate(system.cpu):
    #
    # Tie the cpu ports to the ruby cpu ports
    #
    cpu.icache_port = system.ruby.cpu_ruby_ports[i].port
    cpu.dcache_port = system.ruby.cpu_ruby_ports[i].port

    '''process = LiveProcess()
    process.executable = options.cmd
    process.cmd = [options.cmd, str(i)]
    '''
    cpu.workload = process

root = Root(system = system)

Simulation.run(options, root, system, FutureClass)
<|endoftext|>


<commit_before><commit_msg>Parallel ASN resolve and allow passing of input file names to makeseeds.py (#2432)<commit_after><|endoftext|>


<commit_before>import numpy as np

from methodbase import MethodBase
from line import Line

class InterpBase(MethodBase):
  # def do(self, line, dx, xrange = None):
  #   if len(line.x) == 0:
  #     return Line([], [], line.name)

  #   if xrange is not None:
  #     X1, X2 = xrange
  #   else:
  #     X1, X2 = min(line.x), max(line.x)

  #   x = np.arange(X1, X2, dx)
  #   y = self.func(line)(x)
  #   return Line(x, y, line.name)

  def func(self, line):
    raise NotImplementedError()

class InterpLinear(InterpBase):
  name    = 'linear'
  label   = 'Linear'

  def func(self, line):
    from scipy.interpolate import interp1d
    return interp1d(line.x, line.y, 'linear')

class InterpScipy(InterpBase):
  def func(self, line):
    import scipy.interpolate as interp
    return getattr(interp, self.clsname)(line.x, line.y)

class InterpCubicSpline(InterpScipy):
  name    = 'cubic_spline'
  label   = 'Cubic spline'
  clsname = 'CubicSpline'

class InterpBarycentric(InterpScipy):
  name    = 'barycentric'
  label   = 'Barycentric'
  clsname = 'BarycentricInterpolator'

class InterpKrogh(InterpScipy):
  name    = 'krogh'
  label   = 'Krogh'
  clsname = 'KroghInterpolator'

class InterpPchip(InterpScipy):
  name    = 'pchip'
  label   = 'Pchip'
  clsname = 'PchipInterpolator'

class InterpAkima(InterpScipy):
  name    = 'akima'
  label   = 'Akima'
  clsname = 'Akima1DInterpolator'
<commit_msg>Cleanup commented out code<commit_after>import numpy as np

from methodbase import MethodBase
from line import Line

class InterpBase(MethodBase):
  def func(self, line):
    raise NotImplementedError()

class InterpLinear(InterpBase):
  name    = 'linear'
  label   = 'Linear'

  def func(self, line):
    from scipy.interpolate import interp1d
    return interp1d(line.x, line.y, 'linear')

class InterpScipy(InterpBase):
  def func(self, line):
    import scipy.interpolate as interp
    return getattr(interp, self.clsname)(line.x, line.y)

class InterpCubicSpline(InterpScipy):
  name    = 'cubic_spline'
  label   = 'Cubic spline'
  clsname = 'CubicSpline'

class InterpBarycentric(InterpScipy):
  name    = 'barycentric'
  label   = 'Barycentric'
  clsname = 'BarycentricInterpolator'

class InterpKrogh(InterpScipy):
  name    = 'krogh'
  label   = 'Krogh'
  clsname = 'KroghInterpolator'

class InterpPchip(InterpScipy):
  name    = 'pchip'
  label   = 'Pchip'
  clsname = 'PchipInterpolator'

class InterpAkima(InterpScipy):
  name    = 'akima'
  label   = 'Akima'
  clsname = 'Akima1DInterpolator'
<|endoftext|>


<commit_before>"""Shows filters of a convolutional neural network.

Matthew Alger
The Australian National University
2016
"""

import argparse

import h5py
import matplotlib.pyplot as plt
import numpy

from crowdastro.config import config

def main(model_path, weights_path, training_h5_path):
    import keras.models
    with open(model_path, 'r') as model_json:
        model = keras.models.model_from_json(model_json.read())
    model.compile(loss='binary_crossentropy', optimizer='adadelta')
    model.load_weights(weights_path)
    conv_layer_0 = model.get_weights()[0]
    conv_layer_1 = model.get_weights()[2]
    conv_layer_2 = model.get_weights()[4]

    # n_filters = conv_layer_0.shape[0]
    # plt.figure(figsize=[8, 4])
    # for i, filt in enumerate(conv_layer_0):
    #     plt.subplot(4, 8, i + 1)
    #     plt.axis('off')
    #     filt = filt[0]
    #     plt.pcolor(filt, cmap='viridis')
    # plt.subplots_adjust(hspace=0.05, wspace=0.05, left=0.00, bottom=0.00,
    #                     top=1.00, right=1.00)
    # plt.show()

    # n_filters = conv_layer_1.shape[0] * conv_layer_1.shape[1]
    # plt.figure(figsize=[8, 8])
    # for i, filts in enumerate(conv_layer_1):
    #     for j, filt in enumerate(filts):
    #         plt.subplot(32, 32, i * 32 + j + 1)
    #         plt.axis('off')
    #         plt.pcolor(filt, cmap='viridis')
    # plt.subplots_adjust(hspace=0.05, wspace=0.05, left=0.00, bottom=0.00,
    #                     top=1.00, right=1.00)
    # plt.show()

    encoded_0 = keras.backend.function([model.layers[0].input],
                                       [model.layers[0].output])
    encoded_1 = keras.backend.function([model.layers[0].input],
                                       [model.layers[3].output])
    encoded_2 = keras.backend.function([model.layers[0].input],
                                       [model.layers[5].output])
    print('Weights shapes:', [i.shape for i in model.get_weights()])
    print([(i, l) for i, l in enumerate(model.layers)])

    with h5py.File(training_h5_path, 'r') as f_h5:
        n_raw_features = config['surveys'][
            f_h5.attrs['ir_survey']]['n_features']
        for i in numpy.random.randint(f_h5['raw_features'].shape[0],
                                      size=2):
            img = f_h5['raw_features'][i, n_raw_features:].reshape(
                (1, 1, 32, 32))

            # # Print shapes out.
            # for j, layer in enumerate(model.layers):
            #     f = keras.backend.function([model.layers[0].input],
            #                                [layer.output])
            #     print(layer)
            #     print(j)
            #     print(f([img])[0].shape)

            out_0 = encoded_0([img])[0].reshape((32, 29, 29))
            out_1 = encoded_1([img])[0].reshape((32, 11, 11))
            out_2 = encoded_2([img])[0].reshape((32, 5, 5))

            plt.figure(figsize=[2 + 4 + 4 + 4, 8])

            ax = plt.subplot2grid((8, 2 + 4 + 4 + 4), (3, 0), colspan=2,
                                  rowspan=2)
            ax.pcolor(img[0, 0], cmap='viridis')
            ax.axis('off')
            for j, convolved in enumerate(out_0):
                ax = plt.subplot2grid((8, 2 + 4 + 4 + 4), (j // 4, 2 + j % 4))
                plt.axis('off')
                plt.pcolor(convolved, cmap='viridis')
            for j, convolved in enumerate(out_1):
                ax = plt.subplot2grid((8, 2 + 4 + 4 + 4), (j // 4, 6 + j % 4))
                plt.axis('off')
                plt.pcolor(convolved, cmap='viridis')
            for j, convolved in enumerate(out_2):
                ax = plt.subplot2grid((8, 2 + 4 + 4 + 4), (j // 4, 10 + j % 4))
                plt.axis('off')
                plt.pcolor(convolved, cmap='viridis')
            plt.show()

def _populate_parser(parser):
    parser.description = 'Shows filters of a convolutional neural network.'
    parser.add_argument('--model', help='path to model JSON',
                        default='data/model.json')
    parser.add_argument('--weights', help='path to weights HDF5',
                        default='data/weights.h5')
    parser.add_argument('--training', help='path to training HDF5',
                        default='data/training.h5')

def _main(args):
    main(args.model, args.weights, args.training)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    _populate_parser(parser)
    args = parser.parse_args()
    _main(args)
<commit_msg>Updated show_filters to match thesis plots<commit_after>"""Shows filters of a convolutional neural network.

Matthew Alger
The Australian National University
2016
"""

import argparse

import h5py
import matplotlib.pyplot as plt
import numpy

from crowdastro.config import config

def main(model_path, weights_path, training_h5_path):
    import keras.models
    with open(model_path, 'r') as model_json:
        model = keras.models.model_from_json(model_json.read())
    model.compile(loss='binary_crossentropy', optimizer='adadelta')
    model.load_weights(weights_path)

    for i, layer in enumerate(model.layers):
        print(i, layer)

    conv_layer_0 = model.layers[0].get_weights()[0]
    conv_layer_1 = model.layers[3].get_weights()[0]
    conv_layer_2 = model.layers[6].get_weights()[0]

    # plt.figure(figsize=[8, 4])
    # for i, filt in enumerate(conv_layer_0):
    #     plt.subplot(4, 8, i + 1)
    #     plt.axis('off')
    #     filt = filt[0]
    #     plt.pcolor(filt, cmap='viridis')
    # plt.subplots_adjust(hspace=0.05, wspace=0.05, left=0.00, bottom=0.00,
    #                     top=1.00, right=1.00)
    # plt.show()

    # plt.figure(figsize=[8, 8])
    # for i, filts in enumerate(conv_layer_1):
    #     for j, filt in enumerate(filts):
    #         plt.subplot(32, 32, i * 32 + j + 1)
    #         plt.axis('off')
    #         plt.pcolor(filt, cmap='viridis')
    # plt.subplots_adjust(hspace=0.05, wspace=0.05, left=0.00, bottom=0.00,
    #                     top=1.00, right=1.00)
    # plt.show()

    # plt.figure(figsize=[8, 8])
    # for i, filts in enumerate(conv_layer_2):
    #     for j, filt in enumerate(filts):
    #         plt.subplot(32, 32, i * 32 + j + 1)
    #         plt.axis('off')
    #         plt.pcolor(filt, cmap='viridis')
    # plt.subplots_adjust(hspace=0.05, wspace=0.05, left=0.00, bottom=0.00,
    #                     top=1.00, right=1.00)
    # plt.show()

    encoded_0 = keras.backend.function([model.layers[0].input],
                                       [model.layers[0].output])
    encoded_1 = keras.backend.function([model.layers[0].input],
                                       [model.layers[3].output])
    encoded_2 = keras.backend.function([model.layers[0].input],
                                       [model.layers[6].output])
    print('Weights shapes:', [i.shape for i in model.get_weights()])
    print([(i, l) for i, l in enumerate(model.layers)])

    with h5py.File(training_h5_path, 'r') as f_h5:
        n_raw_features = config['surveys'][
            f_h5.attrs['ir_survey']]['n_features']
        for i in numpy.random.randint(f_h5['raw_features'].shape[0],
                                      size=2):
            img = f_h5['raw_features'][i, n_raw_features:].reshape(
                (1, 1, 32, 32))

            # # Print shapes out.
            # for j, layer in enumerate(model.layers):
            #     f = keras.backend.function([model.layers[0].input],
            #                                [layer.output])
            #     print(layer)
            #     print(j)
            #     print(f([img])[0].shape)
            out_0 = encoded_0([img])[0].reshape((32, 29, 29))
            out_1 = encoded_1([img])[0].reshape((32, 11, 11))
            out_2 = encoded_2([img])[0].reshape((32, 2, 2))

            plt.figure(figsize=[2 + 4 + 4 + 4, 8])

            ax = plt.subplot2grid((8, 2 + 4 + 4 + 4), (3, 0), colspan=2,
                                  rowspan=2)
            ax.pcolor(img[0, 0], cmap='viridis')
            ax.axis('off')
            for j, convolved in enumerate(out_0):
                ax = plt.subplot2grid((8, 2 + 4 + 4 + 4), (j // 4, 2 + j % 4))
                plt.axis('off')
                plt.pcolor(convolved, cmap='viridis')
            for j, convolved in enumerate(out_1):
                ax = plt.subplot2grid((8, 2 + 4 + 4 + 4), (j // 4, 6 + j % 4))
                plt.axis('off')
                plt.pcolor(convolved, cmap='viridis')
            for j, convolved in enumerate(out_2):
                ax = plt.subplot2grid((8, 2 + 4 + 4 + 4), (j // 4, 10 + j % 4))
                plt.axis('off')
                plt.pcolor(convolved, cmap='viridis')
            plt.show()

def _populate_parser(parser):
    parser.description = 'Shows filters of a convolutional neural network.'
    parser.add_argument('--model', help='path to model JSON',
                        default='data/model.json')
    parser.add_argument('--weights', help='path to weights HDF5',
                        default='data/weights.h5')
    parser.add_argument('--training', help='path to training HDF5',
                        default='data/training.h5')

def _main(args):
    main(args.model, args.weights, args.training)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    _populate_parser(parser)
    args = parser.parse_args()
    _main(args)
<|endoftext|>