<commit_before>import json
import logging
import random
import time
from datetime import datetime, timedelta

import zmq
from django.conf import settings
from django.core.management.base import NoArgsCommand
from django.db import transaction
from django.utils import timezone

from job_runner.apps.job_runner.models import KillRequest, Run, Worker

logger = logging.getLogger(__name__)

class Command(NoArgsCommand):
    help = 'Broadcast runs and kill-requests to workers'

    publisher = None
    """
    Holds the ZMQ ``publisher`` instance, which will publish to the workers.
    """

    @transaction.commit_manually
    def handle_noargs(self, **options):
        logger.info('Starting queue broadcaster')
        context = zmq.Context(1)

        # setup the publisher to which the workers are subscribing
        self.publisher = context.socket(zmq.PUB)
        self.publisher.bind(
            'tcp://*:{0}'.format(settings.JOB_RUNNER_BROADCASTER_PORT))

        # setup the event publisher which is publishing to the WS server
        self.event_publisher = context.socket(zmq.PUB)
        self.event_publisher.connect('tcp://{0}:{1}'.format(
            settings.JOB_RUNNER_WS_SERVER_HOSTNAME,
            settings.JOB_RUNNER_WS_SERVER_PORT,
        ))

        # give the subscribers some time to (re-)connect.
        time.sleep(2)

        ping_delta = timedelta(
            seconds=settings.JOB_RUNNER_WORKER_PING_INTERVAL)
        next_ping_request = datetime.utcnow()

        while True:
            if next_ping_request <= datetime.utcnow():
                self._broadcast_worker_ping()
                next_ping_request = datetime.utcnow() + ping_delta
            self._broadcast_runs()
            self._broadcast_kill_requests()
            transaction.commit()
            time.sleep(5)

        self.publisher.close()
        self.event_publisher.close()
        context.term()

    @transaction.commit_manually
    def _broadcast_runs(self):
        """
        Broadcast runs that are scheduled to run now.

        When the job has ``job__enqueue_is_enabled`` set to ``False``, its
        runs are not broadcasted, unless they are scheduled manually
        (``is_manual`` set to ``True``).

        """
        enqueueable_runs = Run.objects.enqueueable().select_related()

        # Use select_for_update so that the enqueueable runs will be locked.
        # This is to make sure that in case of multiple broadcasters we're not
        # creating any duplicate runs in case of run_on_all_workers=True.
        enqueueable_runs = enqueueable_runs.select_for_update()
        broadcasted = {}
        to_broadcast = []

        try:
            for run in enqueueable_runs:
                # schedule the run if we haven't already scheduled a run for
                # the same job, or when the run.schedule_id is equal to the
                # already scheduled run (which indicates that it needs to be
                # scheduled in parallel).
                if (run.job.pk not in broadcasted or
                        (run.job.pk in broadcasted and
                            broadcasted[run.job.pk] == run.get_schedule_id())):
                    worker = run.worker

                    # reschedule the job (the reschedule method will take care
                    # of checking if we should reschedule the job or not).
                    run.job.reschedule()

                    if not worker:
                        # if the job should run on all workers
                        if run.job.run_on_all_workers:
                            workers = run.job.worker_pool.workers.filter(
                                enqueue_is_enabled=True)
                            if workers.count():
                                broadcasted[run.job.pk] = run.get_schedule_id()

                                for w in workers:
                                    # assign a worker to each run
                                    assigned_run = Run.objects.create(
                                        job=run.job,
                                        schedule_id=run.get_schedule_id(),
                                        worker=w,
                                        schedule_dts=run.schedule_dts,
                                        is_manual=run.is_manual,
                                        schedule_children=run.schedule_children
                                    )
                                    to_broadcast.append((assigned_run, w))

                                # delete the "old" unassigned run
                                run.delete()

                        # select a random worker
                        else:
                            # TODO: take ping response into account?
                            workers = run.job.worker_pool.workers.filter(
                                enqueue_is_enabled=True)
                            if len(workers):
                                # pick a random active worker
                                worker = workers[
                                    random.randint(0, len(workers) - 1)]

                    # this is the case when a run has already a worker assigned
                    # to it, or when we selected a random worker.
                    if worker:
                        to_broadcast.append((run, worker))
                        broadcasted[run.job.pk] = run.get_schedule_id()

        except Exception:
            logger.exception('Something went wrong, rolling back transaction')
            transaction.rollback()
        else:
            transaction.commit()

            for brocast_args in to_broadcast:
                self._broadcast_run(*brocast_args)

    def _broadcast_run(self, run, worker):
        """
        Broadcast ``run`` to ``worker``.
        """
        message = [
            'master.broadcast.{0}'.format(worker.api_key),
            json.dumps({'run_id': run.id, 'action': 'enqueue'})
        ]
        logger.info('Sending: {0}'.format(message))
        self.publisher.send_multipart(message)

    def _broadcast_kill_requests(self):
        """
        Broadcast kill-requests.
        """
        kill_requests = KillRequest.objects.killable().select_related()

        for kill_request in kill_requests:
            run = kill_request.run
            worker = run.worker
            message = [
                'master.broadcast.{0}'.format(worker.api_key),
                json.dumps({
                    'kill_request_id': kill_request.id,
                    'action': 'kill',
                })
            ]
            logger.debug('Sending: {0}'.format(message))
            self.publisher.send_multipart(message)

    def _broadcast_worker_ping(self):
        """
        Broadcast ping-request to all the workers.
        """
        workers = Worker.objects.all()

        for worker in workers:
            message = [
                'master.broadcast.{0}'.format(worker.api_key),
                json.dumps({
                    'action': 'ping',
                })
            ]
            logger.debug('Sending: {0}'.format(message))
            self.publisher.send_multipart(message)

    def _find_unresponsive_workers_and_mark_runs_as_failed(self):
        """
        Detect unresponsive workers and mark their active runs as failed.
        """
        workers = Worker.objects.all()
        failed_intervals = \
            settings.JOB_RUNNER_WORKER_MARK_JOB_FAILED_AFTER_INTERVALS
        ping_interval = settings.JOB_RUNNER_WORKER_PING_INTERVAL
        ping_margin = settings.JOB_RUNNER_WORKER_PING_MARGIN

        acceptable_delta = timedelta(
            seconds=(failed_intervals * ping_interval) + ping_margin)

        for worker in workers:
            if worker.ping_response_dts + acceptable_delta < timezone.now():
                self._mark_worker_runs_as_failed(
                    worker,
                    'Worker does not respond to ping requests.'
                )

    @transaction.commit_manually
    def _mark_worker_runs_as_failed(self, worker, message):
        """
        Mark the runs that are not finished yet as failed for ``worker``.

        :param worker:
            The worker for which to mark the runs as failed.

        :param message:
            A ``str`` containing a message why the runs were marked as failed.

        """
        run_ids = []

        try:
            active_runs = worker.run_set.filter(return_dts__isnull=True)
            active_runs = active_runs.select_for_update()

            for run in active_runs:
                logger.warning('Marking run {0} as failed.'.format(run.pk))
                run.mark_failed(message)
                run_ids.append(run.pk)

        except Exception:
            logger.exception('Something went wrong, rolling back transaction')
            transaction.rollback()
        else:
            transaction.commit()

        # broadcast events after transaction has been comitted :-)
        for run_id in run_ids:
            self.event_publisher.send_multipart([
                'worker.event',
                json.dumps({
                    'event': 'returned',
                    'run_id': run_id,
                    'kind': 'run',
                })
            ])
<commit_msg>Wire up _find_unresponsive_workers_and_mark_runs_as_failed.<commit_after>import json
import logging
import random
import time
from datetime import datetime, timedelta

import zmq
from django.conf import settings
from django.core.management.base import NoArgsCommand
from django.db import transaction
from django.utils import timezone

from job_runner.apps.job_runner.models import KillRequest, Run, Worker

logger = logging.getLogger(__name__)

class Command(NoArgsCommand):
    help = 'Broadcast runs and kill-requests to workers'

    publisher = None
    """
    Holds the ZMQ ``publisher`` instance, which will publish to the workers.
    """

    @transaction.commit_manually
    def handle_noargs(self, **options):
        logger.info('Starting queue broadcaster')
        context = zmq.Context(1)

        # setup the publisher to which the workers are subscribing
        self.publisher = context.socket(zmq.PUB)
        self.publisher.bind(
            'tcp://*:{0}'.format(settings.JOB_RUNNER_BROADCASTER_PORT))

        # setup the event publisher which is publishing to the WS server
        self.event_publisher = context.socket(zmq.PUB)
        self.event_publisher.connect('tcp://{0}:{1}'.format(
            settings.JOB_RUNNER_WS_SERVER_HOSTNAME,
            settings.JOB_RUNNER_WS_SERVER_PORT,
        ))

        # give the subscribers some time to (re-)connect.
        time.sleep(2)

        ping_delta = timedelta(
            seconds=settings.JOB_RUNNER_WORKER_PING_INTERVAL)
        next_ping_request = datetime.utcnow()

        while True:
            if next_ping_request <= datetime.utcnow():
                self._broadcast_worker_ping()
                if settings.JOB_RUNNER_WORKER_MARK_JOB_FAILED_AFTER_INTERVALS:
                    self._find_unresponsive_workers_and_mark_runs_as_failed()
                next_ping_request = datetime.utcnow() + ping_delta
            self._broadcast_runs()
            self._broadcast_kill_requests()
            transaction.commit()
            time.sleep(5)

        self.publisher.close()
        self.event_publisher.close()
        context.term()

    @transaction.commit_manually
    def _broadcast_runs(self):
        """
        Broadcast runs that are scheduled to run now.

        When the job has ``job__enqueue_is_enabled`` set to ``False``, its
        runs are not broadcasted, unless they are scheduled manually
        (``is_manual`` set to ``True``).

        """
        enqueueable_runs = Run.objects.enqueueable().select_related()

        # Use select_for_update so that the enqueueable runs will be locked.
        # This is to make sure that in case of multiple broadcasters we're not
        # creating any duplicate runs in case of run_on_all_workers=True.
        enqueueable_runs = enqueueable_runs.select_for_update()
        broadcasted = {}
        to_broadcast = []

        try:
            for run in enqueueable_runs:
                # schedule the run if we haven't already scheduled a run for
                # the same job, or when the run.schedule_id is equal to the
                # already scheduled run (which indicates that it needs to be
                # scheduled in parallel).
                if (run.job.pk not in broadcasted or
                        (run.job.pk in broadcasted and
                            broadcasted[run.job.pk] == run.get_schedule_id())):
                    worker = run.worker

                    # reschedule the job (the reschedule method will take care
                    # of checking if we should reschedule the job or not).
                    run.job.reschedule()

                    if not worker:
                        # if the job should run on all workers
                        if run.job.run_on_all_workers:
                            workers = run.job.worker_pool.workers.filter(
                                enqueue_is_enabled=True)
                            if workers.count():
                                broadcasted[run.job.pk] = run.get_schedule_id()

                                for w in workers:
                                    # assign a worker to each run
                                    assigned_run = Run.objects.create(
                                        job=run.job,
                                        schedule_id=run.get_schedule_id(),
                                        worker=w,
                                        schedule_dts=run.schedule_dts,
                                        is_manual=run.is_manual,
                                        schedule_children=run.schedule_children
                                    )
                                    to_broadcast.append((assigned_run, w))

                                # delete the "old" unassigned run
                                run.delete()

                        # select a random worker
                        else:
                            # TODO: take ping response into account?
                            workers = run.job.worker_pool.workers.filter(
                                enqueue_is_enabled=True)
                            if len(workers):
                                # pick a random active worker
                                worker = workers[
                                    random.randint(0, len(workers) - 1)]

                    # this is the case when a run has already a worker assigned
                    # to it, or when we selected a random worker.
                    if worker:
                        to_broadcast.append((run, worker))
                        broadcasted[run.job.pk] = run.get_schedule_id()

        except Exception:
            logger.exception('Something went wrong, rolling back transaction')
            transaction.rollback()
        else:
            transaction.commit()

            for brocast_args in to_broadcast:
                self._broadcast_run(*brocast_args)

    def _broadcast_run(self, run, worker):
        """
        Broadcast ``run`` to ``worker``.
        """
        message = [
            'master.broadcast.{0}'.format(worker.api_key),
            json.dumps({'run_id': run.id, 'action': 'enqueue'})
        ]
        logger.info('Sending: {0}'.format(message))
        self.publisher.send_multipart(message)

    def _broadcast_kill_requests(self):
        """
        Broadcast kill-requests.
        """
        kill_requests = KillRequest.objects.killable().select_related()

        for kill_request in kill_requests:
            run = kill_request.run
            worker = run.worker
            message = [
                'master.broadcast.{0}'.format(worker.api_key),
                json.dumps({
                    'kill_request_id': kill_request.id,
                    'action': 'kill',
                })
            ]
            logger.debug('Sending: {0}'.format(message))
            self.publisher.send_multipart(message)

    def _broadcast_worker_ping(self):
        """
        Broadcast ping-request to all the workers.
        """
        workers = Worker.objects.all()

        for worker in workers:
            message = [
                'master.broadcast.{0}'.format(worker.api_key),
                json.dumps({
                    'action': 'ping',
                })
            ]
            logger.debug('Sending: {0}'.format(message))
            self.publisher.send_multipart(message)

    def _find_unresponsive_workers_and_mark_runs_as_failed(self):
        """
        Detect unresponsive workers and mark their active runs as failed.
        """
        workers = Worker.objects.all()
        failed_intervals = \
            settings.JOB_RUNNER_WORKER_MARK_JOB_FAILED_AFTER_INTERVALS
        ping_interval = settings.JOB_RUNNER_WORKER_PING_INTERVAL
        ping_margin = settings.JOB_RUNNER_WORKER_PING_MARGIN

        acceptable_delta = timedelta(
            seconds=(failed_intervals * ping_interval) + ping_margin)

        for worker in workers:
            if worker.ping_response_dts + acceptable_delta < timezone.now():
                self._mark_worker_runs_as_failed(
                    worker,
                    'Worker does not respond to ping requests.'
                )

    @transaction.commit_manually
    def _mark_worker_runs_as_failed(self, worker, message):
        """
        Mark the runs that are not finished yet as failed for ``worker``.

        :param worker:
            The worker for which to mark the runs as failed.

        :param message:
            A ``str`` containing a message why the runs were marked as failed.

        """
        run_ids = []

        try:
            active_runs = worker.run_set.filter(return_dts__isnull=True)
            active_runs = active_runs.select_for_update()

            for run in active_runs:
                logger.warning('Marking run {0} as failed.'.format(run.pk))
                run.mark_failed(message)
                run_ids.append(run.pk)

        except Exception:
            logger.exception('Something went wrong, rolling back transaction')
            transaction.rollback()
        else:
            transaction.commit()

        # broadcast events after transaction has been comitted :-)
        for run_id in run_ids:
            self.event_publisher.send_multipart([
                'worker.event',
                json.dumps({
                    'event': 'returned',
                    'run_id': run_id,
                    'kind': 'run',
                })
            ])
<|endoftext|>


<commit_before>import click
import os
import json
from typing import List

CONFIG_PATH = click.get_app_dir("qs") + "config.json"

def _load_config() -> None:
    if _config_exists():
        with open(CONFIG_PATH, 'r') as f:
            config = json.load(f)
    else:
        config = _create_config()
    return config

def _config_exists() -> None:
    if os.path.isfile(CONFIG_PATH):
        return True
    else:
        return False

def _create_config() -> None:
    config = {'BASE_DIR': os.path.expanduser("~/Projects")}
    with open('config.json', 'w') as f:
        json.dump(config, f)
    return config

def _get_sub_dirs(base_dir: str) -> List[str]:
    return list(filter(lambda x: os.path.isdir(os.path.join(base_dir, x)), os.listdir(base_dir)))

def _get_git_repos(dirs: List[str], base_dir: str) -> List[str]:
    pass

@click.group()
@click.pass_context
def main(ctx):
    """A simple CLI to aid in common, repetitive development tasks"""
    ctx.obj = {}
    ctx.obj['CONFIG'] = _load_config()

@main.command()
@click.option('--base-dir', '-p', is_flag=True, help='Base Projects Folder')
def config(ctx, base_dir):
    config = _load_config()
    if base_dir:
        config["BASE_DIR"] = base_dir

@main.command()
@click.pass_context
def sync(ctx):
    base_dir = ctx.obj['CONFIG']['BASE_DIR']
    dirs = _get_sub_dirs(base_dir)
    git_repos = _get_git_repos(dirs, base_dir)

@main.command()
def test():
    click.echo(os.getcwd())
<commit_msg>Fleshed out _get_git_repos<commit_after>import click
import os
import json
from typing import List

CONFIG_PATH = click.get_app_dir("qs") + "config.json"

def _load_config() -> None:
    if _config_exists():
        with open(CONFIG_PATH, 'r') as f:
            config = json.load(f)
    else:
        config = _create_config()
    return config

def _config_exists() -> None:
    if os.path.isfile(CONFIG_PATH):
        return True
    else:
        return False

def _create_config() -> None:
    config = {'BASE_DIR': os.path.expanduser("~/Projects")}
    with open('config.json', 'w') as f:
        json.dump(config, f)
    return config

def _get_sub_dirs(base_dir: str) -> List[str]:
    return list(filter(lambda x: os.path.isdir(os.path.join(base_dir, x)), os.listdir(base_dir)))

def _get_git_repos(dirs: List[str], base_dir: str) -> List[str]:
    repos = []
    for directory in dirs:
        path = os.path.join(base_dir, directory + ".git/")
        if os.path.isdir(path):
            repos.append(directory)
    return repos

@click.group()
@click.pass_context
def main(ctx):
    """A simple CLI to aid in common, repetitive development tasks"""
    ctx.obj = {}
    ctx.obj['CONFIG'] = _load_config()

@main.command()
@click.option('--base-dir', '-p', is_flag=True, help='Base Projects Folder')
def config(ctx, base_dir):
    config = _load_config()
    if base_dir:
        config["BASE_DIR"] = base_dir

@main.command()
@click.pass_context
def sync(ctx):
    base_dir = ctx.obj['CONFIG']['BASE_DIR']
    dirs = _get_sub_dirs(base_dir)
    git_repos = _get_git_repos(dirs, base_dir)

@main.command()
def test():
    click.echo(os.getcwd())
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-----------------------------------------------------------------------------
#   Copyright (c) 2008-2011, David P. D. Moss. All rights reserved.
#
#   Released under the BSD license. See the LICENSE file for details.
#-----------------------------------------------------------------------------
#
#   DISCLAIMER
#
#   netaddr is not sponsored nor endorsed by the IEEE.
#
#   Use of data from the IEEE (Institute of Electrical and Electronics
#   Engineers) is subject to copyright. See the following URL for
#   details :-
#
#    U{http://www.ieee.org/web/publications/rights/legal.html}
#
#   IEEE data files included with netaddr are not modified in any way but are
#   parsed and made available to end users through an API. There is no
#   guarantee that referenced files are not out of date.
#
#   See README file and source code for URLs to latest copies of the relevant
#   files.
#
#-----------------------------------------------------------------------------
"""
Provides access to public OUI and IAB registration data published by the IEEE.

More details can be found at the following URLs :-

    - IEEE Home Page - U{http://www.ieee.org/}
    - Registration Authority Home Page - U{http://standards.ieee.org/regauth/}
"""

import os as _os
import sys as _sys
import os.path as _path
import csv as _csv

from netaddr.core import Subscriber, Publisher, dos2unix

#-----------------------------------------------------------------------------

#: Path to local copy of IEEE OUI Registry data file.
OUI_REGISTRY = _path.join(_path.dirname(__file__), 'oui.txt')
#: Path to netaddr OUI index file.
OUI_METADATA = _path.join(_path.dirname(__file__), 'oui.idx')

#: OUI index lookup dictionary.
OUI_INDEX = {}

#: Path to local copy of IEEE IAB Registry data file.
IAB_REGISTRY = _path.join(_path.dirname(__file__), 'iab.txt')

#: Path to netaddr IAB index file.
IAB_METADATA = _path.join(_path.dirname(__file__), 'iab.idx')

#: IAB index lookup dictionary.
IAB_INDEX = {}

#-----------------------------------------------------------------------------
class FileIndexer(Subscriber):
    """
    A concrete Subscriber that receives OUI record offset information that is
    written to an index data file as a set of comma separated records.
    """
    def __init__(self, index_file):
        """
        Constructor.

        @param index_file: a file-like object or name of index file where
            index records will be written.
        """
        if hasattr(index_file, 'readline') and hasattr(index_file, 'tell'):
            self.fh = index_file
        else:
            self.fh = open(index_file, 'w')

        self.writer = _csv.writer(self.fh, lineterminator="\n")

    def update(self, data):
        """
        Receives and writes index data to a CSV data file.

        @param data: record containing offset record information.
        """
        self.writer.writerow(data)

#-----------------------------------------------------------------------------
class OUIIndexParser(Publisher):
    """
    A concrete Publisher that parses OUI (Organisationally Unique Identifier)
    records from IEEE text-based registration files

    It notifies registered Subscribers as each record is encountered, passing
    on the record's position relative to the start of the file (offset) and
    the size of the record (in bytes).

    The file processed by this parser is available online from this URL :-

        - U{http://standards.ieee.org/regauth/oui/oui.txt}

    This is a sample of the record structure expected::

        00-CA-FE   (hex)        ACME CORPORATION
        00CAFE     (base 16)        ACME CORPORATION
                        1 MAIN STREET
                        SPRINGFIELD
                        UNITED STATES
    """
    def __init__(self, ieee_file):
        """
        Constructor.

        @param ieee_file: a file-like object or name of file containing OUI
            records. When using a file-like object always open it in binary
            mode otherwise offsets will probably misbehave.
        """
        super(OUIIndexParser, self).__init__()

        if hasattr(ieee_file, 'readline') and hasattr(ieee_file, 'tell'):
            self.fh = ieee_file
        else:
            self.fh = open(ieee_file)

    def parse(self):
        """
        Starts the parsing process which detects records and notifies
        registered subscribers as it finds each OUI record.
        """
        skip_header = True
        record = None
        size = 0

        while True:
            line = self.fh.readline() # unbuffered to obtain correct offsets

            if not line:
                break   # EOF, we're done

            if skip_header and '(hex)' in line:
                skip_header = False

            if skip_header:
                #   ignoring header section
                continue

            if '(hex)' in line:
                #   record start
                if record is not None:
                    #   a complete record.
                    record.append(size)
                    self.notify(record)

                size = len(line)
                offset = (self.fh.tell() - len(line))
                oui = line.split()[0]
                index = int(oui.replace('-', ''), 16)
                record = [index, offset]
            else:
                #   within record
                size += len(line)

        #   process final record on loop exit
        record.append(size)
        self.notify(record)

#-----------------------------------------------------------------------------
class IABIndexParser(Publisher):
    """
    A concrete Publisher that parses IAB (Individual Address Block) records
    from IEEE text-based registration files

    It notifies registered Subscribers as each record is encountered, passing
    on the record's position relative to the start of the file (offset) and
    the size of the record (in bytes).

    The file processed by this parser is available online from this URL :-

        - U{http://standards.ieee.org/regauth/oui/iab.txt}

    This is a sample of the record structure expected::

        00-50-C2   (hex)        ACME CORPORATION
        ABC000-ABCFFF     (base 16)        ACME CORPORATION
                        1 MAIN STREET
                        SPRINGFIELD
                        UNITED STATES
    """
    def __init__(self, ieee_file):
        """
        Constructor.

        @param ieee_file: a file-like object or name of file containing IAB
            records. When using a file-like object always open it in binary
            mode otherwise offsets will probably misbehave.
        """
        super(IABIndexParser, self).__init__()

        if hasattr(ieee_file, 'readline') and hasattr(ieee_file, 'tell'):
            self.fh = ieee_file
        else:
            self.fh = open(ieee_file)

    def parse(self):
        """
        Starts the parsing process which detects records and notifies
        registered subscribers as it finds each IAB record.
        """
        skip_header = True
        record = None
        size = 0
        while True:
            line = self.fh.readline()   # unbuffered

            if not line:
                break   # EOF, we're done

            if skip_header and '(hex)' in line:
                skip_header = False

            if skip_header:
                #   ignoring header section
                continue

            if '(hex)' in line:
                #   record start
                if record is not None:
                    record.append(size)
                    self.notify(record)

                offset = (self.fh.tell() - len(line))
                iab_prefix = line.split()[0]
                index = iab_prefix
                record = [index, offset]
                size = len(line)
            elif '(base 16)' in line:
                #   within record
                size += len(line)
                prefix = record[0].replace('-', '')
                suffix = line.split()[0]
                suffix = suffix.split('-')[0]
                record[0] = (int(prefix + suffix, 16)) >> 12
            else:
                #   within record
                size += len(line)

        #   process final record on loop exit
        record.append(size)
        self.notify(record)

#-----------------------------------------------------------------------------
def create_indices():
    """Create indices for OUI and IAB file based lookups"""
    oui_parser = OUIIndexParser(OUI_REGISTRY)
    oui_parser.attach(FileIndexer(OUI_METADATA))
    oui_parser.parse()

    iab_parser = IABIndexParser(IAB_REGISTRY)
    iab_parser.attach(FileIndexer(IAB_METADATA))
    iab_parser.parse()

#-----------------------------------------------------------------------------
def load_indices():
    """Load OUI and IAB lookup indices into memory"""
    for row in _csv.reader(open(OUI_METADATA)):
        (key, offset, size) = [int(_) for _ in row]
        OUI_INDEX.setdefault(key, [])
        OUI_INDEX[key].append((offset, size))

    for row in _csv.reader(open(IAB_METADATA)):
        (key, offset, size) = [int(_) for _ in row]
        IAB_INDEX.setdefault(key, [])
        IAB_INDEX[key].append((offset, size))

#-----------------------------------------------------------------------------
def get_latest_files():
    """Download the latest files from the IEEE"""
    if _sys.version_info[0] == 3:
        #   Python 3.x
        from urllib.request import Request, urlopen
    else:
        #   Python 2.x
        from urllib2 import Request, urlopen

    urls = [
        'http://standards.ieee.org/regauth/oui/oui.txt',
        'http://standards.ieee.org/regauth/oui/iab.txt',
    ]

    for url in urls:
        _sys.stdout.write('downloading latest copy of %s\n' % url)
        request = Request(url)
        response = urlopen(request)
        save_path = _path.dirname(__file__)
        filename = _path.join(save_path, _os.path.basename(response.geturl()))
        fh = open(filename, 'w')
        fh.write(response.read())
        fh.close()

        #   Make sure the line endings are consistent across platforms.
        dos2unix(filename)

#-----------------------------------------------------------------------------
if __name__ == '__main__':
    #   Generate indices when module is executed as a script.
    get_latest_files()
    create_indices()
else:
    #   On module load read indices in memory to enable lookups.
    load_indices()

<commit_msg>- added execute permissions to this file<commit_after>#!/usr/bin/env python
#-----------------------------------------------------------------------------
#   Copyright (c) 2008-2011, David P. D. Moss. All rights reserved.
#
#   Released under the BSD license. See the LICENSE file for details.
#-----------------------------------------------------------------------------
#
#   DISCLAIMER
#
#   netaddr is not sponsored nor endorsed by the IEEE.
#
#   Use of data from the IEEE (Institute of Electrical and Electronics
#   Engineers) is subject to copyright. See the following URL for
#   details :-
#
#    U{http://www.ieee.org/web/publications/rights/legal.html}
#
#   IEEE data files included with netaddr are not modified in any way but are
#   parsed and made available to end users through an API. There is no
#   guarantee that referenced files are not out of date.
#
#   See README file and source code for URLs to latest copies of the relevant
#   files.
#
#-----------------------------------------------------------------------------
"""
Provides access to public OUI and IAB registration data published by the IEEE.

More details can be found at the following URLs :-

    - IEEE Home Page - U{http://www.ieee.org/}
    - Registration Authority Home Page - U{http://standards.ieee.org/regauth/}
"""

import os as _os
import sys as _sys
import os.path as _path
import csv as _csv

from netaddr.core import Subscriber, Publisher, dos2unix

#-----------------------------------------------------------------------------

#: Path to local copy of IEEE OUI Registry data file.
OUI_REGISTRY = _path.join(_path.dirname(__file__), 'oui.txt')
#: Path to netaddr OUI index file.
OUI_METADATA = _path.join(_path.dirname(__file__), 'oui.idx')

#: OUI index lookup dictionary.
OUI_INDEX = {}

#: Path to local copy of IEEE IAB Registry data file.
IAB_REGISTRY = _path.join(_path.dirname(__file__), 'iab.txt')

#: Path to netaddr IAB index file.
IAB_METADATA = _path.join(_path.dirname(__file__), 'iab.idx')

#: IAB index lookup dictionary.
IAB_INDEX = {}

#-----------------------------------------------------------------------------
class FileIndexer(Subscriber):
    """
    A concrete Subscriber that receives OUI record offset information that is
    written to an index data file as a set of comma separated records.
    """
    def __init__(self, index_file):
        """
        Constructor.

        @param index_file: a file-like object or name of index file where
            index records will be written.
        """
        if hasattr(index_file, 'readline') and hasattr(index_file, 'tell'):
            self.fh = index_file
        else:
            self.fh = open(index_file, 'w')

        self.writer = _csv.writer(self.fh, lineterminator="\n")

    def update(self, data):
        """
        Receives and writes index data to a CSV data file.

        @param data: record containing offset record information.
        """
        self.writer.writerow(data)

#-----------------------------------------------------------------------------
class OUIIndexParser(Publisher):
    """
    A concrete Publisher that parses OUI (Organisationally Unique Identifier)
    records from IEEE text-based registration files

    It notifies registered Subscribers as each record is encountered, passing
    on the record's position relative to the start of the file (offset) and
    the size of the record (in bytes).

    The file processed by this parser is available online from this URL :-

        - U{http://standards.ieee.org/regauth/oui/oui.txt}

    This is a sample of the record structure expected::

        00-CA-FE   (hex)        ACME CORPORATION
        00CAFE     (base 16)        ACME CORPORATION
                        1 MAIN STREET
                        SPRINGFIELD
                        UNITED STATES
    """
    def __init__(self, ieee_file):
        """
        Constructor.

        @param ieee_file: a file-like object or name of file containing OUI
            records. When using a file-like object always open it in binary
            mode otherwise offsets will probably misbehave.
        """
        super(OUIIndexParser, self).__init__()

        if hasattr(ieee_file, 'readline') and hasattr(ieee_file, 'tell'):
            self.fh = ieee_file
        else:
            self.fh = open(ieee_file)

    def parse(self):
        """
        Starts the parsing process which detects records and notifies
        registered subscribers as it finds each OUI record.
        """
        skip_header = True
        record = None
        size = 0

        while True:
            line = self.fh.readline() # unbuffered to obtain correct offsets

            if not line:
                break   # EOF, we're done

            if skip_header and '(hex)' in line:
                skip_header = False

            if skip_header:
                #   ignoring header section
                continue

            if '(hex)' in line:
                #   record start
                if record is not None:
                    #   a complete record.
                    record.append(size)
                    self.notify(record)

                size = len(line)
                offset = (self.fh.tell() - len(line))
                oui = line.split()[0]
                index = int(oui.replace('-', ''), 16)
                record = [index, offset]
            else:
                #   within record
                size += len(line)

        #   process final record on loop exit
        record.append(size)
        self.notify(record)

#-----------------------------------------------------------------------------
class IABIndexParser(Publisher):
    """
    A concrete Publisher that parses IAB (Individual Address Block) records
    from IEEE text-based registration files

    It notifies registered Subscribers as each record is encountered, passing
    on the record's position relative to the start of the file (offset) and
    the size of the record (in bytes).

    The file processed by this parser is available online from this URL :-

        - U{http://standards.ieee.org/regauth/oui/iab.txt}

    This is a sample of the record structure expected::

        00-50-C2   (hex)        ACME CORPORATION
        ABC000-ABCFFF     (base 16)        ACME CORPORATION
                        1 MAIN STREET
                        SPRINGFIELD
                        UNITED STATES
    """
    def __init__(self, ieee_file):
        """
        Constructor.

        @param ieee_file: a file-like object or name of file containing IAB
            records. When using a file-like object always open it in binary
            mode otherwise offsets will probably misbehave.
        """
        super(IABIndexParser, self).__init__()

        if hasattr(ieee_file, 'readline') and hasattr(ieee_file, 'tell'):
            self.fh = ieee_file
        else:
            self.fh = open(ieee_file)

    def parse(self):
        """
        Starts the parsing process which detects records and notifies
        registered subscribers as it finds each IAB record.
        """
        skip_header = True
        record = None
        size = 0
        while True:
            line = self.fh.readline()   # unbuffered

            if not line:
                break   # EOF, we're done

            if skip_header and '(hex)' in line:
                skip_header = False

            if skip_header:
                #   ignoring header section
                continue

            if '(hex)' in line:
                #   record start
                if record is not None:
                    record.append(size)
                    self.notify(record)

                offset = (self.fh.tell() - len(line))
                iab_prefix = line.split()[0]
                index = iab_prefix
                record = [index, offset]
                size = len(line)
            elif '(base 16)' in line:
                #   within record
                size += len(line)
                prefix = record[0].replace('-', '')
                suffix = line.split()[0]
                suffix = suffix.split('-')[0]
                record[0] = (int(prefix + suffix, 16)) >> 12
            else:
                #   within record
                size += len(line)

        #   process final record on loop exit
        record.append(size)
        self.notify(record)

#-----------------------------------------------------------------------------
def create_indices():
    """Create indices for OUI and IAB file based lookups"""
    oui_parser = OUIIndexParser(OUI_REGISTRY)
    oui_parser.attach(FileIndexer(OUI_METADATA))
    oui_parser.parse()

    iab_parser = IABIndexParser(IAB_REGISTRY)
    iab_parser.attach(FileIndexer(IAB_METADATA))
    iab_parser.parse()

#-----------------------------------------------------------------------------
def load_indices():
    """Load OUI and IAB lookup indices into memory"""
    for row in _csv.reader(open(OUI_METADATA)):
        (key, offset, size) = [int(_) for _ in row]
        OUI_INDEX.setdefault(key, [])
        OUI_INDEX[key].append((offset, size))

    for row in _csv.reader(open(IAB_METADATA)):
        (key, offset, size) = [int(_) for _ in row]
        IAB_INDEX.setdefault(key, [])
        IAB_INDEX[key].append((offset, size))

#-----------------------------------------------------------------------------
def get_latest_files():
    """Download the latest files from the IEEE"""
    if _sys.version_info[0] == 3:
        #   Python 3.x
        from urllib.request import Request, urlopen
    else:
        #   Python 2.x
        from urllib2 import Request, urlopen

    urls = [
        'http://standards.ieee.org/regauth/oui/oui.txt',
        'http://standards.ieee.org/regauth/oui/iab.txt',
    ]

    for url in urls:
        _sys.stdout.write('downloading latest copy of %s\n' % url)
        request = Request(url)
        response = urlopen(request)
        save_path = _path.dirname(__file__)
        filename = _path.join(save_path, _os.path.basename(response.geturl()))
        fh = open(filename, 'w')
        fh.write(response.read())
        fh.close()

        #   Make sure the line endings are consistent across platforms.
        dos2unix(filename)

#-----------------------------------------------------------------------------
if __name__ == '__main__':
    #   Generate indices when module is executed as a script.
    get_latest_files()
    create_indices()
else:
    #   On module load read indices in memory to enable lookups.
    load_indices()

<|endoftext|>


<commit_before>import pprint
from pigui.button import Button
from pigui.canvas import Canvas
from pigui.color import *
from pigui.label import Label

import netifaces

__author__ = 'richard'

class NetworkSettings(Canvas):
    def __init__(self, name,interface):
        super(NetworkSettings,self).__init__(name)
        self.interface = interface

    def __setup__(self, screen):

        self.button = Button(screen)
        self.label = Label(screen)

        self.register_widgets(self.button)
        self.register_widgets(self.label)

        self.label.add_label('welcome', 'Network Settings', 125, 5)
        self.label.add_attribute('welcome', 'size', 34)

        self.label.add_label('welcome_undertext', 'For interface: ' + self.interface, 125, 30)
        self.label.add_attribute('welcome_undertext', 'size', 15)

        self.button.add_button('back', self.runner.back_canvas, 0, 0, 30, 30)
        self.button.add_attributes('back', 'text', '<<')
        self.button.add_attributes('back', 'color', cream)
        self.button.add_attributes('back', 'text_size', 20)

        #information labels
        self.label.add_label('ip_addr_label','IP Address',30,60)
        self.label.add_attribute('ip_addr_label','size',20)
        self.label.add_label('ip_addr', '',90,60)
        self.label.add_attribute('ip_addr','size',20)

        self.label.add_label('mac_addr_label','MAC Address',30,75)
        self.label.add_attribute('mac_addr_label','',20)
        self.label.add_label('mac_addr','',90,75)
        self.label.add_attribute('mac_addr','size',20)

        self.get_information()

    def get_information(self):
        pprint.pprint(netifaces.ifaddresses(self.interface))

        info = netifaces.ifaddress(self.interface)
        #get the ip addr
        if 2 in info:
            self.label.set_text('ip_addr',info[2]['addr'] + ' bc:' + info[2]['broadcast'] + ' nm:' + info[2]['netmask'])

        #get mac_address
        if 17 in info:
           self.label.set_text('mac_addr',info[17]['addr'])<commit_msg>fixed misspelling on ifadresses<commit_after>import pprint
from pigui.button import Button
from pigui.canvas import Canvas
from pigui.color import *
from pigui.label import Label

import netifaces

__author__ = 'richard'

class NetworkSettings(Canvas):
    def __init__(self, name,interface):
        super(NetworkSettings,self).__init__(name)
        self.interface = interface

    def __setup__(self, screen):

        self.button = Button(screen)
        self.label = Label(screen)

        self.register_widgets(self.button)
        self.register_widgets(self.label)

        self.label.add_label('welcome', 'Network Settings', 125, 5)
        self.label.add_attribute('welcome', 'size', 34)

        self.label.add_label('welcome_undertext', 'For interface: ' + self.interface, 125, 30)
        self.label.add_attribute('welcome_undertext', 'size', 15)

        self.button.add_button('back', self.runner.back_canvas, 0, 0, 30, 30)
        self.button.add_attributes('back', 'text', '<<')
        self.button.add_attributes('back', 'color', cream)
        self.button.add_attributes('back', 'text_size', 20)

        #information labels
        self.label.add_label('ip_addr_label','IP Address',30,60)
        self.label.add_attribute('ip_addr_label','size',20)
        self.label.add_label('ip_addr', '',90,60)
        self.label.add_attribute('ip_addr','size',20)

        self.label.add_label('mac_addr_label','MAC Address',30,75)
        self.label.add_attribute('mac_addr_label','',20)
        self.label.add_label('mac_addr','',90,75)
        self.label.add_attribute('mac_addr','size',20)

        self.get_information()

    def get_information(self):
        pprint.pprint(netifaces.ifaddresses(self.interface))

        info = netifaces.ifaddresses(self.interface)
        #get the ip addr
        if 2 in info:
            self.label.set_text('ip_addr',info[2]['addr'] + ' bc:' + info[2]['broadcast'] + ' nm:' + info[2]['netmask'])

        #get mac_address
        if 17 in info:
           self.label.set_text('mac_addr',info[17]['addr'])<|endoftext|>


<commit_before>"""Release data for NetworkX."""

#    Copyright (C) 2004-2007 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    Distributed under the terms of the GNU Lesser General Public License
#    http://www.gnu.org/copyleft/lesser.html

__author__ = """Aric Hagberg (hagberg@lanl.gov)\nPieter Swart (swart@lanl.gov)\nDan Schult (dschult@colgate.edu)"""

name = 'networkx'
version = '0.35.1'

description = "Python package for creating and manipulating graphs and networks"

long_description = \
"""
NetworkX is a Python package for the creation, manipulation, and
study of the structure, dynamics, and functions of complex networks.

"""
license = 'LGPL'
authors = {'Hagberg' : ('Aric Hagberg','hagberg@lanl.gov'),
           'Schult' : ('Dan Schult','dschult@colgate.edu'),
           'Swart' : ('Pieter Swart','swart@lanl.gov')
           }
url = 'http://networkx.lanl.gov/'
download_url="http://networkx.lanl.gov/download"
platforms = ['Linux','Mac OSX','Windows XP/2000/NT']
keywords = ['Networks', 'Graph Theory', 'Mathematics', 'network', 'graph', 'discrete mathematics', 'math']
classifiers = [
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'Intended Audience :: Science/Research',
        'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Topic :: Software Development :: Libraries :: Python Modules',
        'Topic :: Scientific/Engineering :: Bio-Informatics',
        'Topic :: Scientific/Engineering :: Information Analysis',
        'Topic :: Scientific/Engineering :: Mathematics',
        'Topic :: Scientific/Engineering :: Physics',
        ]

# Get date dynamically
import time
date = time.asctime()
del time

<commit_msg>bump release to 0.36-svn again<commit_after>"""Release data for NetworkX."""

#    Copyright (C) 2004-2007 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    Distributed under the terms of the GNU Lesser General Public License
#    http://www.gnu.org/copyleft/lesser.html

__author__ = """Aric Hagberg (hagberg@lanl.gov)\nPieter Swart (swart@lanl.gov)\nDan Schult (dschult@colgate.edu)"""

name = 'networkx'
version = '0.36-svn'

description = "Python package for creating and manipulating graphs and networks"

long_description = \
"""
NetworkX is a Python package for the creation, manipulation, and
study of the structure, dynamics, and functions of complex networks.

"""
license = 'LGPL'
authors = {'Hagberg' : ('Aric Hagberg','hagberg@lanl.gov'),
           'Schult' : ('Dan Schult','dschult@colgate.edu'),
           'Swart' : ('Pieter Swart','swart@lanl.gov')
           }
url = 'http://networkx.lanl.gov/'
download_url="http://networkx.lanl.gov/download"
platforms = ['Linux','Mac OSX','Windows XP/2000/NT']
keywords = ['Networks', 'Graph Theory', 'Mathematics', 'network', 'graph', 'discrete mathematics', 'math']
classifiers = [
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'Intended Audience :: Science/Research',
        'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Topic :: Software Development :: Libraries :: Python Modules',
        'Topic :: Scientific/Engineering :: Bio-Informatics',
        'Topic :: Scientific/Engineering :: Information Analysis',
        'Topic :: Scientific/Engineering :: Mathematics',
        'Topic :: Scientific/Engineering :: Physics',
        ]

# Get date dynamically
import time
date = time.asctime()
del time

<|endoftext|>


<commit_before>import os
import argparse

from neuralmonkey.logging import log, log_print
from neuralmonkey.config.configuration import Configuration
from neuralmonkey.learning_utils import (evaluation, run_on_dataset,
                                         print_final_evaluation)

CONFIG = Configuration()
CONFIG.add_argument('tf_manager')
CONFIG.add_argument('output')
CONFIG.add_argument('postprocess')
CONFIG.add_argument('evaluation')
CONFIG.add_argument('runners')
CONFIG.add_argument('batch_size')
CONFIG.add_argument('threads', required=False, default=4)
CONFIG.add_argument('runners_batch_size', required=False, default=None)
# ignore arguments which are just for training
CONFIG.ignore_argument('val_dataset')
CONFIG.ignore_argument('trainer')
CONFIG.ignore_argument('name')
CONFIG.ignore_argument('train_dataset')
CONFIG.ignore_argument('epochs')
CONFIG.ignore_argument('test_datasets')
CONFIG.ignore_argument('initial_variables')
CONFIG.ignore_argument('validation_period')
CONFIG.ignore_argument('val_preview_input_series')
CONFIG.ignore_argument('val_preview_output_series')
CONFIG.ignore_argument('val_preview_num_examples')
CONFIG.ignore_argument('logging_period')
CONFIG.ignore_argument('visualize_embeddings')
CONFIG.ignore_argument('minimize')
CONFIG.ignore_argument('random_seed')
CONFIG.ignore_argument('save_n_best')
CONFIG.ignore_argument('overwrite_output_dir')

def default_variable_file(output_dir):
    variables_file = os.path.join(output_dir, "variables.data")
    cont_index = 1

    def continuation_file():
        return os.path.join(output_dir,
                            "variables.data.cont-{}.best".format(cont_index))
    while os.path.exists(continuation_file()):
        variables_file = continuation_file()
        cont_index += 1

    return variables_file

def initialize_for_running(output_dir, tf_manager, variable_files) -> None:
    """Restore either default variables of from configuration.

    Arguments:
       output_dir: Training output directory.
       tf_manager: TensorFlow manager.
       variable_files: Files with variables to be restored or None if the
           default variables should be used.
    """
    # pylint: disable=no-member
    log_print("")

    if variable_files is None:
        default_varfile = default_variable_file(output_dir)

        log("Default variable file '{}' will be used for loading variables."
            .format(default_varfile))

        variable_files = [default_varfile]

    for vfile in variable_files:
        if not os.path.exists("{}.index".format(vfile)):
            log("Index file for var prefix {} does not exist".format(vfile),
                color="red")
            exit(1)

    tf_manager.restore(variable_files)

    log_print("")

def main() -> None:
    # pylint: disable=no-member,broad-except
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("config", metavar="INI-FILE",
                        help="the configuration file of the experiment")
    parser.add_argument('datasets', metavar='INI-TEST-DATASETS',
                        help="the configuration of the test datasets")
    parser.add_argument("-g", "--grid", dest="grid", action="store_true",
                        help="look at the SGE variables for slicing the data")
    args = parser.parse_args()

    test_datasets = Configuration()
    test_datasets.add_argument('test_datasets')
    test_datasets.add_argument('variables')

    CONFIG.load_file(args.config)
    CONFIG.build_model()
    test_datasets.load_file(args.datasets)
    test_datasets.build_model()
    datasets_model = test_datasets.model
    initialize_for_running(CONFIG.model.output, CONFIG.model.tf_manager,
                           datasets_model.variables)

    print("")

    evaluators = [(e[0], e[0], e[1]) if len(e) == 2 else e
                  for e in CONFIG.model.evaluation]

    if args.grid and len(datasets_model.test_datasets) > 1:
        raise ValueError("Only one test dataset supported when using --grid")

    for dataset in datasets_model.test_datasets:
        if args.grid:
            if ("SGE_TASK_FIRST" not in os.environ
                    or "SGE_TASK_LAST" not in os.environ
                    or "SGE_TASK_STEPSIZE" not in os.environ
                    or "SGE_TASK_ID" not in os.environ):
                raise EnvironmentError(
                    "Some SGE environment variables are missing")

            length = int(os.environ["SGE_TASK_STEPSIZE"])
            start = int(os.environ["SGE_TASK_ID"]) - 1
            end = int(os.environ["SGE_TASK_LAST"]) - 1

            if start + length > end:
                length = end - start + 1

            log("Running grid task {} starting at {} with step {}"
                .format(start // length, start, length))

            dataset = dataset.subset(start, length)

        if CONFIG.model.runners_batch_size is None:
            runners_batch_size = CONFIG.model.batch_size
        else:
            runners_batch_size = CONFIG.model.runners_batch_size

        execution_results, output_data = run_on_dataset(
            CONFIG.model.tf_manager, CONFIG.model.runners,
            dataset, CONFIG.model.postprocess, write_out=True,
            batch_size=runners_batch_size, log_progress=60)
        # TODO what if there is no ground truth
        eval_result = evaluation(evaluators, dataset, CONFIG.model.runners,
                                 execution_results, output_data)
        if eval_result:
            print_final_evaluation(dataset.name, eval_result)

    while len(CONFIG.model.tf_manager.sessions):
        del CONFIG.model.tf_manager.sessions[0]
<commit_msg>pylint complinance fix<commit_after>import os
import argparse

from neuralmonkey.logging import log, log_print
from neuralmonkey.config.configuration import Configuration
from neuralmonkey.learning_utils import (evaluation, run_on_dataset,
                                         print_final_evaluation)

CONFIG = Configuration()
CONFIG.add_argument('tf_manager')
CONFIG.add_argument('output')
CONFIG.add_argument('postprocess')
CONFIG.add_argument('evaluation')
CONFIG.add_argument('runners')
CONFIG.add_argument('batch_size')
CONFIG.add_argument('threads', required=False, default=4)
CONFIG.add_argument('runners_batch_size', required=False, default=None)
# ignore arguments which are just for training
CONFIG.ignore_argument('val_dataset')
CONFIG.ignore_argument('trainer')
CONFIG.ignore_argument('name')
CONFIG.ignore_argument('train_dataset')
CONFIG.ignore_argument('epochs')
CONFIG.ignore_argument('test_datasets')
CONFIG.ignore_argument('initial_variables')
CONFIG.ignore_argument('validation_period')
CONFIG.ignore_argument('val_preview_input_series')
CONFIG.ignore_argument('val_preview_output_series')
CONFIG.ignore_argument('val_preview_num_examples')
CONFIG.ignore_argument('logging_period')
CONFIG.ignore_argument('visualize_embeddings')
CONFIG.ignore_argument('minimize')
CONFIG.ignore_argument('random_seed')
CONFIG.ignore_argument('save_n_best')
CONFIG.ignore_argument('overwrite_output_dir')

def default_variable_file(output_dir):
    variables_file = os.path.join(output_dir, "variables.data")
    cont_index = 1

    def continuation_file():
        return os.path.join(output_dir,
                            "variables.data.cont-{}.best".format(cont_index))
    while os.path.exists(continuation_file()):
        variables_file = continuation_file()
        cont_index += 1

    return variables_file

def initialize_for_running(output_dir, tf_manager, variable_files) -> None:
    """Restore either default variables of from configuration.

    Arguments:
       output_dir: Training output directory.
       tf_manager: TensorFlow manager.
       variable_files: Files with variables to be restored or None if the
           default variables should be used.
    """
    # pylint: disable=no-member
    log_print("")

    if variable_files is None:
        default_varfile = default_variable_file(output_dir)

        log("Default variable file '{}' will be used for loading variables."
            .format(default_varfile))

        variable_files = [default_varfile]

    for vfile in variable_files:
        if not os.path.exists("{}.index".format(vfile)):
            log("Index file for var prefix {} does not exist".format(vfile),
                color="red")
            exit(1)

    tf_manager.restore(variable_files)

    log_print("")

def main() -> None:
    # pylint: disable=no-member,broad-except
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("config", metavar="INI-FILE",
                        help="the configuration file of the experiment")
    parser.add_argument('datasets', metavar='INI-TEST-DATASETS',
                        help="the configuration of the test datasets")
    parser.add_argument("-g", "--grid", dest="grid", action="store_true",
                        help="look at the SGE variables for slicing the data")
    args = parser.parse_args()

    test_datasets = Configuration()
    test_datasets.add_argument('test_datasets')
    test_datasets.add_argument('variables')

    CONFIG.load_file(args.config)
    CONFIG.build_model()
    test_datasets.load_file(args.datasets)
    test_datasets.build_model()
    datasets_model = test_datasets.model
    initialize_for_running(CONFIG.model.output, CONFIG.model.tf_manager,
                           datasets_model.variables)

    print("")

    evaluators = [(e[0], e[0], e[1]) if len(e) == 2 else e
                  for e in CONFIG.model.evaluation]

    if args.grid and len(datasets_model.test_datasets) > 1:
        raise ValueError("Only one test dataset supported when using --grid")

    for dataset in datasets_model.test_datasets:
        if args.grid:
            if ("SGE_TASK_FIRST" not in os.environ
                    or "SGE_TASK_LAST" not in os.environ
                    or "SGE_TASK_STEPSIZE" not in os.environ
                    or "SGE_TASK_ID" not in os.environ):
                raise EnvironmentError(
                    "Some SGE environment variables are missing")

            length = int(os.environ["SGE_TASK_STEPSIZE"])
            start = int(os.environ["SGE_TASK_ID"]) - 1
            end = int(os.environ["SGE_TASK_LAST"]) - 1

            if start + length > end:
                length = end - start + 1

            log("Running grid task {} starting at {} with step {}"
                .format(start // length, start, length))

            dataset = dataset.subset(start, length)

        if CONFIG.model.runners_batch_size is None:
            runners_batch_size = CONFIG.model.batch_size
        else:
            runners_batch_size = CONFIG.model.runners_batch_size

        execution_results, output_data = run_on_dataset(
            CONFIG.model.tf_manager, CONFIG.model.runners,
            dataset, CONFIG.model.postprocess, write_out=True,
            batch_size=runners_batch_size, log_progress=60)
        # TODO what if there is no ground truth
        eval_result = evaluation(evaluators, dataset, CONFIG.model.runners,
                                 execution_results, output_data)
        if eval_result:
            print_final_evaluation(dataset.name, eval_result)

    for _ in range(len(CONFIG.model.tf_manager.sessions)):
        del CONFIG.model.tf_manager.sessions[0]
<|endoftext|>


<commit_before>#!/usr/bin/env python
import rospy
from sensor_msgs.msg import Image
from cv_bridge import CvBridge, CvBridgeError
import cv2
from flexbe_core import EventState, Logger
from flexbe_core.proxy import ProxySubscriberCached

from sensor_msgs.msg import PointCloud2

class TakePictureState(EventState):
	'''
	Stores the picture  of the given topic.

	#> Image 	Image		The received pointcloud.

	<= done 			The picture has been received and stored.

	'''

	def __init__(self):
		super(TakePictureState, self).__init__(outcomes = ['done'],	output_keys = ['Image'])
                self._topic = '/head_xtion/rgb/image_rect'
		self._sub = ProxySubscriberCached({self._topic:Image})

	def execute(self, userdata):

		if self._sub.has_msg(self._topic):
			userdata.Image = self._sub.get_last_msg(self._topic)
		return 'done'

<commit_msg>take picture state updated<commit_after>#!/usr/bin/env python
import rospy
from sensor_msgs.msg import Image
from cv_bridge import CvBridge, CvBridgeError
import cv2
from flexbe_core import EventState, Logger
from flexbe_core.proxy import ProxySubscriberCached

from sensor_msgs.msg import PointCloud2

class TakePictureState(EventState):
	'''
	Stores the picture  of the given topic.

	#> Image 	Image		The received pointcloud.

	<= done 			The picture has been received and stored.

	'''

	def __init__(self):
		super(TakePictureState, self).__init__(outcomes = ['done'],	output_keys = ['Image'])
        	self._topic = '/head_xtion/rgb/image_rect'
		self._sub = ProxySubscriberCached({self._topic:Image})

	def execute(self, userdata):

		if self._sub.has_msg(self._topic):
			userdata.Image = self._sub.get_last_msg(self._topic)
		return 'done'

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
# Copyright (c) 2012 Michael Hull, Michael Vella
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
#
#  - Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
#  - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#-------------------------------------------------------------------------------

class KineticComponent(object):
    def __init__(self):
        pass

class PointCurrent(KineticComponent):
    def __init__(self):
        pass

class IonChannel(KineticComponent):
    def __init__(self):
        pass

class Synapse(KineticComponent):
    def __init__(self):
        pass

class ExtracellularProperties(KineticComponent):
    def __init__(self):
        pass

class IClamp(PointCurrent):
    def __init__(self,current,delay,duration,fraction_along=None):
        self.name = 'IClamp'
        self.type = 'IClamp'
        self.amp = current
        self.delay = delay
        self.dur = duration

        if fraction_along != None:
            self.fraction_along=fraction_along
        else:
            self.fraction_along=0.5

class Nmodl(IonChannel):
    """
    NMODL (NEURON-Compatible) ion channel description
    plan for now is that these still need to be
    precompiled and located in the correct folder
    """
    def __init__(self,name):
        self.type = 'NMODL'
        self.name = name
        self.attributes={}

#for now use default values:
#    @property
#    def attributes(self):
#        """
#        should be a hash table with the attribute:value relationship
#        """
#        return self.__attributes
#
#    @attributes.setter
#    def attributes(self,attribute,value):
#        self.attributes.append((attribute,value))

<commit_msg>adding MOOSE support<commit_after># -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
# Copyright (c) 2012 Michael Hull, Michael Vella
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
#
#  - Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
#  - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#-------------------------------------------------------------------------------

class KineticComponent(object):
    def __init__(self):
        pass

class PointCurrent(KineticComponent):
    def __init__(self):
        pass

class IonChannel(KineticComponent):
    def __init__(self):
        pass

class Synapse(KineticComponent):
    def __init__(self):
        pass

class ExtracellularProperties(KineticComponent):
    def __init__(self):
        pass

class IClamp(PointCurrent):
    """
    Steady point current injection

    :param current: injected current in pA
    :param delay: initiation of current injection, in ms
    :param duration: current injection duration, in ms
    :param fraction_along: fraction along section for current injection
    """

    def __init__(self,current,delay,duration,fraction_along=None):
        self.name = 'IClamp'
        self.amp = current
        self.delay = delay
        self.dur = duration

        if fraction_along != None:
            self.fraction_along = fraction_along
        else:
            self.fraction_along = 0.5

class Nmodl(IonChannel):
    """
    NMODL (NEURON-Compatible) ion channel description
    plan for now is that these still need to be
    precompiled and located in the correct folder
    """
    def __init__(self,name):
        self.type = 'NMODL'
        self.name = name
        self.attributes={}

<|endoftext|>


<commit_before><commit_msg>Update Q5-Add Page to Index: working solution<commit_after><|endoftext|>


<commit_before>import next.utils as utils
import os
import next.constants as constants
import redis
import StringIO

class Memory(object):
    def __init__(self, collection='', exp_uid='', uid_prefix=''):
        self.key_prefix = collection + uid_prefix.format(exp_uid=exp_uid)
        self.cache = None
        self.max_entry_size = 500000000  # 500MB

    def check_prefix(self):
        if self.key_prefix == '':
            utils.debug_print("butler.memory is deprecated."
                              " Change to butler.experiment.memory or butler.algorithm.memory, etc."
                              " wherever appropriate")

    def ensure_connection(self):
        try:
            if self.cache is None:
                self.cache = redis.StrictRedis(host=constants.MINIONREDIS_HOST, port=constants.MINIONREDIS_PORT)
        except Exception as e:
            raise Exception("Butler.Collection.Memory could not connect with RedisDB: {}".format(e))

    def num_entries(self, size):
        if size % self.max_entry_size == 0:
            return size / self.max_entry_size
        else:
            return (size / self.max_entry_size) + 1

    def set(self, key, value):
        self.check_prefix()
        key = self.key_prefix + key
        try:
            self.ensure_connection()
            l = len(value)
            n = self.num_entries(l)
            utils.debug_print("Setting {} in {} entries".format(l, n))
            for i in range(n):
                k = key + ":" + str(i)
                self.cache.set(k, value[i*self.max_entry_size:(i+1)*self.max_entry_size])
            return self.cache.set(key, "{}:{}".format(str(n), str(l)))
        except Exception as e:
            utils.debug_print("Butler.Collection.Memory.set exception: {}".format(e))
            return False

    def set_file(self, key, f):
        self.check_prefix()
        key = self.key_prefix + key
        try:
            self.ensure_connection()
            f.seek(0, os.SEEK_END)
            l = f.tell()
            f.seek(0, 0)
            n = self.num_entries(l)
            utils.debug_print("Setting {} bytes in {} entries".format(l, n))
            for i in range(n):
                k = key + ":" + str(i)
                v = f.read(self.max_entry_size)
                self.cache.set(k, v)
            return self.cache.set(key, "{}:{}".format(str(n), str(l)))
        except Exception as e:
            utils.debug_print("Butler.Collection.Memory.set_file exception: {}".format(e))
            return False

    def get(self, key):
        self.check_prefix()
        try:
            self.ensure_connection()
            key = self.key_prefix + key
            d = self.cache.get(key)
            n, l = d.split(":")
            l = int(l)
            n = int(n)
            ans = ""
            utils.debug_print("Getting {} bytes in {} entries".format(l, n))
            for i in range(n):
                k = key + ":" + str(i)
                ans += self.cache.get(k)
            return ans
        except Exception as e:
            utils.debug_print("Butler.Collection.Memory.get exception: {}".format(e))
            return None

    def get_file(self, key):
        self.check_prefix()
        try:
            self.ensure_connection()
            key = self.key_prefix + key
            d = self.cache.get(key)
            f = StringIO.StringIO()
            n, l = d.split(":")
            l = int(l)
            n = int(n)
            utils.debug_print("Getting {} bytes in {} entries".format(l, n))
            for i in range(n):
                k = key + ":" + str(i)
                f.write(self.cache.get(k))
            f.seek(0, 0)
            return f
        except Exception as e:
            utils.debug_print("Butler.Collection.Memory.get_file exception: {}".format(e))
            return None

    def lock(self, name, **kwargs):
        try:
            self.ensure_connection()
            name = self.key_prefix + name
            return self.cache.lock(name, **kwargs)
        except Exception as e:
            utils.debug_print("Butler.Collection.Memory.lock exception: {}".format(e))
            return None

    def exists(self, key):
        try:
            self.ensure_connection()
            key = self.key_prefix + key
            return self.cache.exists(key)
        except Exception as e:
            utils.debug_print("Butler.Collection.Memory.exists exception: {}".format(e))
            return None

class Collection(object):
    def __init__(self, collection, uid_prefix, exp_uid, db, timing=True):
        self.collection = collection
        self.db = db
        self.exp_uid = exp_uid
        self.uid_prefix = uid_prefix
        self.get_durations = 0.0
        self.set_durations = 0.0
        self.timing = timing
        self.memory = Memory(collection, exp_uid)

    def set(self, uid="", key=None, value=None, exp=None):
        """
        Set an object in the collection, or an entry in an object in the collection.
        * key == None:    collection[uid] = value
        * key != None:    collection[uid][key] = value
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp is None else exp))
        if not key:
            self.timed(self.db.set_doc)(self.collection, uid, value)
        else:
            self.timed(self.db.set)(self.collection, uid, key, value)

    def set_many(self, uid="", key_value_dict=None, exp=None):
        """
        For each key in key_value_dict, sets value by key_value_dict[key]
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp is None else exp))
        return self.timed(self.db.set_many)(self.collection, uid, key_value_dict)

    def get(self, uid="", key=None, pattern=None, exp=None):
        """
        Get an object from the collection (possibly by pattern), or an entry (or entries) from an object in the collection.
        * key == None and pattern == None:                         return collection[uid]
        * key != None and pattern == None and type(key) != list:   return collection[uid][key]
        * key != None and pattern == None and type(key) == list:   return {k: collection[uid][k] for k in key}
        * pattern != None:                                         return collection[uid] matching pattern
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp is None else exp))
        if key is None and pattern is None:
            return self.timed(self.db.get_doc, get=True)(self.collection, uid)
        elif key:
            if isinstance(key, list):
                return self.timed(self.db.get_many, get=True)(self.collection, uid, key)
            else:
                return self.timed(self.db.get, get=True)(self.collection, uid, key)
        else:
            return self.timed(self.db.get_docs_with_filter, get=True)(self.collection, pattern)

    def get_and_delete(self, uid="", key=None, exp=None):
        """
        Get a value from the collection corresponding to the key and then delete the (key,value).
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp is None else exp))
        value = self.timed(self.db.get_and_delete, get=True)(self.collection, uid, key)
        return value

    def exists(self, uid="", key='_id', exp=None):
        """
        Check if an object with the specified uid exists
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp is None else exp))
        return self.timed(self.db.exists, get=True)(self.collection, uid, key)

    def increment(self, uid="", key=None, exp=None, value=1):
        """
        Increment a value (or values) in the collection.
        * key = str:   increment collection[uid][key]

        * value: How much the value should be incremented by.
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp is None else exp))
        return self.timed(self.db.increment, get=True)(self.collection, uid, key, value)

    def increment_many(self, uid="", key_value_dict=None, exp=None):
        """
        For each key in key_value_dict, increments value by key_value_dict[key]

        * values: How much the value should be incremented by.
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp is None else exp))
        return self.timed(self.db.increment_many, get=True)(self.collection, uid, key_value_dict)

    def append(self, uid="", key=None, value=None, exp=None):
        """
        Append a value to collection[uid][key] (which is assumed to be a list)
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp == None else exp))
        self.timed(self.db.append_list)(self.collection, uid, key, value)

    def pop(self, uid="", key=None, value=-1, exp=None):
        """
        Pop a value from collection[uid][key] (which is assumed to be a list)
        value=-1 pops the last element of the list (default)
        value=0 pops the first element of the list
        Other values for "value" will throw error and return a None (not supported in Mongo)
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp == None else exp))
        return self.timed(self.db.pop_list, get=True)(self.collection, uid, key, value)

    def getDurations(self):
        """
        For book keeping purposes only
        """
        return {'duration_dbSet': self.set_durations, 'duration_dbGet': self.get_durations}

    def timed(self, f, get=False):
        if not self.timing:
            return f

        def timed_f(*args, **kw):
            result, dt = utils.timeit(f)(*args, **kw)
            res = None
            if get:
                self.get_durations += dt
                res, didSucceed, message = result
            else:
                self.set_durations += dt
                didSucceed, message = result
            return res

        return timed_f

class Butler(object):
    def __init__(self, app_id, exp_uid, targets, db, ell, alg_label=None, alg_id=None):
        self.app_id = app_id
        self.exp_uid = exp_uid
        self.alg_label = alg_label
        self.alg_id = alg_id
        self.db = db
        self.ell = ell
        self.targets = targets
        self.memory = Memory()

        if self.targets.db is None:
            self.targets.db = self.db
        self.queries = Collection(self.app_id+":queries", "", self.exp_uid, db)
        self.admin = Collection("experiments_admin", "", self.exp_uid, db)
        self.experiment = Collection(self.app_id+":experiments", "{exp_uid}", self.exp_uid, db)
        if alg_label is None:
            self.algorithms = Collection(self.app_id+":algorithms", "{exp_uid}_", self.exp_uid, db)
        else:
            self.algorithms = Collection(self.app_id+":algorithms", "{exp_uid}_"+alg_label, self.exp_uid, db)
        self.participants = Collection(self.app_id+":participants", "", self.exp_uid, db)
        self.dashboard = Collection(self.app_id+":dashboard", "", self.exp_uid, db)
        self.other = Collection(self.app_id+":other", "{exp_uid}_", self.exp_uid, db)

    def log(self, log_name, log_value):
        self.ell.log(self.app_id+":"+log_name, log_value)

    def job(self, task, task_args_json, ignore_result=True, time_limit=0):
        if self.alg_label:
            self.db.submit_job(self.app_id, self.exp_uid,
                               task, task_args_json,
                               self.exp_uid + '_' + self.alg_label,
                               ignore_result, time_limit,
                               alg_id=self.alg_id, alg_label=self.alg_label)
        else:
            self.db.submit_job(self.app_id, self.exp_uid, task, task_args_json, None, ignore_result, time_limit)
<commit_msg>forgot one thing<commit_after>import next.utils as utils
import os
import next.constants as constants
import redis
import StringIO

class Memory(object):
    def __init__(self, collection='', exp_uid='', uid_prefix=''):
        self.key_prefix = collection + uid_prefix.format(exp_uid=exp_uid)
        self.cache = None
        self.max_entry_size = 500000000  # 500MB

    def check_prefix(self):
        if self.key_prefix == '':
            utils.debug_print("butler.memory is deprecated."
                              " Change to butler.experiment.memory or butler.algorithm.memory, etc."
                              " wherever appropriate")

    def ensure_connection(self):
        try:
            if self.cache is None:
                self.cache = redis.StrictRedis(host=constants.MINIONREDIS_HOST, port=constants.MINIONREDIS_PORT)
        except Exception as e:
            raise Exception("Butler.Collection.Memory could not connect with RedisDB: {}".format(e))

    def num_entries(self, size):
        if size % self.max_entry_size == 0:
            return size / self.max_entry_size
        else:
            return (size / self.max_entry_size) + 1

    def set(self, key, value):
        self.check_prefix()
        key = self.key_prefix + key
        try:
            self.ensure_connection()
            l = len(value)
            n = self.num_entries(l)
            utils.debug_print("Setting {} in {} entries".format(l, n))
            for i in range(n):
                k = key + ":" + str(i)
                self.cache.set(k, value[i*self.max_entry_size:(i+1)*self.max_entry_size])
            return self.cache.set(key, "{}:{}".format(str(n), str(l)))
        except Exception as e:
            utils.debug_print("Butler.Collection.Memory.set exception: {}".format(e))
            return False

    def set_file(self, key, f):
        self.check_prefix()
        key = self.key_prefix + key
        try:
            self.ensure_connection()
            f.seek(0, os.SEEK_END)
            l = f.tell()
            f.seek(0, 0)
            n = self.num_entries(l)
            utils.debug_print("Setting {} bytes in {} entries".format(l, n))
            for i in range(n):
                k = key + ":" + str(i)
                v = f.read(self.max_entry_size)
                self.cache.set(k, v)
            return self.cache.set(key, "{}:{}".format(str(n), str(l)))
        except Exception as e:
            utils.debug_print("Butler.Collection.Memory.set_file exception: {}".format(e))
            return False

    def get(self, key):
        self.check_prefix()
        try:
            self.ensure_connection()
            key = self.key_prefix + key
            d = self.cache.get(key)
            n, l = d.split(":")
            l = int(l)
            n = int(n)
            ans = ""
            utils.debug_print("Getting {} bytes in {} entries".format(l, n))
            for i in range(n):
                k = key + ":" + str(i)
                ans += self.cache.get(k)
            return ans
        except Exception as e:
            utils.debug_print("Butler.Collection.Memory.get exception: {}".format(e))
            return None

    def get_file(self, key):
        self.check_prefix()
        try:
            self.ensure_connection()
            key = self.key_prefix + key
            d = self.cache.get(key)
            f = StringIO.StringIO()
            n, l = d.split(":")
            l = int(l)
            n = int(n)
            utils.debug_print("Getting {} bytes in {} entries".format(l, n))
            for i in range(n):
                k = key + ":" + str(i)
                f.write(self.cache.get(k))
            f.seek(0, 0)
            return f
        except Exception as e:
            utils.debug_print("Butler.Collection.Memory.get_file exception: {}".format(e))
            return None

    def lock(self, name, **kwargs):
        try:
            self.ensure_connection()
            name = self.key_prefix + name
            return self.cache.lock(name, **kwargs)
        except Exception as e:
            utils.debug_print("Butler.Collection.Memory.lock exception: {}".format(e))
            return None

    def exists(self, key):
        try:
            self.ensure_connection()
            key = self.key_prefix + key
            return self.cache.exists(key)
        except Exception as e:
            utils.debug_print("Butler.Collection.Memory.exists exception: {}".format(e))
            return None

class Collection(object):
    def __init__(self, collection, uid_prefix, exp_uid, db, timing=True):
        self.collection = collection
        self.db = db
        self.exp_uid = exp_uid
        self.uid_prefix = uid_prefix
        self.get_durations = 0.0
        self.set_durations = 0.0
        self.timing = timing
        self.memory = Memory(collection, exp_uid, uid_prefix)

    def set(self, uid="", key=None, value=None, exp=None):
        """
        Set an object in the collection, or an entry in an object in the collection.
        * key == None:    collection[uid] = value
        * key != None:    collection[uid][key] = value
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp is None else exp))
        if not key:
            self.timed(self.db.set_doc)(self.collection, uid, value)
        else:
            self.timed(self.db.set)(self.collection, uid, key, value)

    def set_many(self, uid="", key_value_dict=None, exp=None):
        """
        For each key in key_value_dict, sets value by key_value_dict[key]
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp is None else exp))
        return self.timed(self.db.set_many)(self.collection, uid, key_value_dict)

    def get(self, uid="", key=None, pattern=None, exp=None):
        """
        Get an object from the collection (possibly by pattern), or an entry (or entries) from an object in the collection.
        * key == None and pattern == None:                         return collection[uid]
        * key != None and pattern == None and type(key) != list:   return collection[uid][key]
        * key != None and pattern == None and type(key) == list:   return {k: collection[uid][k] for k in key}
        * pattern != None:                                         return collection[uid] matching pattern
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp is None else exp))
        if key is None and pattern is None:
            return self.timed(self.db.get_doc, get=True)(self.collection, uid)
        elif key:
            if isinstance(key, list):
                return self.timed(self.db.get_many, get=True)(self.collection, uid, key)
            else:
                return self.timed(self.db.get, get=True)(self.collection, uid, key)
        else:
            return self.timed(self.db.get_docs_with_filter, get=True)(self.collection, pattern)

    def get_and_delete(self, uid="", key=None, exp=None):
        """
        Get a value from the collection corresponding to the key and then delete the (key,value).
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp is None else exp))
        value = self.timed(self.db.get_and_delete, get=True)(self.collection, uid, key)
        return value

    def exists(self, uid="", key='_id', exp=None):
        """
        Check if an object with the specified uid exists
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp is None else exp))
        return self.timed(self.db.exists, get=True)(self.collection, uid, key)

    def increment(self, uid="", key=None, exp=None, value=1):
        """
        Increment a value (or values) in the collection.
        * key = str:   increment collection[uid][key]

        * value: How much the value should be incremented by.
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp is None else exp))
        return self.timed(self.db.increment, get=True)(self.collection, uid, key, value)

    def increment_many(self, uid="", key_value_dict=None, exp=None):
        """
        For each key in key_value_dict, increments value by key_value_dict[key]

        * values: How much the value should be incremented by.
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp is None else exp))
        return self.timed(self.db.increment_many, get=True)(self.collection, uid, key_value_dict)

    def append(self, uid="", key=None, value=None, exp=None):
        """
        Append a value to collection[uid][key] (which is assumed to be a list)
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp == None else exp))
        self.timed(self.db.append_list)(self.collection, uid, key, value)

    def pop(self, uid="", key=None, value=-1, exp=None):
        """
        Pop a value from collection[uid][key] (which is assumed to be a list)
        value=-1 pops the last element of the list (default)
        value=0 pops the first element of the list
        Other values for "value" will throw error and return a None (not supported in Mongo)
        """
        uid = (self.uid_prefix+uid).format(exp_uid=(self.exp_uid if exp == None else exp))
        return self.timed(self.db.pop_list, get=True)(self.collection, uid, key, value)

    def getDurations(self):
        """
        For book keeping purposes only
        """
        return {'duration_dbSet': self.set_durations, 'duration_dbGet': self.get_durations}

    def timed(self, f, get=False):
        if not self.timing:
            return f

        def timed_f(*args, **kw):
            result, dt = utils.timeit(f)(*args, **kw)
            res = None
            if get:
                self.get_durations += dt
                res, didSucceed, message = result
            else:
                self.set_durations += dt
                didSucceed, message = result
            return res

        return timed_f

class Butler(object):
    def __init__(self, app_id, exp_uid, targets, db, ell, alg_label=None, alg_id=None):
        self.app_id = app_id
        self.exp_uid = exp_uid
        self.alg_label = alg_label
        self.alg_id = alg_id
        self.db = db
        self.ell = ell
        self.targets = targets
        self.memory = Memory()

        if self.targets.db is None:
            self.targets.db = self.db
        self.queries = Collection(self.app_id+":queries", "", self.exp_uid, db)
        self.admin = Collection("experiments_admin", "", self.exp_uid, db)
        self.experiment = Collection(self.app_id+":experiments", "{exp_uid}", self.exp_uid, db)
        if alg_label is None:
            self.algorithms = Collection(self.app_id+":algorithms", "{exp_uid}_", self.exp_uid, db)
        else:
            self.algorithms = Collection(self.app_id+":algorithms", "{exp_uid}_"+alg_label, self.exp_uid, db)
        self.participants = Collection(self.app_id+":participants", "", self.exp_uid, db)
        self.dashboard = Collection(self.app_id+":dashboard", "", self.exp_uid, db)
        self.other = Collection(self.app_id+":other", "{exp_uid}_", self.exp_uid, db)

    def log(self, log_name, log_value):
        self.ell.log(self.app_id+":"+log_name, log_value)

    def job(self, task, task_args_json, ignore_result=True, time_limit=0):
        if self.alg_label:
            self.db.submit_job(self.app_id, self.exp_uid,
                               task, task_args_json,
                               self.exp_uid + '_' + self.alg_label,
                               ignore_result, time_limit,
                               alg_id=self.alg_id, alg_label=self.alg_label)
        else:
            self.db.submit_job(self.app_id, self.exp_uid, task, task_args_json, None, ignore_result, time_limit)
<|endoftext|>