<commit_before>import errno
import os
import re
import shutil
import subprocess
import tempfile

def with_all(head, *tail):
    def decorator(fn, *args):
        with head as arg:
            args = args + (arg,)
            if not tail:
                return fn(*args)
            return with_all(*tail)(fn, *args)
    return decorator

class BaseIntegrationServer(object):
    protocols = ('TCP', 'HTTP')

    protocol_re = re.compile(' '.join([
        r'(?P<protocol>[A-Z]+):',
        r'listening on',
        r'(?P<address>(?:[0-9]{1,3}\.){3}[0-9]{1,3}):(?P<port>[0-9]+)',
    ]))

    def __init__(self, address='127.0.0.1'):
        self.address = address
        self.protocol_ports = {}
        self.data_path = tempfile.mkdtemp()

    @property
    def version(self):
        version = os.environ.get('NSQ_VERSION', '0.2.28')
        return tuple([int(v) for v in version.split('.')])

    @property
    def cmd(self):
        raise NotImplementedError('cmd not implemented')

    @property
    def tcp_port(self):
        return self.protocol_ports['TCP']

    @property
    def tcp_address(self):
        return '%s:%d' % (self.address, self.tcp_port)

    @property
    def http_port(self):
        return self.protocol_ports['HTTP']

    @property
    def http_address(self):
        return '%s:%d' % (self.address, self.http_port)

    def _parse_protocol_ports(self):
        while len(self.protocol_ports) < len(self.protocols):
            line = self.child.stderr.readline().decode('utf-8')
            if not line:
                raise Exception('server exited prematurely')

            if 'listening on' not in line:
                continue

            match = self.protocol_re.search(line)
            if not match:
                raise Exception('unexpected line: %r' % line)

            protocol = match.group('protocol')
            if protocol not in self.protocols:
                continue

            port = int(match.group('port'), 10)
            self.protocol_ports[protocol] = port

    def __enter__(self):
        self.child = subprocess.Popen(self.cmd, stderr=subprocess.PIPE)
        self._parse_protocol_ports()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        try:
            self.child.terminate()
        except OSError as error:
            if error.errno == errno.ESRCH:
                return
            raise

        self.child.wait()
        shutil.rmtree(self.data_path)

class NsqdIntegrationServer(BaseIntegrationServer):
    tls_cert = os.path.join(os.path.dirname(__file__), 'cert.pem')
    tls_key = os.path.join(os.path.dirname(__file__), 'key.pem')

    def __init__(self, lookupd=None, **kwargs):
        if self.has_https():
            self.protocols = ('TCP', 'HTTP', 'HTTPS')

        self.lookupd = lookupd
        super(NsqdIntegrationServer, self).__init__(**kwargs)

    def has_https(self):
        return self.version >= (0, 2, 28)

    @property
    def https_port(self):
        return self.protocol_ports['HTTPS']

    @property
    def https_address(self):
        return '%s:%d' % (self.address, self.https_port)

    @property
    def cmd(self):
        cmd = [
            'nsqd',
            '--tcp-address', '%s:0' % self.address,
            '--http-address', '%s:0' % self.address,
            '--data-path', self.data_path,
            '--tls-cert', self.tls_cert,
            '--tls-key', self.tls_key,
        ]

        if self.has_https():
            cmd.extend(['--https-address', '%s:0' % self.address])

        if self.lookupd:
            cmd.extend(['--lookupd-tcp-address', self.lookupd])

        return cmd

class LookupdIntegrationServer(BaseIntegrationServer):
    @property
    def cmd(self):
        return [
            'nsqlookupd',
            '--tcp-address', '%s:0' % self.address,
            '--http-address', '%s:0' % self.address,
        ]
<commit_msg>Pass through integration server output.<commit_after>import errno
import os
import re
import shutil
import subprocess
import sys
import tempfile

def with_all(head, *tail):
    def decorator(fn, *args):
        with head as arg:
            args = args + (arg,)
            if not tail:
                return fn(*args)
            return with_all(*tail)(fn, *args)
    return decorator

class BaseIntegrationServer(object):
    protocols = ('TCP', 'HTTP')

    protocol_re = re.compile(' '.join([
        r'(?P<protocol>[A-Z]+):',
        r'listening on',
        r'(?P<address>(?:[0-9]{1,3}\.){3}[0-9]{1,3}):(?P<port>[0-9]+)',
    ]))

    def __init__(self, address='127.0.0.1'):
        self.address = address
        self.protocol_ports = {}
        self.data_path = tempfile.mkdtemp()

    @property
    def version(self):
        version = os.environ.get('NSQ_VERSION', '0.2.28')
        return tuple([int(v) for v in version.split('.')])

    @property
    def cmd(self):
        raise NotImplementedError('cmd not implemented')

    @property
    def tcp_port(self):
        return self.protocol_ports['TCP']

    @property
    def tcp_address(self):
        return '%s:%d' % (self.address, self.tcp_port)

    @property
    def http_port(self):
        return self.protocol_ports['HTTP']

    @property
    def http_address(self):
        return '%s:%d' % (self.address, self.http_port)

    def _parse_protocol_ports(self):
        while len(self.protocol_ports) < len(self.protocols):
            line = self.child.stderr.readline()
            sys.stderr.write(line)

            if not line:
                raise Exception('server exited prematurely')

            line = line.decode('utf-8')
            if 'listening on' not in line:
                continue

            match = self.protocol_re.search(line)
            if not match:
                raise Exception('unexpected line: %r' % line)

            protocol = match.group('protocol')
            if protocol not in self.protocols:
                continue

            port = int(match.group('port'), 10)
            self.protocol_ports[protocol] = port

    def __enter__(self):
        self.child = subprocess.Popen(self.cmd, stderr=subprocess.PIPE)
        self._parse_protocol_ports()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        try:
            self.child.terminate()
        except OSError as error:
            if error.errno == errno.ESRCH:
                return
            raise

        while True:
            line = self.child.stderr.readline()
            if not line:
                break
            sys.stderr.write(line)

        self.child.wait()
        shutil.rmtree(self.data_path)

class NsqdIntegrationServer(BaseIntegrationServer):
    tls_cert = os.path.join(os.path.dirname(__file__), 'cert.pem')
    tls_key = os.path.join(os.path.dirname(__file__), 'key.pem')

    def __init__(self, lookupd=None, **kwargs):
        if self.has_https():
            self.protocols = ('TCP', 'HTTP', 'HTTPS')

        self.lookupd = lookupd
        super(NsqdIntegrationServer, self).__init__(**kwargs)

    def has_https(self):
        return self.version >= (0, 2, 28)

    @property
    def https_port(self):
        return self.protocol_ports['HTTPS']

    @property
    def https_address(self):
        return '%s:%d' % (self.address, self.https_port)

    @property
    def cmd(self):
        cmd = [
            'nsqd',
            '--tcp-address', '%s:0' % self.address,
            '--http-address', '%s:0' % self.address,
            '--data-path', self.data_path,
            '--tls-cert', self.tls_cert,
            '--tls-key', self.tls_key,
        ]

        if self.has_https():
            cmd.extend(['--https-address', '%s:0' % self.address])

        if self.lookupd:
            cmd.extend(['--lookupd-tcp-address', self.lookupd])

        return cmd

class LookupdIntegrationServer(BaseIntegrationServer):
    @property
    def cmd(self):
        return [
            'nsqlookupd',
            '--tcp-address', '%s:0' % self.address,
            '--http-address', '%s:0' % self.address,
        ]
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Run the webpages_playback automation script."""

import os
import subprocess
import sys

sys.path.insert(0, os.getcwd())

from common.py.utils import gs_utils
from common.py.utils import shell_utils

SKP_VERSION_FILE = 'SKP_VERSION'

def _get_skp_version():
  """Find an unused SKP version."""
  current_skp_version = None
  with open(SKP_VERSION_FILE) as f:
    current_skp_version = int(f.read().rstrip())

  # Find the first SKP version which has no uploaded SKPs.
  new_version = current_skp_version + 1
  while True:
    gs_path = 'playback_%d/skps' % new_version
    if not gs_utils.GSUtils().does_storage_object_exist('chromium-skia-gm',
                                                        gs_path):
      return new_version
    new_version += 1

def main(chrome_src_path, browser_executable):
  browser_executable = os.path.realpath(browser_executable)
  skp_version = _get_skp_version()
  print 'SKP_VERSION=%d' % skp_version

  if os.environ.get('CHROME_HEADLESS'):
    # Start Xvfb if running on a bot.
    try:
      shell_utils.run('sudo Xvfb :0 -screen 0 1280x1024x24 &', shell=True)
    except Exception:
      # It is ok if the above command fails, it just means that DISPLAY=:0
      # is already up.
      pass

  upload_dir = 'playback_%d' % skp_version
  webpages_playback_cmd = [
    'python', os.path.join(os.path.dirname(os.path.realpath(__file__)),
                           'webpages_playback.py'),
    '--page_sets', 'all',
    '--browser_executable', browser_executable,
    '--non-interactive',
    '--upload',
    '--alternate_upload_dir', upload_dir,
    '--chrome_src_path', chrome_src_path,
  ]

  try:
    shell_utils.run(webpages_playback_cmd)
  finally:
    # Clean up any leftover browser instances. This can happen if there are
    # telemetry crashes, processes are not always cleaned up appropriately by
    # the webpagereplay and telemetry frameworks.
    procs = subprocess.check_output(['ps', 'ax'])
    for line in procs.splitlines():
      if browser_executable in line:
        pid = line.strip().split(' ')[0]
        if pid != str(os.getpid()) and not 'python' in line:
          try:
            shell_utils.run(['kill', '-9', pid])
          except shell_utils.CommandFailedException as e:
            print e
        else:
          print 'Refusing to kill self.'

  print 'writing %s: %s' % (SKP_VERSION_FILE, skp_version)
  with open(SKP_VERSION_FILE, 'w') as f:
    f.write(str(skp_version))

if '__main__' == __name__:
  if len(sys.argv) != 3:
    print >> sys.stderr, 'USAGE: %s <chrome src path> <browser executable>'
    sys.exit(1)
  main(*sys.argv[1:])
<commit_msg>Update RecreateSKPs bot to also generate Slimming Paint SKPs with a separate prefix.<commit_after>#!/usr/bin/env python
# Copyright (c) 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Run the webpages_playback automation script."""

import os
import subprocess
import sys

sys.path.insert(0, os.getcwd())

from common.py.utils import gs_utils
from common.py.utils import shell_utils

SKP_VERSION_FILE = 'SKP_VERSION'

def _get_skp_version():
  """Find an unused SKP version."""
  current_skp_version = None
  with open(SKP_VERSION_FILE) as f:
    current_skp_version = int(f.read().rstrip())

  # Find the first SKP version which has no uploaded SKPs.
  new_version = current_skp_version + 1
  while True:
    gs_path = 'playback_%d/skps' % new_version
    if not gs_utils.GSUtils().does_storage_object_exist('chromium-skia-gm',
                                                        gs_path):
      return new_version
    new_version += 1

def main(chrome_src_path, browser_executable):
  browser_executable = os.path.realpath(browser_executable)
  skp_version = _get_skp_version()
  print 'SKP_VERSION=%d' % skp_version

  if os.environ.get('CHROME_HEADLESS'):
    # Start Xvfb if running on a bot.
    try:
      shell_utils.run('sudo Xvfb :0 -screen 0 1280x1024x24 &', shell=True)
    except Exception:
      # It is ok if the above command fails, it just means that DISPLAY=:0
      # is already up.
      pass

  upload_dir = 'playback_%d' % skp_version
  webpages_playback_cmd = [
    'python', os.path.join(os.path.dirname(os.path.realpath(__file__)),
                           'webpages_playback.py'),
    '--page_sets', 'all',
    '--browser_executable', browser_executable,
    '--non-interactive',
    '--upload',
    '--alternate_upload_dir', upload_dir,
    '--chrome_src_path', chrome_src_path,
  ]

  try:
    shell_utils.run(webpages_playback_cmd)
    # Temporary change to enable Slimming Paint runs. See skia:3763.
    webpages_playback_cmd.extend([
        '--skp_prefix', 'sp_',
        '--browser_extra_args', '--enable-slimming-paint',
    ])
    shell_utils.run(webpages_playback_cmd)
  finally:
    # Clean up any leftover browser instances. This can happen if there are
    # telemetry crashes, processes are not always cleaned up appropriately by
    # the webpagereplay and telemetry frameworks.
    procs = subprocess.check_output(['ps', 'ax'])
    for line in procs.splitlines():
      if browser_executable in line:
        pid = line.strip().split(' ')[0]
        if pid != str(os.getpid()) and not 'python' in line:
          try:
            shell_utils.run(['kill', '-9', pid])
          except shell_utils.CommandFailedException as e:
            print e
        else:
          print 'Refusing to kill self.'

  print 'writing %s: %s' % (SKP_VERSION_FILE, skp_version)
  with open(SKP_VERSION_FILE, 'w') as f:
    f.write(str(skp_version))

if '__main__' == __name__:
  if len(sys.argv) != 3:
    print >> sys.stderr, 'USAGE: %s <chrome src path> <browser executable>'
    sys.exit(1)
  main(*sys.argv[1:])
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 03-07-2017 16:50
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This module contains Ensembl models used as Entities and DAO/Services
"""

import config_manager

class Species:
    def __init__(self, ensembl_species_entry):
        self.__ensembl_species_entry = ensembl_species_entry
        self.__logger = config_manager.get_app_config_manager().get_logger_for(__name__)

    def _get_logger(self):
        return self.__logger

    def get_division(self):
        # TODO
        pass

    def get_ncbi_taxonomy_id(self):
        # TODO
        pass

    def get_name(self):
        # TODO
        pass

    def get_ensembl_release(self):
        # TODO
        pass

    def get_display_name(self):
        # TODO
        pass

    def get_assembly_accession(self):
        # TODO
        pass

    def get_strain_collection(self):
        # TODO
        pass

    def get_common_name(self):
        # TODO
        pass

    def get_strain(self):
        # TODO
        pass

    def get_aliases(self):
        # TODO
        pass

    def get_groups(self):
        # TODO
        pass

class SpeciesService:
    # TODO
    pass
<commit_msg>get assembly, placeholder<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 03-07-2017 16:50
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This module contains Ensembl models used as Entities and DAO/Services
"""

import config_manager

class Species:
    def __init__(self, ensembl_species_entry):
        self.__ensembl_species_entry = ensembl_species_entry
        self.__logger = config_manager.get_app_config_manager().get_logger_for(__name__)

    def _get_logger(self):
        return self.__logger

    def get_division(self):
        # TODO
        pass

    def get_ncbi_taxonomy_id(self):
        # TODO
        pass

    def get_name(self):
        # TODO
        pass

    def get_ensembl_release(self):
        # TODO
        pass

    def get_display_name(self):
        # TODO
        pass

    def get_assembly_accession(self):
        # TODO
        pass

    def get_strain_collection(self):
        # TODO
        pass

    def get_common_name(self):
        # TODO
        pass

    def get_strain(self):
        # TODO
        pass

    def get_aliases(self):
        # TODO
        pass

    def get_groups(self):
        # TODO
        pass

    def get_assembly(self):
        # TODO
        pass

class SpeciesService:
    # TODO
    pass
<|endoftext|>


<commit_before>#!/usr/bin/env python
import rospy
import genpy
import pymongo # https://api.mongodb.org/python/2.6.3/
import json
import copy
from bson import json_util
from bson.objectid import ObjectId
from datetime import *
from tms_msg_db.msg import TmsdbStamped, Tmsdb
from tms_msg_db.srv import *
import tms_db_manager.tms_db_util as db_util

client = pymongo.MongoClient('localhost:27017')
db = client.rostmsdb

class TmsDbReader():
    def __init__(self):
        rospy.init_node("tms_db_reader")
        rospy.on_shutdown(self.shutdown)

        db_host = 'localhost'
        db_port = 27017
        self.is_connected = db_util.check_connection(db_host, db_port);
        if not self.is_connected:
            raise Exception("Problem of connection")

        self.collection_list=['data_person','data_robot','data_sensor','data_structure','data_space','data_furniture','data_object']

        self.db_reader_srv = rospy.Service('dbreader', TmsdbGetData, self.dbReaderSrvCallback)

    def dbReaderSrvCallback(self, req):
        rospy.loginfo("receive the TmsdbGetData!")
        temp_dbdata = Tmsdb()

        # req.
        return TmsdbGetDataResponse()
        # temp_dbdata = Tmsdb()
        # current_environment_information = TmsdbStamped()
        #
        # for collection_name in self.collection_list:
        #     cursor = db[collection_name].find({'state':1})
        #     # print(collection_name)
        #     # print(cursor.count())
        #     for doc in cursor:
        #         del doc['_id']
        #         temp_dbdata = db_util.document_to_msg(doc, Tmsdb)
        #         current_environment_information.tmsdb.append(temp_dbdata)
        #
        #     # rospy.loginfo("send db data!")
        #     self.data_pub.publish(current_environment_information)

    def shutdown(self):
        rospy.loginfo("Stopping the node")

if __name__ == '__main__':
    try:
        TmsDbReader()
        rospy.spin()
    except rospy.ROSInterruptException:
        rospy.loginfo("tms_db_reader node terminated.")
<commit_msg>added the Reader Srv Callback<commit_after>#!/usr/bin/env python
import rospy
import genpy
import pymongo # https://api.mongodb.org/python/2.6.3/
import json
import copy
from bson import json_util
from bson.objectid import ObjectId
from datetime import *
from tms_msg_db.msg import TmsdbStamped, Tmsdb
from tms_msg_db.srv import *
import tms_db_manager.tms_db_util as db_util

client = pymongo.MongoClient('localhost:27017')
db = client.rostmsdb

class TmsDbReader():
    def __init__(self):
        rospy.init_node("tms_db_reader")
        rospy.on_shutdown(self.shutdown)

        db_host = 'localhost'
        db_port = 27017
        self.is_connected = db_util.check_connection(db_host, db_port);
        if not self.is_connected:
            raise Exception("Problem of connection")

        self.collection_list=['data_person','data_robot','data_sensor','data_structure','data_space','data_furniture','data_object']

        self.db_reader_srv = rospy.Service('dbreader', TmsdbGetData, self.dbReaderSrvCallback)

    def dbReaderSrvCallback(self, req):
        rospy.loginfo("Received the service call!")
        temp_dbdata = Tmsdb()
        result = False

        try:
            if req.tmsdb.name != "":
                cursor = db['default_data'].find({'name':req.tmsdb.name})
                if cursor[0]['type'] != '':
                    collection_name = "data_" + cursor[0]['type']
                    # print(collection_name)
                    cursor = db[collection_name].find({'name':req.tmsdb.name})
                    for doc in cursor:
                        del doc['_id']
                        temp_dbdata = db_util.document_to_msg(doc, Tmsdb)
                        # print(doc)
                    result = True
                else:
                    result = False
            else:
                result = False
        except:
            result = False

        ret = TmsdbGetDataResponse()

        if result == False:
            temp_dbdata.note = "Wrong request! Try to check the target type and name"

        ret.tmsdb.append(temp_dbdata)
        return ret

    def shutdown(self):
        rospy.loginfo("Stopping the node")

if __name__ == '__main__':
    try:
        TmsDbReader()
        rospy.spin()
    except rospy.ROSInterruptException:
        rospy.loginfo("tms_db_reader node terminated.")
<|endoftext|>


<commit_before>import random
from typing import Optional, Callable, Any
from functools import wraps

from fabric.api import quiet, puts, task, abort, run, settings
from fabric.colors import green as g, red as r, yellow as y

from fabric_utils.healthcheck import check_role_is_up

@task
def docker_swarm_ping_manager() -> Optional[str]:
    # a node may fail but this is perfectly fine
    # perhaps it's under a maintenance?
    with quiet(), settings(abort_exception=Exception, abort_on_prompts=True):
        try:
            return run('docker node ls')
        except Exception:
            return None

def docker_swarm_select_manager(role: str) -> Optional[str]:
    checked_hosts, stderr = check_role_is_up(role, docker_swarm_ping_manager)
    if any(checked_hosts.values()):
        good_hosts = [host for host, status in checked_hosts.items() if status]
        puts(g(f'swarm is healthy. {len(good_hosts)}/{len(checked_hosts)} available managers: {", ".join(good_hosts)}'))
        return random.choice(good_hosts)
    else:
        failed_hosts = ', '.join([host for host, status in checked_hosts.items() if not status])
        abort(r(f'swarm is not healthy. all managers failed: {failed_hosts}'))
        return None

@task
def docker_swarm_restart(label: str, value: str, stack: str,
                         no_serial: bool = False, no_wait: bool = False) -> None:
    """
    Restart swarm services by their deploy labels defined in the compose file.

    :param label: service label defined in the compose file (e.g. mybook.deploy.group)
    :param value: service label value (e.g. uwsgi)
    :param stack: name of the stack where the labeled service is defined
    :param no_serial: Execute the update command on all nodes at once ignoring the parallelism mode
    :param no_wait: Do not wait for currently services to exit gracefully
    """
    service_format = '{{.Name}}'
    service_names = run(f'docker stack services '
                        f'--format "{service_format}" --filter label={label}={value} '
                        f'{stack}')
    if 'Nothing found' in service_names:
        abort(r(f'no services found matching label "{label}={value}"'))
    else:
        command = 'docker service update --quiet --force --no-healthcheck'
        if no_serial:
            command = f'{command} --update-parallelism=0'
        if no_wait:
            command = f'{command} --stop-grace-period=1s'
        for service_name in service_names.splitlines():
            puts(y(f'restarting service {service_name}'))
            run(f'{command} {service_name}')

def with_swarm_node(role: str) -> Callable:
    """
    Pick a random swarm node and pass it as a keyword arg to the decorated function
    """
    def decorator(func: Callable) -> Callable:
        @wraps(func)
        def wrapper(*task_args: Any, **task_kwargs: Any) -> Any:
            swarm_node = docker_swarm_select_manager(role)
            task_kwargs['node'] = swarm_node
            return func(*task_args, **task_kwargs)
        return wrapper
    return decorator
<commit_msg>swarm: fix args order for check_role_is_up #9 (#9)<commit_after>import random
from typing import Optional, Callable, Any
from functools import wraps

from fabric.api import quiet, puts, task, abort, run, settings
from fabric.colors import green as g, red as r, yellow as y

from fabric_utils.healthcheck import check_role_is_up

@task
def docker_swarm_ping_manager() -> Optional[str]:
    # a node may fail but this is perfectly fine
    # perhaps it's under a maintenance?
    with quiet(), settings(abort_exception=Exception, abort_on_prompts=True):
        try:
            return run('docker node ls')
        except Exception:
            return None

def docker_swarm_select_manager(role: str) -> Optional[str]:
    checked_hosts, stderr = check_role_is_up(docker_swarm_ping_manager, role=role)
    if any(checked_hosts.values()):
        good_hosts = [host for host, status in checked_hosts.items() if status]
        puts(g(f'swarm is healthy. {len(good_hosts)}/{len(checked_hosts)} available managers: {", ".join(good_hosts)}'))
        return random.choice(good_hosts)
    else:
        failed_hosts = ', '.join([host for host, status in checked_hosts.items() if not status])
        abort(r(f'swarm is not healthy. all managers failed: {failed_hosts}'))
        return None

@task
def docker_swarm_restart(label: str, value: str, stack: str,
                         no_serial: bool = False, no_wait: bool = False) -> None:
    """
    Restart swarm services by their deploy labels defined in the compose file.

    :param label: service label defined in the compose file (e.g. mybook.deploy.group)
    :param value: service label value (e.g. uwsgi)
    :param stack: name of the stack where the labeled service is defined
    :param no_serial: Execute the update command on all nodes at once ignoring the parallelism mode
    :param no_wait: Do not wait for currently services to exit gracefully
    """
    service_format = '{{.Name}}'
    service_names = run(f'docker stack services '
                        f'--format "{service_format}" --filter label={label}={value} '
                        f'{stack}')
    if 'Nothing found' in service_names:
        abort(r(f'no services found matching label "{label}={value}"'))
    else:
        command = 'docker service update --quiet --force --no-healthcheck'
        if no_serial:
            command = f'{command} --update-parallelism=0'
        if no_wait:
            command = f'{command} --stop-grace-period=1s'
        for service_name in service_names.splitlines():
            puts(y(f'restarting service {service_name}'))
            run(f'{command} {service_name}')

def with_swarm_node(role: str) -> Callable:
    """
    Pick a random swarm node and pass it as a keyword arg to the decorated function
    """
    def decorator(func: Callable) -> Callable:
        @wraps(func)
        def wrapper(*task_args: Any, **task_kwargs: Any) -> Any:
            swarm_node = docker_swarm_select_manager(role)
            task_kwargs['node'] = swarm_node
            return func(*task_args, **task_kwargs)
        return wrapper
    return decorator
<|endoftext|>


<commit_before>'''
This is the benchmark of 4 different implementations of rectified linear
activation in Theano.
Two types of computations are tested w.r.t. each implementation: fprop and grad

Results: in seconds, float32 (details in the code)

Implementations tested, CPU (fprop, bprop), GPU (fprop, bprop), (final score)
a) ScalarRectifier:       (2.32, 2.40)    (1.36, 2.67)    (8.75)
b) T.max(.0, x):          (5.19, 3.65)    (1.38, 2.38)    (12.60)
c) x*(x>0.):              (2.85, 2.84)    (1.31, 2.91)    (9.91)
d) T.switch(x<0., 0., x): (2.32, 1.41)    (1.41, 2.84)    (8.39)

Conlusion:
In terms of efficiency, d) > a) > c) > b)

Written by Li and Fred.

'''
import theano
import theano.tensor as T
from theano.tensor import elemwise

import numpy
import time

floatX = 'float32'
relu = lambda x: T.maximum(0.0, x)
relu_ = lambda x: x * (x > 0)
relu__ = lambda x: T.switch(x < 0., 0., x)

class ScalarRectifier(scalar.UnaryScalarOp):
    """
    .. todo::

        WRITEME
    """
    @staticmethod
    def st_impl(x):
        """
        .. todo::

            WRITEME
        """
        return x * (x > 0.0)

    def impl(self, x):
        """
        .. todo::

            WRITEME
        """
        return ScalarRectifier.st_impl(x)

    def grad(self, (x,), (gz,)):
        """
        .. todo::

            WRITEME
        """
        return [(x > 0.0) * gz]

    def c_code(self, node, name, (x,), (z,), sub):
        """
        .. todo::

            WRITEME
        """
        if node.inputs[0].type == scalar.float32:
            return """%(z)s = %(x)s < 0.0f ? 0.0 : %(x)s;""" % locals()
        elif node.inputs[0].type == scalar.float64:
            return """%(z)s = %(x)s < 0.0 ? 0.0 : %(x)s;""" % locals()
        else:
            raise NotImplementedError('only floatingpoint is implemented')

    def c_code_cache_version(self):
        """
        .. todo::

            WRITEME
        """
        v = super(ScalarRectifier, self).c_code_cache_version()
        if v:
            return (2,) + v
        else:
            return v

scalar_rectifier = ScalarRectifier(scalar.upgrade_to_float,
                                   name='scalar_rectifier')
rectifier = elemwise.Elemwise(scalar_rectifier, name='rectifier')

def test_scalar_rectifier():
    # verify the new op rectifier produces the same results as relu
    x = T.fmatrix('inputs')
    y1 = relu(x)
    y2 = rectifier(x)
    y3 = relu_(x)
    y4 = relu__(x)

    f1 = theano.function(inputs=[x], outputs=y1, name='benchmark_1_forward')
    f2 = theano.function(inputs=[x], outputs=y2, name='benchmark_2_forward')
    f3 = theano.function(inputs=[x], outputs=y3, name='benchmark_3_forward')
    f4 = theano.function(inputs=[x], outputs=y4, name='benchmark_4_forward')

    g1 = theano.function(inputs=[x], outputs=T.grad(y1.sum(),x), name='benchmark_1_grad')
    g2 = theano.function(inputs=[x], outputs=T.grad(y2.sum(),x), name='benchmark_2_grad')
    g3 = theano.function(inputs=[x], outputs=T.grad(y3.sum(),x), name='benchmark_3_grad')
    g4 = theano.function(inputs=[x], outputs=T.grad(y4.sum(),x), name='benchmark_4_grad')

    for i in range(10):
        value = numpy.random.uniform(-1,1,size=(100,500)).astype(floatX)
        numpy.testing.assert_array_equal(f1(value), f2(value),
                                         err_msg='arrays not equal' )

        numpy.testing.assert_array_equal(f1(value), f3(value),
                                         err_msg='arrays not equal' )

        numpy.testing.assert_array_equal(f1(value), f4(value),
                                         err_msg='arrays not equal' )

        numpy.testing.assert_array_equal(g1(value), g2(value),
                                         err_msg='grad:arrays not equal' )

        numpy.testing.assert_array_equal(g1(value), g3(value),
                                         err_msg='grad:arrays not equal' )

        numpy.testing.assert_array_equal(g1(value), g4(value),
                                         err_msg='grad:arrays not equal' )

def benchmark_single_op():
    x = T.ftensor4('inputs')

    ops = [
        rectifier(x).sum(), # new
        relu_(x).sum(), # old
        relu(x).sum(), # alter, short for alternative
        relu__(x).sum(), # alter 2
        T.grad(rectifier(x).sum(),x), # grad_new
        T.grad(relu_(x).sum(),x), # grad_old
        T.grad(relu(x).sum(),x), # grad_alter
        T.grad(relu__(x).sum(),x), # grad_alter2
    ]

    names = ['fprop_new', 'fprop_old', 'fprop_alter', 'fprop_alter2',
             'grad_new', 'grad_old', 'grad_alter', 'grad_alter2']

    value = numpy.random.uniform(size=(512,32,32,100)).astype(floatX)
    times = []
    for op, name in zip(ops, names):
        f = theano.function(inputs=[x], outputs=op, name=name)
        n_loops = 10

        t0 = time.time()
        for i in range(n_loops):
            f(value)
        t1 = time.time()
        benchmark = t1-t0
        times.append(benchmark)
        print name
        theano.printing.debugprint(f, print_type=True)
    print names
    print times

def benchmark_all():
    benchmark_single_op()

if __name__ == '__main__':
    benchmark_all()
    #test_scalar_rectifier()<commit_msg>better<commit_after>'''
This is the benchmark of 4 different implementations of rectified linear
activation in Theano.
Two types of computations are tested w.r.t. each implementation: fprop and grad

Results: in seconds, float32 (details in the code)

Implementations tested, CPU (fprop, bprop), GPU (fprop, bprop), (final score)
a) ScalarRectifier:       (2.32, 2.40)    (1.36, 2.67)    (8.75)
b) T.max(.0, x):          (5.19, 3.65)    (1.38, 2.38)    (12.60)
c) x*(x>0.):              (2.85, 2.84)    (1.31, 2.91)    (9.91)
d) T.switch(x<0., 0., x): (2.32, 1.41)    (1.41, 2.84)    (8.39)

Conlusion:
In terms of efficiency, d) > a) > c) > b)

Written by Li and Fred.

'''
import theano
import theano.tensor as T
from theano.tensor import elemwise

import numpy
import time

floatX = 'float32'
relu = lambda x: T.maximum(0.0, x)
relu_ = lambda x: x * (x > 0)
relu__ = lambda x: T.switch(x < 0., 0., x)

def test_scalar_rectifier():
    # verify the new op rectifier produces the same results as relu
    x = T.fmatrix('inputs')
    y1 = relu(x)
    y3 = relu_(x)
    y4 = relu__(x)

    f1 = theano.function(inputs=[x], outputs=y1, name='benchmark_1_forward')
    f3 = theano.function(inputs=[x], outputs=y3, name='benchmark_3_forward')
    f4 = theano.function(inputs=[x], outputs=y4, name='benchmark_4_forward')

    g1 = theano.function(inputs=[x], outputs=T.grad(y1.sum(),x), name='benchmark_1_grad')
    g3 = theano.function(inputs=[x], outputs=T.grad(y3.sum(),x), name='benchmark_3_grad')
    g4 = theano.function(inputs=[x], outputs=T.grad(y4.sum(),x), name='benchmark_4_grad')

    for i in range(10):
        value = numpy.random.uniform(-1,1,size=(100,500)).astype(floatX)

        numpy.testing.assert_array_equal(f1(value), f3(value),
                                         err_msg='arrays not equal' )

        numpy.testing.assert_array_equal(f1(value), f4(value),
                                         err_msg='arrays not equal' )

        numpy.testing.assert_array_equal(g1(value), g3(value),
                                         err_msg='grad:arrays not equal' )

        numpy.testing.assert_array_equal(g1(value), g4(value),
                                         err_msg='grad:arrays not equal' )

def benchmark_single_op():
    x = T.ftensor4('inputs')

    ops = [
        relu_(x).sum(), # old
        relu(x).sum(), # alter, short for alternative
        relu__(x).sum(), # alter 2
        T.grad(relu_(x).sum(),x), # grad_old
        T.grad(relu(x).sum(),x), # grad_alter
        T.grad(relu__(x).sum(),x), # grad_alter2
    ]

    names = ['fprop_old', 'fprop_alter', 'fprop_alter2',
             'grad_old', 'grad_alter', 'grad_alter2']

    value = numpy.random.uniform(size=(512,32,32,100)).astype(floatX)
    times = []
    for op, name in zip(ops, names):
        f = theano.function(inputs=[x], outputs=op, name=name)
        n_loops = 10

        t0 = time.time()
        for i in range(n_loops):
            f(value)
        t1 = time.time()
        benchmark = t1-t0
        times.append(benchmark)
        print name
        theano.printing.debugprint(f, print_type=True)
    print names
    print times

def benchmark_all():
    benchmark_single_op()

if __name__ == '__main__':
    benchmark_all()
    #test_scalar_rectifier()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
import sys
import ssl
import signal
import logging
import logging.config
from . import utils
from . import config
from .compat import asyncio
from .compat import reload_module
from collections import defaultdict

try:
    import pkg_resources
    from pkg_resources import iter_entry_points
    HAS_PKG_RESOURCES = True
except ImportError:  # pragma: no cover
    HAS_PKG_RESOURCES = False
    version = ''
else:
    try:
        version = pkg_resources.get_distribution('irc3').version
    except pkg_resources.DistributionNotFound:
        version = ''

class Registry:
    """Store (and hide from api) plugins events and stuff"""

    def __init__(self):
        self.reset(reloading=False)

    def reset(self, reloading=True):
        self.events_re = {
            'in': [], 'out': [],
            'dcc_in': [], 'dcc_out': [],
        }
        self.events = {
            'in': defaultdict(list),
            'out': defaultdict(list),
            'dcc_in': defaultdict(list),
            'dcc_out': defaultdict(list),
        }

        self.scanned = []
        self.includes = set()

        if reloading:
            self.reloading = self.plugins.copy()
        else:
            self.reloading = {}
            self.plugins = {}

    def get_event_matches(self, data, iotype='in'):
        events = self.events[iotype]
        for regexp, cregexp in self.events_re[iotype]:
            match = cregexp(data)
            if match is not None:
                yield match, events[regexp]

class IrcObject:

    nick = None
    server = False
    plugin_category = '__irc3_plugin__'
    logging_config = config.LOGGING

    defaults = dict(
        port=6667,
        timeout=320,
        max_lag=60,
        asynchronous=True,
        max_length=512,
        testing=False,
        ssl=False,
        ssl_verify=False,
        encoding='utf8',
        loop=None,
    )

    def __init__(self, *ini, **config):
        config['version'] = version
        self.config = utils.Config(dict(self.defaults, *ini, **config))
        logging.config.dictConfig(self.logging_config)
        if self.server:
            self.log = logging.getLogger('irc3d')
        else:
            self.log = logging.getLogger('irc3.' + (self.nick or 'd'))
        self.original_nick = self.nick
        if config.get('verbose') or config.get('debug'):
            logging.getLogger('irc3').setLevel(logging.DEBUG)
            logging.getLogger('irc3d').setLevel(logging.DEBUG)
        else:
            level = config.get('level')
            if level is not None:
                level = getattr(logging, str(level), level)
                self.log.setLevel(level)
        self.encoding = self.config['encoding']

        self.loop = self.config.loop
        if self.loop is None:
            try:
                self.loop = asyncio.get_event_loop()
            except RuntimeError:
                self.loop = asyncio.new_event_loop()
                asyncio.set_event_loop(self.loop)

        # python 3.4.1 do not have a create_task method. check for it
        self.create_task = getattr(self.loop, 'create_task', self.create_task)

        self.registry = Registry()

        self.include(*self.config.get('includes', []))

    def create_task(self, coro):  # pragma: no cover
        # python 3.4.1 fallback
        return asyncio.ensure_future(coro, loop=self.loop)

    def get_plugin(self, ob):
        plugins = self.registry.plugins
        includes = self.registry.includes
        reloading = self.registry.reloading

        if isinstance(ob, str):
            ob_name = ob
            ob = utils.maybedotted(ob_name)
            if ob_name not in plugins:
                names = list(plugins)
                raise LookupError(
                    'Plugin %s not found in %s' % (ob_name, names))
        else:
            ob_name = ob.__module__ + '.' + ob.__name__
        if ob_name not in plugins:
            self.log.debug("Register plugin '%s'", ob_name)
            for dotted in getattr(ob, 'requires', []):
                if dotted not in includes:
                    self.include(dotted)
            plugins[ob_name] = ob(self)
        elif ob_name in reloading and hasattr(ob, 'reload'):
            instance = reloading.pop(ob_name)
            if instance.__class__ is not ob:
                self.log.debug("Reloading plugin '%s'", ob_name)
                plugins[ob_name] = ob.reload(instance)
        return plugins[ob_name]

    def recompile(self):
        events_re = self.registry.events_re
        for iotype in ('in', 'out'):
            events = self.registry.events[iotype]
            for i, (regexp, cregexp) in enumerate(events_re[iotype]):
                e = events[regexp][0]
                events_re[i] = (regexp, e.compile(self.config))

    def attach_events(self, *events, **kwargs):
        """Attach one or more events to the bot instance"""
        reg = self.registry
        insert = 'insert' in kwargs
        for e in events:
            cregexp = e.compile(self.config)
            regexp = getattr(e.regexp, 're', e.regexp)
            if regexp not in reg.events[e.iotype]:
                if insert:
                    reg.events_re[e.iotype].insert(0, (regexp, cregexp))
                else:
                    reg.events_re[e.iotype].append((regexp, cregexp))
            if insert:
                reg.events[e.iotype][regexp].insert(0, e)
            else:
                reg.events[e.iotype][regexp].append(e)

    def detach_events(self, *events):
        """Detach one or more events from the bot instance"""
        reg = self.registry
        delete = defaultdict(list)

        # remove from self.events
        all_events = reg.events
        for e in events:
            regexp = getattr(e.regexp, 're', e.regexp)
            iotype = e.iotype
            if e in all_events[iotype].get(regexp, []):
                all_events[iotype][regexp].remove(e)
                if not all_events[iotype][regexp]:
                    del all_events[iotype][regexp]
                    # need to delete from self.events_re
                    delete[iotype].append(regexp)

        # delete from events_re
        for iotype, regexps in delete.items():
            reg.events_re[iotype] = [r for r in reg.events_re[iotype]
                                     if r[0] not in regexps]

    def include(self, *modules, **kwargs):
        reg = self.registry
        categories = kwargs.get('venusian_categories',
                                self.venusian_categories)
        scanner = self.venusian.Scanner(context=self)
        for module in modules:
            if module in reg.includes:
                self.log.warn('%s included twice', module)
            else:
                reg.includes.add(module)
                try:
                    module = utils.maybedotted(module)
                except LookupError as exc:
                    if HAS_PKG_RESOURCES:
                        entry_points = iter_entry_points(
                            'irc3.loader',
                            module
                        )
                        try:
                            module = next(entry_points).load()
                        except StopIteration:
                            raise exc
                    else:
                        raise exc
                # we have to manualy check for plugins. venusian no longer
                # support to attach both a class and methods
                for klass in list(module.__dict__.values()):
                    if not isinstance(klass, type):
                        continue
                    if klass.__module__ == module.__name__:
                        if getattr(klass, self.plugin_category, False) is True:
                            self.get_plugin(klass)
                reg.scanned.append((module.__name__, categories))
                scanner.scan(module, categories=categories)

    def reload(self, *modules):
        """Reload one or more plugins"""
        self.notify('before_reload')

        if 'configfiles' in self.config:
            # reload configfiles
            self.log.info('Reloading configuration...')
            cfg = utils.parse_config(
                self.server and 'server' or 'bot', *self.config['configfiles'])
            self.config.update(cfg)

        self.log.info('Reloading python code...')
        if not modules:
            modules = self.registry.includes
        scanned = list(reversed(self.registry.scanned))

        # reset includes and events
        self.registry.reset()

        to_scan = []
        for module_name, categories in scanned:
            if module_name in modules:
                module = utils.maybedotted(module_name)
                reload_module(module)
            to_scan.append((module_name, categories))

        # rescan all modules
        for module_name, categories in to_scan:
            self.include(module_name, venusian_categories=categories)

        self.registry.reloading = {}

        self.notify('after_reload')

    def notify(self, event, exc=None, client=None):
        for p in self.registry.plugins.values():
            meth = getattr(p, event, None)
            if meth is not None:
                if client is not None:
                    meth(client=client)
                else:
                    meth()

    def dispatch(self, data, iotype='in', client=None):
        str = utils.IrcString
        create_task = self.create_task
        call_soon = self.loop.call_soon
        for match, events in self.registry.get_event_matches(data, iotype):
            match = match.groupdict()
            for key, value in match.items():
                if value is not None:
                    match[key] = str(value)
            # backwards compatibility fix for IRCv3.2 tag support:
            # If no tags (None-value), exclude from dictionary
            if match.get("tags", True) is None:
                del match["tags"]
            if client is not None:
                # server / dcc chat
                match['client'] = client
            for e in events:
                if e.iscoroutine is True:
                    create_task(e.callback(**match))
                else:
                    call_soon(e.async_callback, match)

    def call_many(self, callback, args):
        """callback is run with each arg but run a call per second"""
        if isinstance(callback, str):
            callback = getattr(self, callback)
        f = None
        for arg in args:
            f = callback(*arg)
        return f

    def get_ssl_context(self):
        if self.config.ssl:  # pragma: no cover
            try:
                create_default_context = ssl.create_default_context
            except AttributeError:  # py < 2.7.9
                return True
            else:
                if self.server:
                    context = create_default_context(ssl.Purpose.CLIENT_AUTH)
                else:
                    context = create_default_context(ssl.Purpose.SERVER_AUTH)
                verify_mode = self.config.ssl_verify
                if verify_mode is not False:
                    if not isinstance(verify_mode, int):
                        # CERT_NONE / CERT_OPTIONAL / CERT_REQUIRED
                        verify_mode = getattr(ssl, verify_mode.upper())
                    if verify_mode == ssl.CERT_NONE:
                        context.check_hostname = False
                    context.verify_mode = verify_mode
                return context
        return None

    def create_connection(self):
        protocol = utils.maybedotted(self.config.connection)
        protocol = type(protocol.__name__, (protocol,), {'factory': self})
        if self.server:  # pragma: no cover
            self.log.debug('Starting {servername}...'.format(**self.config))
            factory = self.loop.create_server
        else:
            self.log.debug('Starting {nick}...'.format(**self.config))
            factory = self.loop.create_connection
        if self.config.get('sock_factory'):
            sock_factory = utils.maybedotted(self.config.sock_factory)
            args = dict(
                sock=sock_factory(self, self.config.host, self.config.port),
                ssl=self.get_ssl_context(),
            )
            if args.get('ssl'):
                args["server_hostname"] = self.config.host
        else:
            args = dict(
                host=self.config.host,
                port=self.config.port,
                ssl=self.get_ssl_context()
            )
            if self.config.get('vhost'):
                args["local_addr"] = (self.config.vhost, 0)
        t = asyncio.Task(factory(protocol, **args), loop=self.loop)
        t.add_done_callback(self.connection_made)
        return self.loop

    def add_signal_handlers(self):
        """Register handlers for UNIX signals (SIGHUP/SIGINT)"""
        try:
            self.loop.add_signal_handler(signal.SIGHUP, self.SIGHUP)
        except (RuntimeError, AttributeError):  # pragma: no cover
            # windows
            pass
        try:
            self.loop.add_signal_handler(signal.SIGINT, self.SIGINT)
        except (RuntimeError, NotImplementedError):  # pragma: no cover
            # annaconda
            pass

    def run(self, forever=True):
        """start the bot"""
        loop = self.create_connection()
        self.add_signal_handlers()
        if forever:
            loop.run_forever()

    @classmethod
    def from_config(cls, cfg, **kwargs):
        """return an instance configured with the ``cfg`` dict"""
        cfg = dict(cfg, **kwargs)
        pythonpath = cfg.get('pythonpath', [])
        if 'here' in cfg:
            pythonpath.append(cfg['here'])
        for path in pythonpath:
            sys.path.append(os.path.expanduser(path))
        prog = cls.server and 'irc3d' or 'irc3'
        if cfg.get('debug'):
            cls.venusian_categories.append(prog + '.debug')
        if cfg.get('interactive'):  # pragma: no cover
            import irc3.testing
            context = getattr(irc3.testing, cls.__name__)(**cfg)
        else:
            context = cls(**cfg)
        if cfg.get('raw'):
            context.include('irc3.plugins.log',
                            venusian_categories=[prog + '.debug'])
        return context

    @classmethod
    def from_argv(cls, argv=None, **kwargs):
        prog = cls.server and 'irc3d' or 'irc3'
        doc = """
        Run an {prog} instance from a config file

        Usage: {prog} [options] <config>...

        Options:

        -h, --help          Display this help and exit
        --version           Output version information and exit
        --logdir DIRECTORY  Log directory to use instead of stderr
        --logdate           Show datetimes in console output
        --host HOST         Server name or ip
        --port PORT         Server port
        -v,--verbose        Increase verbosity
        -r,--raw            Show raw irc log on the console
        -d,--debug          Add some debug commands/utils
        -i,--interactive    Load a ipython console with a bot instance
        """.format(prog=prog)
        if not cls.server:
            doc += """
            --help-page         Output a reST page containing commands help
            """.strip()
        import os
        import docopt
        import textwrap
        args = argv or sys.argv[1:]
        args = docopt.docopt(textwrap.dedent(doc), args, version=version)
        cfg = utils.parse_config(
            cls.server and 'server' or 'bot', *args['<config>'])
        cfg.update(
            verbose=args['--verbose'],
            debug=args['--debug'],
        )
        cfg.update(kwargs)
        if args['--host']:  # pragma: no cover
            host = args['--host']
            cfg['host'] = host
            if host in ('127.0.0.1', 'localhost'):
                cfg['ssl'] = False
        if args['--port']:  # pragma: no cover
            cfg['port'] = args['--port']
        if args['--logdir'] or 'logdir' in cfg:
            logdir = os.path.expanduser(args['--logdir'] or cfg.get('logdir'))
            cls.logging_config = config.get_file_config(logdir)
        if args['--logdate']:  # pragma: no cover
            fmt = cls.logging_config['formatters']['console']
            fmt['format'] = config.TIMESTAMPED_FMT
        if args.get('--help-page'):  # pragma: no cover
            for v in cls.logging_config['handlers'].values():
                v['level'] = 'ERROR'
        if args['--raw']:
            cfg['raw'] = True
        context = cls.from_config(cfg)
        if args.get('--help-page'):  # pragma: no cover
            context.print_help_page()
        elif args['--interactive']:  # pragma: no cover
            import IPython
            IPython.embed()
            sys.exit(0)
        else:
            context.run(forever=not bool(kwargs))
        if kwargs or argv:
            return context
<commit_msg>logging.warn is deprecated.<commit_after># -*- coding: utf-8 -*-
import os
import sys
import ssl
import signal
import logging
import logging.config
from . import utils
from . import config
from .compat import asyncio
from .compat import reload_module
from collections import defaultdict

try:
    import pkg_resources
    from pkg_resources import iter_entry_points
    HAS_PKG_RESOURCES = True
except ImportError:  # pragma: no cover
    HAS_PKG_RESOURCES = False
    version = ''
else:
    try:
        version = pkg_resources.get_distribution('irc3').version
    except pkg_resources.DistributionNotFound:
        version = ''

class Registry:
    """Store (and hide from api) plugins events and stuff"""

    def __init__(self):
        self.reset(reloading=False)

    def reset(self, reloading=True):
        self.events_re = {
            'in': [], 'out': [],
            'dcc_in': [], 'dcc_out': [],
        }
        self.events = {
            'in': defaultdict(list),
            'out': defaultdict(list),
            'dcc_in': defaultdict(list),
            'dcc_out': defaultdict(list),
        }

        self.scanned = []
        self.includes = set()

        if reloading:
            self.reloading = self.plugins.copy()
        else:
            self.reloading = {}
            self.plugins = {}

    def get_event_matches(self, data, iotype='in'):
        events = self.events[iotype]
        for regexp, cregexp in self.events_re[iotype]:
            match = cregexp(data)
            if match is not None:
                yield match, events[regexp]

class IrcObject:

    nick = None
    server = False
    plugin_category = '__irc3_plugin__'
    logging_config = config.LOGGING

    defaults = dict(
        port=6667,
        timeout=320,
        max_lag=60,
        asynchronous=True,
        max_length=512,
        testing=False,
        ssl=False,
        ssl_verify=False,
        encoding='utf8',
        loop=None,
    )

    def __init__(self, *ini, **config):
        config['version'] = version
        self.config = utils.Config(dict(self.defaults, *ini, **config))
        logging.config.dictConfig(self.logging_config)
        if self.server:
            self.log = logging.getLogger('irc3d')
        else:
            self.log = logging.getLogger('irc3.' + (self.nick or 'd'))
        self.original_nick = self.nick
        if config.get('verbose') or config.get('debug'):
            logging.getLogger('irc3').setLevel(logging.DEBUG)
            logging.getLogger('irc3d').setLevel(logging.DEBUG)
        else:
            level = config.get('level')
            if level is not None:
                level = getattr(logging, str(level), level)
                self.log.setLevel(level)
        self.encoding = self.config['encoding']

        self.loop = self.config.loop
        if self.loop is None:
            try:
                self.loop = asyncio.get_event_loop()
            except RuntimeError:
                self.loop = asyncio.new_event_loop()
                asyncio.set_event_loop(self.loop)

        # python 3.4.1 do not have a create_task method. check for it
        self.create_task = getattr(self.loop, 'create_task', self.create_task)

        self.registry = Registry()

        self.include(*self.config.get('includes', []))

    def create_task(self, coro):  # pragma: no cover
        # python 3.4.1 fallback
        return asyncio.ensure_future(coro, loop=self.loop)

    def get_plugin(self, ob):
        plugins = self.registry.plugins
        includes = self.registry.includes
        reloading = self.registry.reloading

        if isinstance(ob, str):
            ob_name = ob
            ob = utils.maybedotted(ob_name)
            if ob_name not in plugins:
                names = list(plugins)
                raise LookupError(
                    'Plugin %s not found in %s' % (ob_name, names))
        else:
            ob_name = ob.__module__ + '.' + ob.__name__
        if ob_name not in plugins:
            self.log.debug("Register plugin '%s'", ob_name)
            for dotted in getattr(ob, 'requires', []):
                if dotted not in includes:
                    self.include(dotted)
            plugins[ob_name] = ob(self)
        elif ob_name in reloading and hasattr(ob, 'reload'):
            instance = reloading.pop(ob_name)
            if instance.__class__ is not ob:
                self.log.debug("Reloading plugin '%s'", ob_name)
                plugins[ob_name] = ob.reload(instance)
        return plugins[ob_name]

    def recompile(self):
        events_re = self.registry.events_re
        for iotype in ('in', 'out'):
            events = self.registry.events[iotype]
            for i, (regexp, cregexp) in enumerate(events_re[iotype]):
                e = events[regexp][0]
                events_re[i] = (regexp, e.compile(self.config))

    def attach_events(self, *events, **kwargs):
        """Attach one or more events to the bot instance"""
        reg = self.registry
        insert = 'insert' in kwargs
        for e in events:
            cregexp = e.compile(self.config)
            regexp = getattr(e.regexp, 're', e.regexp)
            if regexp not in reg.events[e.iotype]:
                if insert:
                    reg.events_re[e.iotype].insert(0, (regexp, cregexp))
                else:
                    reg.events_re[e.iotype].append((regexp, cregexp))
            if insert:
                reg.events[e.iotype][regexp].insert(0, e)
            else:
                reg.events[e.iotype][regexp].append(e)

    def detach_events(self, *events):
        """Detach one or more events from the bot instance"""
        reg = self.registry
        delete = defaultdict(list)

        # remove from self.events
        all_events = reg.events
        for e in events:
            regexp = getattr(e.regexp, 're', e.regexp)
            iotype = e.iotype
            if e in all_events[iotype].get(regexp, []):
                all_events[iotype][regexp].remove(e)
                if not all_events[iotype][regexp]:
                    del all_events[iotype][regexp]
                    # need to delete from self.events_re
                    delete[iotype].append(regexp)

        # delete from events_re
        for iotype, regexps in delete.items():
            reg.events_re[iotype] = [r for r in reg.events_re[iotype]
                                     if r[0] not in regexps]

    def include(self, *modules, **kwargs):
        reg = self.registry
        categories = kwargs.get('venusian_categories',
                                self.venusian_categories)
        scanner = self.venusian.Scanner(context=self)
        for module in modules:
            if module in reg.includes:
                self.log.warning('%s included twice', module)
            else:
                reg.includes.add(module)
                try:
                    module = utils.maybedotted(module)
                except LookupError as exc:
                    if HAS_PKG_RESOURCES:
                        entry_points = iter_entry_points(
                            'irc3.loader',
                            module
                        )
                        try:
                            module = next(entry_points).load()
                        except StopIteration:
                            raise exc
                    else:
                        raise exc
                # we have to manualy check for plugins. venusian no longer
                # support to attach both a class and methods
                for klass in list(module.__dict__.values()):
                    if not isinstance(klass, type):
                        continue
                    if klass.__module__ == module.__name__:
                        if getattr(klass, self.plugin_category, False) is True:
                            self.get_plugin(klass)
                reg.scanned.append((module.__name__, categories))
                scanner.scan(module, categories=categories)

    def reload(self, *modules):
        """Reload one or more plugins"""
        self.notify('before_reload')

        if 'configfiles' in self.config:
            # reload configfiles
            self.log.info('Reloading configuration...')
            cfg = utils.parse_config(
                self.server and 'server' or 'bot', *self.config['configfiles'])
            self.config.update(cfg)

        self.log.info('Reloading python code...')
        if not modules:
            modules = self.registry.includes
        scanned = list(reversed(self.registry.scanned))

        # reset includes and events
        self.registry.reset()

        to_scan = []
        for module_name, categories in scanned:
            if module_name in modules:
                module = utils.maybedotted(module_name)
                reload_module(module)
            to_scan.append((module_name, categories))

        # rescan all modules
        for module_name, categories in to_scan:
            self.include(module_name, venusian_categories=categories)

        self.registry.reloading = {}

        self.notify('after_reload')

    def notify(self, event, exc=None, client=None):
        for p in self.registry.plugins.values():
            meth = getattr(p, event, None)
            if meth is not None:
                if client is not None:
                    meth(client=client)
                else:
                    meth()

    def dispatch(self, data, iotype='in', client=None):
        str = utils.IrcString
        create_task = self.create_task
        call_soon = self.loop.call_soon
        for match, events in self.registry.get_event_matches(data, iotype):
            match = match.groupdict()
            for key, value in match.items():
                if value is not None:
                    match[key] = str(value)
            # backwards compatibility fix for IRCv3.2 tag support:
            # If no tags (None-value), exclude from dictionary
            if match.get("tags", True) is None:
                del match["tags"]
            if client is not None:
                # server / dcc chat
                match['client'] = client
            for e in events:
                if e.iscoroutine is True:
                    create_task(e.callback(**match))
                else:
                    call_soon(e.async_callback, match)

    def call_many(self, callback, args):
        """callback is run with each arg but run a call per second"""
        if isinstance(callback, str):
            callback = getattr(self, callback)
        f = None
        for arg in args:
            f = callback(*arg)
        return f

    def get_ssl_context(self):
        if self.config.ssl:  # pragma: no cover
            try:
                create_default_context = ssl.create_default_context
            except AttributeError:  # py < 2.7.9
                return True
            else:
                if self.server:
                    context = create_default_context(ssl.Purpose.CLIENT_AUTH)
                else:
                    context = create_default_context(ssl.Purpose.SERVER_AUTH)
                verify_mode = self.config.ssl_verify
                if verify_mode is not False:
                    if not isinstance(verify_mode, int):
                        # CERT_NONE / CERT_OPTIONAL / CERT_REQUIRED
                        verify_mode = getattr(ssl, verify_mode.upper())
                    if verify_mode == ssl.CERT_NONE:
                        context.check_hostname = False
                    context.verify_mode = verify_mode
                return context
        return None

    def create_connection(self):
        protocol = utils.maybedotted(self.config.connection)
        protocol = type(protocol.__name__, (protocol,), {'factory': self})
        if self.server:  # pragma: no cover
            self.log.debug('Starting {servername}...'.format(**self.config))
            factory = self.loop.create_server
        else:
            self.log.debug('Starting {nick}...'.format(**self.config))
            factory = self.loop.create_connection
        if self.config.get('sock_factory'):
            sock_factory = utils.maybedotted(self.config.sock_factory)
            args = dict(
                sock=sock_factory(self, self.config.host, self.config.port),
                ssl=self.get_ssl_context(),
            )
            if args.get('ssl'):
                args["server_hostname"] = self.config.host
        else:
            args = dict(
                host=self.config.host,
                port=self.config.port,
                ssl=self.get_ssl_context()
            )
            if self.config.get('vhost'):
                args["local_addr"] = (self.config.vhost, 0)
        t = asyncio.Task(factory(protocol, **args), loop=self.loop)
        t.add_done_callback(self.connection_made)
        return self.loop

    def add_signal_handlers(self):
        """Register handlers for UNIX signals (SIGHUP/SIGINT)"""
        try:
            self.loop.add_signal_handler(signal.SIGHUP, self.SIGHUP)
        except (RuntimeError, AttributeError):  # pragma: no cover
            # windows
            pass
        try:
            self.loop.add_signal_handler(signal.SIGINT, self.SIGINT)
        except (RuntimeError, NotImplementedError):  # pragma: no cover
            # annaconda
            pass

    def run(self, forever=True):
        """start the bot"""
        loop = self.create_connection()
        self.add_signal_handlers()
        if forever:
            loop.run_forever()

    @classmethod
    def from_config(cls, cfg, **kwargs):
        """return an instance configured with the ``cfg`` dict"""
        cfg = dict(cfg, **kwargs)
        pythonpath = cfg.get('pythonpath', [])
        if 'here' in cfg:
            pythonpath.append(cfg['here'])
        for path in pythonpath:
            sys.path.append(os.path.expanduser(path))
        prog = cls.server and 'irc3d' or 'irc3'
        if cfg.get('debug'):
            cls.venusian_categories.append(prog + '.debug')
        if cfg.get('interactive'):  # pragma: no cover
            import irc3.testing
            context = getattr(irc3.testing, cls.__name__)(**cfg)
        else:
            context = cls(**cfg)
        if cfg.get('raw'):
            context.include('irc3.plugins.log',
                            venusian_categories=[prog + '.debug'])
        return context

    @classmethod
    def from_argv(cls, argv=None, **kwargs):
        prog = cls.server and 'irc3d' or 'irc3'
        doc = """
        Run an {prog} instance from a config file

        Usage: {prog} [options] <config>...

        Options:

        -h, --help          Display this help and exit
        --version           Output version information and exit
        --logdir DIRECTORY  Log directory to use instead of stderr
        --logdate           Show datetimes in console output
        --host HOST         Server name or ip
        --port PORT         Server port
        -v,--verbose        Increase verbosity
        -r,--raw            Show raw irc log on the console
        -d,--debug          Add some debug commands/utils
        -i,--interactive    Load a ipython console with a bot instance
        """.format(prog=prog)
        if not cls.server:
            doc += """
            --help-page         Output a reST page containing commands help
            """.strip()
        import os
        import docopt
        import textwrap
        args = argv or sys.argv[1:]
        args = docopt.docopt(textwrap.dedent(doc), args, version=version)
        cfg = utils.parse_config(
            cls.server and 'server' or 'bot', *args['<config>'])
        cfg.update(
            verbose=args['--verbose'],
            debug=args['--debug'],
        )
        cfg.update(kwargs)
        if args['--host']:  # pragma: no cover
            host = args['--host']
            cfg['host'] = host
            if host in ('127.0.0.1', 'localhost'):
                cfg['ssl'] = False
        if args['--port']:  # pragma: no cover
            cfg['port'] = args['--port']
        if args['--logdir'] or 'logdir' in cfg:
            logdir = os.path.expanduser(args['--logdir'] or cfg.get('logdir'))
            cls.logging_config = config.get_file_config(logdir)
        if args['--logdate']:  # pragma: no cover
            fmt = cls.logging_config['formatters']['console']
            fmt['format'] = config.TIMESTAMPED_FMT
        if args.get('--help-page'):  # pragma: no cover
            for v in cls.logging_config['handlers'].values():
                v['level'] = 'ERROR'
        if args['--raw']:
            cfg['raw'] = True
        context = cls.from_config(cfg)
        if args.get('--help-page'):  # pragma: no cover
            context.print_help_page()
        elif args['--interactive']:  # pragma: no cover
            import IPython
            IPython.embed()
            sys.exit(0)
        else:
            context.run(forever=not bool(kwargs))
        if kwargs or argv:
            return context
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import threading
import cmd

ENGINE_NAME = 'simple UCI chess engine'
AUTHOR_NAME = 'Alexey Syromyatnikov'

class Analyzer(threading.Thread):
    def __init__(self):
        super(Analyzer, self).__init__()
        self.debug = False

    def run(self):
        pass

class EngineShell(cmd.Cmd):
    intro = ''
    prompt = ''
    file = None

    def __init__(self):
        super(EngineShell, self).__init__()
        self.analyzer = Analyzer()

    def do_uci(self, arg):
        print('id name', ENGINE_NAME)
        print('id author', AUTHOR_NAME)
        print('uciok')

    def do_debug(self, arg):
        arg = arg.split()
        if arg:
            arg = arg[0]
        else:
            return
        if arg == 'on':
            self.analyzer.debug = True
        elif arg == 'off':
            self.analyzer.debug = False

    def do_isready(self, arg):
        pass

    def do_setoption(self, arg):
        pass

    def do_ucinewgame(self, arg):
        pass

    def do_position(self, arg):
        pass

    def do_stop(self, arg):
        pass

    def do_quit(self, arg):
        pass

    def default(self, arg):
        pass

if __name__ == '__main__':
    EngineShell().cmdloop()
<commit_msg>postinit, do_stop<commit_after># -*- coding: utf-8 -*-

import threading
import cmd
import chess

ENGINE_NAME = 'simple UCI chess engine'
AUTHOR_NAME = 'Alexey Syromyatnikov'

class Analyzer(threading.Thread):
    def __init__(self):
        super(Analyzer, self).__init__()
        self.debug = False
        self.is_working = threading.Event()
        self.is_working.clear()
        self.is_conscious = threading.Condition()
        self.is_bestmove_ready = threading.Event()
        self.is_bestmove_ready.clear()

        self._bestmove = chess.Move.null()

    @property
    def bestmove(self):
        return self._bestmove.uci()

    def run(self):
        pass

class EngineShell(cmd.Cmd):
    intro = ''
    prompt = ''
    file = None

    def __init__(self):
        super(EngineShell, self).__init__()
        self.postinitialized = False

    def postinit(self):
        self.analyzer = Analyzer()
        self.postinitialized = True

    def do_uci(self, arg):
        print('id name', ENGINE_NAME)
        print('id author', AUTHOR_NAME)
        print('uciok')

    def do_debug(self, arg):
        arg = arg.split()
        if arg:
            arg = arg[0]
        else:
            return
        if arg == 'on':
            self.analyzer.debug = True
        elif arg == 'off':
            self.analyzer.debug = False

    def do_isready(self, arg):
        if not self.postinitialized:
            self.postinit()
        if self.analyzer.is_working.is_set():
            with self.analyzer.is_conscious:
                self.analyzer.is_conscious.wait()
        print('readyok')

    def do_setoption(self, arg):
        pass

    def do_ucinewgame(self, arg):
        pass

    def do_position(self, arg):
        pass

    def do_go(self, arg):
        pass

    def do_stop(self, arg):
        if self.analyzer.is_working.is_set():
            self.analyzer.is_working.clear()
            with self.analyzer.is_bestmove_ready:
                self.analyzer.is_bestmove_ready.wait()
        print(self.analyzer.bestmove)

    def do_quit(self, arg):
        pass

    def default(self, arg):
        pass

if __name__ == '__main__':
    EngineShell().cmdloop()
<|endoftext|>


<commit_before>"""

Camoco Library - CoAnalysis of Molecular Components

CacheMoneyCorn

"""

__license__ = """

The "MIT" License

Copyright (c) 2017 Robert Schaefer

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.

"""

__version__ = '0.5.1'

import sys
import os
import numpy

import pyximport
pyximport.install(setup_args={
    "include_dirs":numpy.get_include()
})

import matplotlib
matplotlib.use('Agg')

from .Config import cf
from .Camoco import Camoco
from .Expr import Expr
from .COB import COB
from .RefGen import RefGen
#from .RefGenDist import *
#from .PCCUP import *

from .Ontology import Ontology,Term
from .GWAS import GWAS
from .Locus import Locus
#from .Tools import available_datasets,del_dataset
#from .Tools import mv_dataset,redescribe_dataset
#from .GEO import Family
from .GOnt import GOnt
#from .Overlap import Overlap

# Create yourself
Camoco.create('Camoco','Mother Database')
<commit_msg>bump patch version<commit_after>"""

Camoco Library - CoAnalysis of Molecular Components

CacheMoneyCorn

"""

__license__ = """

The "MIT" License

Copyright (c) 2017 Robert Schaefer

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.

"""

__version__ = '0.5.2'

import sys
import os
import numpy

import pyximport
pyximport.install(setup_args={
    "include_dirs":numpy.get_include()
})

import matplotlib
matplotlib.use('Agg')

from .Config import cf
from .Camoco import Camoco
from .Expr import Expr
from .COB import COB
from .RefGen import RefGen
#from .RefGenDist import *
#from .PCCUP import *

from .Ontology import Ontology,Term
from .GWAS import GWAS
from .Locus import Locus
#from .Tools import available_datasets,del_dataset
#from .Tools import mv_dataset,redescribe_dataset
#from .GEO import Family
from .GOnt import GOnt
#from .Overlap import Overlap

# Create yourself
Camoco.create('Camoco','Mother Database')
<|endoftext|>


<commit_before>from itertools import tee, chain

from zipline.gens.utils import roundrobin, done_message
from zipline.gens.sort import date_sort
from zipline.gens.merge import merge
from zipline.gens.transform import StatefulTransform

def date_sorted_sources(*sources):
    """
    Takes an iterable of sources, generating namestrings and
    piping their output into date_sort.
    """

    for source in sources:
        assert iter(source), "Source %s not iterable" % source
        assert hasattr(source, 'get_hash'), "No get_hash"

    # Get name hashes to pass to date_sort.
    names = [source.get_hash() for source in sources]

    # Convert the list of generators into a flat stream by pulling
    # one element at a time from each.
    stream_in = roundrobin(sources, names)

    # Guarantee the flat stream will be sorted by date, using
    # source_id as tie-breaker, which is fully deterministic (given
    # deterministic string representation for all args/kwargs)

    return date_sort(stream_in, names)

def merged_transforms(sorted_stream, *transforms):
    """
    A generator that takes the expected output of a date_sort, pipes
    it through a given set of transforms, and runs the results
    through a merge to output a unified stream. tnfms should be a
    list of pointers to generator functions. tnfm_args should be a
    list of tuples, representing the arguments to be passed to each
    transform.  tnfm_kwargs should be a list of dictionaries
    representing keyword arguments to each transform.
    """
    for transform in transforms:
        assert isinstance(transform, StatefulTransform)
        transform.merged = True
        transform.sequential = False

    # Generate expected hashes for each transform
    namestrings = [tnfm.get_hash() for tnfm in transforms]

    # Create a copy of the stream for each transform.
    split = tee(sorted_stream, len(transforms))

    # Package a stream copy with each StatefulTransform instance.
    bundles = zip(transforms, split)

    # Convert the copies into transform streams.
    tnfm_gens = [tnfm.transform(stream) for tnfm, stream in bundles]

    # Roundrobin the outputs of our transforms to create a single flat
    # stream.
    to_merge = roundrobin(tnfm_gens, namestrings)

    # Pipe the stream into merge.
    merged = merge(to_merge, namestrings)

    dt_aliased = alias_dt(merged)
    # Return the merged events.
    return add_done(dt_aliased)

def sequential_transforms(stream_in, *transforms):
    """
    Apply each transform in transforms sequentially to each event in stream_in.
    Each transform application will add a new entry indexed to the transform's
    hash string.
    """

    assert isinstance(transforms, (list, tuple))

    for tnfm in transforms:
        tnfm.sequential = True
        tnfm.merged = False

    # Recursively apply all transforms to the stream.
    stream_out = reduce(lambda stream, tnfm: tnfm.transform(stream),
                        transforms,
                        stream_in)

    dt_aliased = alias_dt(stream_out)
    return add_done(dt_aliased)

def alias_dt(stream_in):
    """
    Alias the dt field to datetime on each message.
    """
    for message in stream_in:
        message['datetime'] = message['dt']
        yield message

# Add a done message to a stream.
def add_done(stream_in):
    return chain(stream_in, [done_message('Composite')])
<commit_msg>Cleaned up depracated merged_transforms.<commit_after>from itertools import chain

from zipline.gens.utils import roundrobin, done_message
from zipline.gens.sort import date_sort

def date_sorted_sources(*sources):
    """
    Takes an iterable of sources, generating namestrings and
    piping their output into date_sort.
    """

    for source in sources:
        assert iter(source), "Source %s not iterable" % source
        assert hasattr(source, 'get_hash'), "No get_hash"

    # Get name hashes to pass to date_sort.
    names = [source.get_hash() for source in sources]

    # Convert the list of generators into a flat stream by pulling
    # one element at a time from each.
    stream_in = roundrobin(sources, names)

    # Guarantee the flat stream will be sorted by date, using
    # source_id as tie-breaker, which is fully deterministic (given
    # deterministic string representation for all args/kwargs)

    return date_sort(stream_in, names)

def sequential_transforms(stream_in, *transforms):
    """
    Apply each transform in transforms sequentially to each event in stream_in.
    Each transform application will add a new entry indexed to the transform's
    hash string.
    """

    assert isinstance(transforms, (list, tuple))

    for tnfm in transforms:
        tnfm.sequential = True
        tnfm.merged = False

    # Recursively apply all transforms to the stream.
    stream_out = reduce(lambda stream, tnfm: tnfm.transform(stream),
                        transforms,
                        stream_in)

    dt_aliased = alias_dt(stream_out)
    return add_done(dt_aliased)

def alias_dt(stream_in):
    """
    Alias the dt field to datetime on each message.
    """
    for message in stream_in:
        message['datetime'] = message['dt']
        yield message

# Add a done message to a stream.
def add_done(stream_in):
    return chain(stream_in, [done_message('Composite')])
<|endoftext|>