<commit_before>import sys
import shutil
import os
try:
    import docker
    from docker.errors import DockerException
    from girder_worker.docker import nvidia
    from requests.exceptions import ReadTimeout
except ImportError:
    # These imports will not be available on the girder side.
    pass
from girder_worker.app import app, Task
from girder_worker import logger
from girder_worker.docker import utils
from girder_worker.docker.stream_adapter import DockerStreamPushAdapter
from girder_worker.docker.io import (
    FileDescriptorReader,
    FDWriteStreamConnector,
    FDReadStreamConnector,
    FDStreamConnector,
    StdStreamWriter
)
from girder_worker.docker.transforms import (
    ContainerStdErr,
    ContainerStdOut,
    _TemporaryVolumeBase,
    TemporaryVolume
)
from girder_worker_utils import _walk_obj

BLACKLISTED_DOCKER_RUN_ARGS = ['tty', 'detach']

def _pull_image(image):
    """
    Pulls the specified Docker image onto this worker.
    """
    client = docker.from_env(version='auto')
    try:
        client.images.pull(image)
    except DockerException:
        logger.exception('Error pulling Docker image %s:' % image)
        raise

def _run_container(image, container_args,  **kwargs):
    # TODO we could allow configuration of non default socket
    client = docker.from_env(version='auto')
    if nvidia.is_nvidia_image(client.api, image):
        client = nvidia.NvidiaDockerClient.from_env(version='auto')

    logger.info('Running container: image: %s args: %s kwargs: %s'
                % (image, container_args, kwargs))
    try:
        return client.containers.run(image, container_args, **kwargs)
    except nvidia.NvidiaConnectionError:
        try:
            logger.info('Running nvidia container without nvidia support: image: %s' % image)
            client = docker.from_env(version='auto')
            return client.containers.run(image, container_args, **kwargs)
        except DockerException:
            logger.exception('Exception when running docker container without nvidia support.')
            raise
    except DockerException:
        logger.exception('Exception when running docker container')
        raise

class _SocketReader(FileDescriptorReader):
    """
    Used to mediate the difference between the python 2/3 implementation of docker-py
    with python 2 attach_socket(...) returns a socket like object, with python 3
    it returns an instance of SocketIO.
    """
    def __init__(self, socket):
        self._socket = socket

    def read(self, n):
        # socket
        if hasattr(self._socket, 'recv'):
            return self._socket.recv(n)

        # SocketIO
        return self._socket.read(n)

    def fileno(self):
        return self._socket.fileno()

    def close(self):
        self._socket.close()

def _run_select_loop(task, container, read_stream_connectors, write_stream_connectors):
    stdout = None
    stderr = None
    try:
        # attach to standard streams
        stdout = container.attach_socket(params={
            'stdout': True,
            'logs': True,
            'stream': True
        })

        stderr = container.attach_socket(params={
            'stderr': True,
            'logs': True,
            'stream': True
        })

        def exit_condition():
            container.reload()
            return container.status in {'exited', 'dead'} or task.canceled

        # Look for ContainerStdOut and ContainerStdErr instances that need
        # to be replace with the real container streams.
        stdout_connected = False
        for read_stream_connector in read_stream_connectors:
            if isinstance(read_stream_connector.input, ContainerStdOut):
                stdout_reader = _SocketReader(stdout)
                read_stream_connector.output = DockerStreamPushAdapter(read_stream_connector.output)
                read_stream_connector.input = stdout_reader
                stdout_connected = True
                break

        stderr_connected = False
        for read_stream_connector in read_stream_connectors:
            if isinstance(read_stream_connector.input, ContainerStdErr):
                stderr_reader = _SocketReader(stderr)
                read_stream_connector.output = DockerStreamPushAdapter(read_stream_connector.output)
                read_stream_connector.input = stderr_reader
                stderr_connected = True
                break

        # If not stdout and stderr connection has been provided just use
        # sys.stdXXX
        if not stdout_connected:
            stdout_reader = _SocketReader(stdout)
            connector = FDReadStreamConnector(
                stdout_reader,
                DockerStreamPushAdapter(StdStreamWriter(sys.stdout)))
            read_stream_connectors.append(connector)

        if not stderr_connected:
            stderr_reader = _SocketReader(stderr)
            connector = FDReadStreamConnector(
                stderr_reader,
                DockerStreamPushAdapter(StdStreamWriter(sys.stderr)))
            read_stream_connectors.append(connector)

        # Run select loop
        utils.select_loop(exit_condition=exit_condition,
                          readers=read_stream_connectors,
                          writers=write_stream_connectors)

        if task.canceled:
            try:
                container.stop()
            # Catch the ReadTimeout from requests and wait for container to
            # exit. See https://github.com/docker/docker-py/issues/1374 for
            # more details.
            except ReadTimeout:
                tries = 10
                while tries > 0:
                    container.reload()
                    if container.status == 'exited':
                        break

                if container.status != 'exited':
                    msg = 'Unable to stop container: %s' % container.id
                    logger.error(msg)
            except DockerException as dex:
                logger.error(dex)
                raise

        container.reload()
        exit_code = container.attrs['State']['ExitCode']
        if exit_code != 0:
            raise DockerException('Non-zero exit code from docker container (%d).' % exit_code)
    finally:
        # Close our stdout and stderr sockets
        if stdout:
            stdout.close()
        if stderr:
            stderr.close()

def _handle_streaming_args(args):
    processed_args = []
    write_streams = []
    read_streams = []

    def _maybe_path(arg):
        if hasattr(arg, 'path'):
            return arg.path()

        # Don't pass anything
        return ''

    for arg in args:
        if isinstance(arg, FDStreamConnector):
            if isinstance(arg, FDWriteStreamConnector):
                write_streams.append(arg)
                arg = _maybe_path(arg.output)

            elif isinstance(arg, FDReadStreamConnector):
                read_streams.append(arg)
                arg = _maybe_path(arg.input)

        processed_args.append(arg)

    return (processed_args, read_streams, write_streams)

class _RequestDefaultTemporaryVolume(_TemporaryVolumeBase):
    def __init__(self):
        super(_RequestDefaultTemporaryVolume, self).__init__(None, None)
        self._make_paths()

    def transform(self, **kwargs):
        self._transformed = True

class DockerTask(Task):

    def _maybe_transform_argument(self, arg):
        return super(DockerTask, self)._maybe_transform_argument(
            arg, task=self, _default_temp_volume=self.request._default_temp_volume)

    def _maybe_transform_result(self, idx, result):
        return super(DockerTask, self)._maybe_transform_result(
            idx, result, _default_temp_volume=self.request._default_temp_volume)

    def __call__(self, *args, **kwargs):
        default_temp_volume = _RequestDefaultTemporaryVolume()
        self.request._default_temp_volume = default_temp_volume

        volumes = kwargs.setdefault('volumes', {})
        # If we have a list of volumes, the user provide a list of Volume objects,
        # we need to transform them.
        temp_volumes = []
        if isinstance(volumes, list):
            # See if we have been passed any TemporaryVolume instances.
            for v in volumes:
                if isinstance(v, TemporaryVolume):
                    temp_volumes.append(v)

            # First call the transform method, this we replace default temp volumes
            # with the instance associated with this task create above. That is any
            # reference to TemporaryVolume.default
            _walk_obj(volumes, self._maybe_transform_argument)

            # Now convert them to JSON
            def _json(volume):
                return volume._repr_json_()

            volumes = _walk_obj(volumes, _json)
            # We then need to merge them into a single dict and it will be ready
            # for docker-py.
            volumes = {k: v for volume in volumes for k, v in volume.items()}
            kwargs['volumes'] = volumes

        volumes.update(default_temp_volume._repr_json_())

        super(DockerTask, self).__call__(*args, **kwargs)

        # Set the permission to allow cleanup of temp directories
        temp_volumes = [v for v in temp_volumes if os.path.exists(v.host_path)]
        to_chmod = temp_volumes[:]
        # If our default_temp_volume instance has been transformed then we
        # know it has been used and we have to clean it up.
        if default_temp_volume._transformed:
            to_chmod.append(default_temp_volume)
            temp_volumes.append(default_temp_volume)

        if len(to_chmod) > 0:
            utils.chmod_writable([v.host_path for v in to_chmod])

        for v in temp_volumes:
            shutil.rmtree(v.host_path)

def _docker_run(task, image, pull_image=True, entrypoint=None, container_args=None,
                volumes={}, remove_container=True, stream_connectors=[], **kwargs):

    if pull_image:
        logger.info('Pulling Docker image: %s', image)
        _pull_image(image)

    if entrypoint is not None:
        if not isinstance(entrypoint, (list, tuple)):
            entrypoint = [entrypoint]

    run_kwargs = {
        'tty': False,
        'volumes': volumes,
        'detach': True
    }

    # Allow run args to be overridden,filter out any we don't want to override
    extra_run_kwargs = {k: v for k, v in kwargs.items() if k not
                        in BLACKLISTED_DOCKER_RUN_ARGS}
    run_kwargs.update(extra_run_kwargs)

    if entrypoint is not None:
        run_kwargs['entrypoint'] = entrypoint

    (container_args, read_streams, write_streams) = \
        _handle_streaming_args(container_args)

    for connector in stream_connectors:
        if isinstance(connector, FDReadStreamConnector):
            read_streams.append(connector)
        elif isinstance(connector, FDWriteStreamConnector):
            write_streams.append(connector)
        else:
            raise TypeError(
                "Expected 'FDReadStreamConnector' or 'FDWriterStreamConnector', received '%s'"
                % type(connector))

    # We need to open any read streams before starting the container, so the
    # underling named pipes are opened for read.
    for stream in read_streams:
        stream.open()

    container = _run_container(image, container_args, **run_kwargs)
    try:
        _run_select_loop(task, container, read_streams, write_streams)
    finally:
        if container and remove_container:
            container.reload()
            # If the container is still running issue a warning
            if container.status == 'running':
                logger.warning('Container is still running, unable to remove.')
            else:
                container.remove()

    # return an array of None's equal to number of entries in the girder_result_hooks
    # header, in order to trigger processing of the container outputs.
    results = []
    if hasattr(task.request, 'girder_result_hooks'):
        results = (None,) * len(task.request.girder_result_hooks)

    return results

@app.task(base=DockerTask, bind=True)
def docker_run(task, image, pull_image=True, entrypoint=None, container_args=None,
               volumes={}, remove_container=True, **kwargs):
    return _docker_run(
        task, image, pull_image, entrypoint, container_args, volumes,
        remove_container, **kwargs)
<commit_msg>Fixes #266. Allow use of default container_args param<commit_after>import sys
import shutil
import os
try:
    import docker
    from docker.errors import DockerException
    from girder_worker.docker import nvidia
    from requests.exceptions import ReadTimeout
except ImportError:
    # These imports will not be available on the girder side.
    pass
from girder_worker.app import app, Task
from girder_worker import logger
from girder_worker.docker import utils
from girder_worker.docker.stream_adapter import DockerStreamPushAdapter
from girder_worker.docker.io import (
    FileDescriptorReader,
    FDWriteStreamConnector,
    FDReadStreamConnector,
    FDStreamConnector,
    StdStreamWriter
)
from girder_worker.docker.transforms import (
    ContainerStdErr,
    ContainerStdOut,
    _TemporaryVolumeBase,
    TemporaryVolume
)
from girder_worker_utils import _walk_obj

BLACKLISTED_DOCKER_RUN_ARGS = ['tty', 'detach']

def _pull_image(image):
    """
    Pulls the specified Docker image onto this worker.
    """
    client = docker.from_env(version='auto')
    try:
        client.images.pull(image)
    except DockerException:
        logger.exception('Error pulling Docker image %s:' % image)
        raise

def _run_container(image, container_args,  **kwargs):
    # TODO we could allow configuration of non default socket
    client = docker.from_env(version='auto')
    if nvidia.is_nvidia_image(client.api, image):
        client = nvidia.NvidiaDockerClient.from_env(version='auto')

    logger.info('Running container: image: %s args: %s kwargs: %s'
                % (image, container_args, kwargs))
    try:
        return client.containers.run(image, container_args, **kwargs)
    except nvidia.NvidiaConnectionError:
        try:
            logger.info('Running nvidia container without nvidia support: image: %s' % image)
            client = docker.from_env(version='auto')
            return client.containers.run(image, container_args, **kwargs)
        except DockerException:
            logger.exception('Exception when running docker container without nvidia support.')
            raise
    except DockerException:
        logger.exception('Exception when running docker container')
        raise

class _SocketReader(FileDescriptorReader):
    """
    Used to mediate the difference between the python 2/3 implementation of docker-py
    with python 2 attach_socket(...) returns a socket like object, with python 3
    it returns an instance of SocketIO.
    """
    def __init__(self, socket):
        self._socket = socket

    def read(self, n):
        # socket
        if hasattr(self._socket, 'recv'):
            return self._socket.recv(n)

        # SocketIO
        return self._socket.read(n)

    def fileno(self):
        return self._socket.fileno()

    def close(self):
        self._socket.close()

def _run_select_loop(task, container, read_stream_connectors, write_stream_connectors):
    stdout = None
    stderr = None
    try:
        # attach to standard streams
        stdout = container.attach_socket(params={
            'stdout': True,
            'logs': True,
            'stream': True
        })

        stderr = container.attach_socket(params={
            'stderr': True,
            'logs': True,
            'stream': True
        })

        def exit_condition():
            container.reload()
            return container.status in {'exited', 'dead'} or task.canceled

        # Look for ContainerStdOut and ContainerStdErr instances that need
        # to be replace with the real container streams.
        stdout_connected = False
        for read_stream_connector in read_stream_connectors:
            if isinstance(read_stream_connector.input, ContainerStdOut):
                stdout_reader = _SocketReader(stdout)
                read_stream_connector.output = DockerStreamPushAdapter(read_stream_connector.output)
                read_stream_connector.input = stdout_reader
                stdout_connected = True
                break

        stderr_connected = False
        for read_stream_connector in read_stream_connectors:
            if isinstance(read_stream_connector.input, ContainerStdErr):
                stderr_reader = _SocketReader(stderr)
                read_stream_connector.output = DockerStreamPushAdapter(read_stream_connector.output)
                read_stream_connector.input = stderr_reader
                stderr_connected = True
                break

        # If not stdout and stderr connection has been provided just use
        # sys.stdXXX
        if not stdout_connected:
            stdout_reader = _SocketReader(stdout)
            connector = FDReadStreamConnector(
                stdout_reader,
                DockerStreamPushAdapter(StdStreamWriter(sys.stdout)))
            read_stream_connectors.append(connector)

        if not stderr_connected:
            stderr_reader = _SocketReader(stderr)
            connector = FDReadStreamConnector(
                stderr_reader,
                DockerStreamPushAdapter(StdStreamWriter(sys.stderr)))
            read_stream_connectors.append(connector)

        # Run select loop
        utils.select_loop(exit_condition=exit_condition,
                          readers=read_stream_connectors,
                          writers=write_stream_connectors)

        if task.canceled:
            try:
                container.stop()
            # Catch the ReadTimeout from requests and wait for container to
            # exit. See https://github.com/docker/docker-py/issues/1374 for
            # more details.
            except ReadTimeout:
                tries = 10
                while tries > 0:
                    container.reload()
                    if container.status == 'exited':
                        break

                if container.status != 'exited':
                    msg = 'Unable to stop container: %s' % container.id
                    logger.error(msg)
            except DockerException as dex:
                logger.error(dex)
                raise

        container.reload()
        exit_code = container.attrs['State']['ExitCode']
        if exit_code != 0:
            raise DockerException('Non-zero exit code from docker container (%d).' % exit_code)
    finally:
        # Close our stdout and stderr sockets
        if stdout:
            stdout.close()
        if stderr:
            stderr.close()

def _handle_streaming_args(args):
    processed_args = []
    write_streams = []
    read_streams = []

    def _maybe_path(arg):
        if hasattr(arg, 'path'):
            return arg.path()

        # Don't pass anything
        return ''

    for arg in args:
        if isinstance(arg, FDStreamConnector):
            if isinstance(arg, FDWriteStreamConnector):
                write_streams.append(arg)
                arg = _maybe_path(arg.output)

            elif isinstance(arg, FDReadStreamConnector):
                read_streams.append(arg)
                arg = _maybe_path(arg.input)

        processed_args.append(arg)

    return (processed_args, read_streams, write_streams)

class _RequestDefaultTemporaryVolume(_TemporaryVolumeBase):
    def __init__(self):
        super(_RequestDefaultTemporaryVolume, self).__init__(None, None)
        self._make_paths()

    def transform(self, **kwargs):
        self._transformed = True

class DockerTask(Task):

    def _maybe_transform_argument(self, arg):
        return super(DockerTask, self)._maybe_transform_argument(
            arg, task=self, _default_temp_volume=self.request._default_temp_volume)

    def _maybe_transform_result(self, idx, result):
        return super(DockerTask, self)._maybe_transform_result(
            idx, result, _default_temp_volume=self.request._default_temp_volume)

    def __call__(self, *args, **kwargs):
        default_temp_volume = _RequestDefaultTemporaryVolume()
        self.request._default_temp_volume = default_temp_volume

        volumes = kwargs.setdefault('volumes', {})
        # If we have a list of volumes, the user provide a list of Volume objects,
        # we need to transform them.
        temp_volumes = []
        if isinstance(volumes, list):
            # See if we have been passed any TemporaryVolume instances.
            for v in volumes:
                if isinstance(v, TemporaryVolume):
                    temp_volumes.append(v)

            # First call the transform method, this we replace default temp volumes
            # with the instance associated with this task create above. That is any
            # reference to TemporaryVolume.default
            _walk_obj(volumes, self._maybe_transform_argument)

            # Now convert them to JSON
            def _json(volume):
                return volume._repr_json_()

            volumes = _walk_obj(volumes, _json)
            # We then need to merge them into a single dict and it will be ready
            # for docker-py.
            volumes = {k: v for volume in volumes for k, v in volume.items()}
            kwargs['volumes'] = volumes

        volumes.update(default_temp_volume._repr_json_())

        super(DockerTask, self).__call__(*args, **kwargs)

        # Set the permission to allow cleanup of temp directories
        temp_volumes = [v for v in temp_volumes if os.path.exists(v.host_path)]
        to_chmod = temp_volumes[:]
        # If our default_temp_volume instance has been transformed then we
        # know it has been used and we have to clean it up.
        if default_temp_volume._transformed:
            to_chmod.append(default_temp_volume)
            temp_volumes.append(default_temp_volume)

        if len(to_chmod) > 0:
            utils.chmod_writable([v.host_path for v in to_chmod])

        for v in temp_volumes:
            shutil.rmtree(v.host_path)

def _docker_run(task, image, pull_image=True, entrypoint=None, container_args=None,
                volumes=None, remove_container=True, stream_connectors=None, **kwargs):
    volumes = volumes or {}
    stream_connectors = stream_connectors or []
    container_args = container_args or []

    if pull_image:
        logger.info('Pulling Docker image: %s', image)
        _pull_image(image)

    if entrypoint is not None and not isinstance(entrypoint, (list, tuple)):
        entrypoint = [entrypoint]

    run_kwargs = {
        'tty': False,
        'volumes': volumes,
        'detach': True
    }

    # Allow run args to be overridden,filter out any we don't want to override
    extra_run_kwargs = {k: v for k, v in kwargs.items() if k not in BLACKLISTED_DOCKER_RUN_ARGS}
    run_kwargs.update(extra_run_kwargs)

    if entrypoint is not None:
        run_kwargs['entrypoint'] = entrypoint

    container_args, read_streams, write_streams = _handle_streaming_args(container_args)

    for connector in stream_connectors:
        if isinstance(connector, FDReadStreamConnector):
            read_streams.append(connector)
        elif isinstance(connector, FDWriteStreamConnector):
            write_streams.append(connector)
        else:
            raise TypeError(
                "Expected 'FDReadStreamConnector' or 'FDWriterStreamConnector', received '%s'"
                % type(connector))

    # We need to open any read streams before starting the container, so the
    # underling named pipes are opened for read.
    for stream in read_streams:
        stream.open()

    container = _run_container(image, container_args, **run_kwargs)
    try:
        _run_select_loop(task, container, read_streams, write_streams)
    finally:
        if container and remove_container:
            container.reload()
            # If the container is still running issue a warning
            if container.status == 'running':
                logger.warning('Container is still running, unable to remove.')
            else:
                container.remove()

    # return an array of None's equal to number of entries in the girder_result_hooks
    # header, in order to trigger processing of the container outputs.
    results = []
    if hasattr(task.request, 'girder_result_hooks'):
        results = (None,) * len(task.request.girder_result_hooks)

    return results

@app.task(base=DockerTask, bind=True)
def docker_run(task, image, pull_image=True, entrypoint=None, container_args=None,
               volumes=None, remove_container=True, **kwargs):
    return _docker_run(
        task, image, pull_image, entrypoint, container_args, volumes,
        remove_container, **kwargs)
<|endoftext|>


<commit_before>import os
from utils import Reader

if __name__ == '__main__':
    working_directory = os.getcwd()
    files = Reader.read_directory(working_directory)
    for f in files:
        tweets = Reader.read_file(f)
        for tweet in tweets:
            print '{}, {}'.format(tweet.verb(), tweet.timestamp())

<commit_msg>Return only english tweet and print the body of the tweet for analysis via other tools<commit_after>import os
from utils import Reader

if __name__ == '__main__':
    working_directory = os.getcwd()
    files = Reader.read_directory(working_directory)
    for f in files:
        tweets = Reader.read_file(f)
        eng_tweets = filter(lambda t: t.language() == 'en', tweets)
        for tweet in tweets:
            print '{}, {}, {}'.format(tweet.verb(), tweet.timestamp(), tweet.body())

<|endoftext|>


<commit_before># Copyright (C) 2013 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: david@reciprocitylabs.com
# Maintained By: david@reciprocitylabs.com

import threading
from ggrc import db
from ggrc.app import app
from ggrc.models import create_db, drop_db
from wsgiref.simple_server import make_server

use_migrations = True

def before_all(context):
  context.base_url = 'http://localhost:9000'
  create_db(use_migrations)
  app.debug = False
  app.testing = True
  context.server = make_server('', 9000, app)
  context.thread = threading.Thread(target=context.server.serve_forever)
  context.thread.start()

def after_all(context):
  context.server.shutdown()
  context.thread.join()
  db.session.remove()
  drop_db(use_migrations)
<commit_msg>Don't use migrations for tests<commit_after># Copyright (C) 2013 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: david@reciprocitylabs.com
# Maintained By: david@reciprocitylabs.com

import threading
from ggrc import db
from ggrc.app import app
from ggrc.models import create_db, drop_db
from wsgiref.simple_server import make_server

use_migrations = False

def before_all(context):
  context.base_url = 'http://localhost:9000'
  create_db(use_migrations)
  app.debug = False
  app.testing = True
  context.server = make_server('', 9000, app)
  context.thread = threading.Thread(target=context.server.serve_forever)
  context.thread.start()

def after_all(context):
  context.server.shutdown()
  context.thread.join()
  db.session.remove()
  drop_db(use_migrations)
<|endoftext|>


<commit_before>##########################################################################
#
#  Copyright (c) 2008, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#
#     * Neither the name of Image Engine Design nor the names of any
#       other contributors to this software may be used to endorse or
#       promote products derived from this software without specific prior
#       written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import maya.OpenMaya
import maya.cmds
import IECoreMaya
import _IECoreMaya
import StringUtil

class TransientParameterisedHolderNode( maya.OpenMaya.MFnDependencyNode ) :

	def __init__( self, layoutName, classNameOrParameterised, classVersion=None, envVarName=None ) :
		""" Creates a temporary TransientParameterisedHolderNode in order to present the UI for the specified
		    parameterised object in the given layout. The node is automatically deleted when the holding
		    layout is destroyed """

		nodeName = maya.cmds.createNode( "ieTransientParameterisedHolderNode", skipSelect = True )

		# Script jobs aren't available from maya.cmds. Maya Python bindings generate swig warnings
		# such as "swig/python detected a memory leak of type 'MCallbackId *', no destructor found"
		IECoreMaya.mel( 'scriptJob -uiDeleted "%s" "delete %s" -protected' % ( layoutName, nodeName ) )

		object = StringUtil.dependencyNodeFromString( nodeName )

		maya.OpenMaya.MFnDependencyNode.__init__( self, object )

		if isinstance( classNameOrParameterised, str ) :
			res = _IECoreMaya._parameterisedHolderSetParameterised( self, classNameOrParameterised, classVersion, envVarName )
		else :
			assert( not classVersion )
			assert( not envVarName )

			res = _IECoreMaya._parameterisedHolderSetParameterised( self, classNameOrParameterised )

		parameterised = self.getParameterised()[0]

		if parameterised:
			maya.cmds.setParent( layoutName )
			IECoreMaya.ParameterUI.create( object, parameterised.parameters() )
			maya.cmds.setParent( layoutName )

		return res

	def getParameterised( self ) :

		return _IECoreMaya._parameterisedHolderGetParameterised( self )
<commit_msg>Added a todo.<commit_after>##########################################################################
#
#  Copyright (c) 2008, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#
#     * Neither the name of Image Engine Design nor the names of any
#       other contributors to this software may be used to endorse or
#       promote products derived from this software without specific prior
#       written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import maya.OpenMaya
import maya.cmds
import IECoreMaya
import _IECoreMaya
import StringUtil

## \todo To follow the form of the rest of this module, this should be named FnTransientParameterisedHolderNode,
# and derived from FnParameterisedHolder. The __init__ method should just wrap around an already existing node,
# and the existing init functionality should be placed in a static method called create().
class TransientParameterisedHolderNode( maya.OpenMaya.MFnDependencyNode ) :

	def __init__( self, layoutName, classNameOrParameterised, classVersion=None, envVarName=None ) :
		""" Creates a temporary TransientParameterisedHolderNode in order to present the UI for the specified
		    parameterised object in the given layout. The node is automatically deleted when the holding
		    layout is destroyed """

		nodeName = maya.cmds.createNode( "ieTransientParameterisedHolderNode", skipSelect = True )

		# Script jobs aren't available from maya.cmds. Maya Python bindings generate swig warnings
		# such as "swig/python detected a memory leak of type 'MCallbackId *', no destructor found"
		IECoreMaya.mel( 'scriptJob -uiDeleted "%s" "delete %s" -protected' % ( layoutName, nodeName ) )

		object = StringUtil.dependencyNodeFromString( nodeName )

		maya.OpenMaya.MFnDependencyNode.__init__( self, object )

		if isinstance( classNameOrParameterised, str ) :
			res = _IECoreMaya._parameterisedHolderSetParameterised( self, classNameOrParameterised, classVersion, envVarName )
		else :
			assert( not classVersion )
			assert( not envVarName )

			res = _IECoreMaya._parameterisedHolderSetParameterised( self, classNameOrParameterised )

		parameterised = self.getParameterised()[0]

		if parameterised:
			maya.cmds.setParent( layoutName )
			IECoreMaya.ParameterUI.create( object, parameterised.parameters() )
			maya.cmds.setParent( layoutName )

		return res

	## \todo This will be unecessary if this class is derived from FnParameterisedHolder.
	def getParameterised( self ) :

		return _IECoreMaya._parameterisedHolderGetParameterised( self )
<|endoftext|>


<commit_before># Copyright 2020 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A tool to test sigma rules.
This tool can be used to verify your rules before running an analyzer.
It also does not require you to have a full blown Timesketch instance up and running.
Example way of running the tool:
  $ sigma_verify_rules.py --config_file ../data/sigma_config.yaml ../data/linux/
"""

import logging
import os
import argparse
import sys
import codecs

import sigma.parser.exceptions

from sigma.backends import elasticsearch as sigma_elasticsearch
import sigma.configuration as sigma_configuration
from sigma.parser import collection as sigma_collection
logging.basicConfig(level=os.environ.get('LOGLEVEL', 'ERROR'))

def get_codepath():
    """Return the absolute path to where the tool is run from."""
    path = __file__
    if path.startswith(os.path.sep):
        return path

    dirname = os.path.dirname(path)
    for sys_path in sys.path:
        if sys_path.endswith(dirname):
            return sys_path
    return dirname

def run_verifier(rules_path, config_file_path):
    """Run an sigma parsing test on a given dir and returns results from the run.

    Args:
        rules_path: the path to the rules.
        config_file_path: the path to a config file that contains mapping data.
    Raises:
        IOError: if the path to either test or analyzer file does not exist
                 or if the analyzer module or class cannot be loaded.
    Returns:
        a tuple of lists:
            - sigma_verified_rules with rules that can be added
            - sigma_rules_with_problems with rules that should not be added
    """
    if not os.path.isdir(rules_path):
        raise IOError('Rules not found at path: {0:s}'.format(
            rules_path))
    if not os.path.isfile(config_file_path):
        raise IOError('Config file path not found at path: {0:s}'.format(
            config_file_path))

    sigma_config_path = config_file_path

    with open(sigma_config_path, 'r') as sigma_config_file:
        sigma_config_con = sigma_config_file.read()
    sigma_config = sigma_configuration.SigmaConfiguration(sigma_config_con)
    sigma_backend = sigma_elasticsearch.ElasticsearchQuerystringBackend(sigma_config, {})
    sigma_verified_rules = []
    sigma_rules_with_problems = []

    for dirpath, dirnames, files in os.walk(rules_path):

        if 'deprecated' in dirnames:
            dirnames.remove('deprecated')

        rule_extensions = ("yml","yaml")

        for rule_filename in files:
            if rule_filename.lower().endswith(rule_extensions):

                # if a sub dir is found, append it to be scanned for rules
                if os.path.isdir(os.path.join(rules_path, rule_filename)):
                    logging.debug(
                        'This is a directory, skipping: {0:s}'.format(
                            rule_filename))
                    continue

                tag_name, _, _ = rule_filename.rpartition('.')
                rule_file_path = os.path.join(dirpath, rule_filename)
                rule_file_path = os.path.abspath(rule_file_path)
                logging.info('[sigma] Reading rules from {0!s}'.format(
                    rule_file_path))
                with codecs.open(rule_file_path, 'r',
                    encoding="utf-8") as rule_file:
                    try:
                        rule_file_content = rule_file.read()
                        parser = sigma_collection.SigmaCollectionParser(
                            rule_file_content, sigma_config, None)
                        parsed_sigma_rules = parser.generate(sigma_backend)
                    except (NotImplementedError) as exception:
                        logging.error(
                            '{0:s} Error NotImplementedError generating rule in file {1:s}: {1!s}'
                                .format(rule_filename,rule_file_path, exception))

                        sigma_rules_with_problems.append(rule_file_path)
                        continue
                    except (sigma.parser.exceptions.SigmaParseError, TypeError) as exception:
                        logging.error(
                            '{0:s} Error generating rule in file {1:s} '
                            'you should not use this rule in Timesketch: {2!s}'
                                .format(rule_filename,rule_file_path, exception))
                        sigma_rules_with_problems.append(rule_file_path)
                        continue
                    sigma_verified_rules.append(rule_file_path)
    return sigma_verified_rules,sigma_rules_with_problems

if __name__ == '__main__':
    code_path = get_codepath()
    # We want to ensure our mocked libraries get loaded first.
    sys.path.insert(0, code_path)

    description = (
        'Mock an sigma analyzer run. This tool is intended for developers '
        'of sigma rules as well as Timesketch server admins. '
        'The tool can also be used for automatic testing to make sure the '
        'rules are still working as intended.')
    epilog = (
        'Remember to feed the tool with proper rule data.'
    )

    arguments = argparse.ArgumentParser(
        description=description, allow_abbrev=True)
    arguments.add_argument(
        '--config_file', '--file', dest='config_file_path', action='store',
        default='', type=str, metavar='PATH_TO_TEST_FILE', help=(
            'Path to the file containing the config data to feed sigma '
        ))
    arguments.add_argument(
        'rules_path', action='store', default='', type=str,
        metavar='PATH_TO_RULES', help='Path to the rules to test.')

    try:
        options = arguments.parse_args()
    except UnicodeEncodeError:
        print(arguments.format_help())
        sys.exit(1)

    if not os.path.isfile(options.config_file_path):
        print('Config file not found.')
        sys.exit(1)

    if not os.path.isdir(options.rules_path):
        print('The path to the rules does not exist ({0:s})'.format(
            options.rules_path))
        sys.exit(1)

    sigma_verified_rules, sigma_rules_with_problems = run_verifier(
        rules_path=options.rules_path,
        config_file_path=options.config_file_path)

    print('### You should NOT import the following rules ###')
    for badrule in sigma_rules_with_problems:
        print(badrule)

    print('### You can import the following rules ###')
    for goodrule in sigma_verified_rules:
        print(goodrule)

<commit_msg>fix some feedback from reviewer<commit_after># Copyright 2020 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A tool to test sigma rules.
This tool can be used to verify your rules before running an analyzer.
It also does not require you to have a full blown Timesketch instance up and running.
Example way of running the tool:
  $ sigma_verify_rules.py --config_file ../data/sigma_config.yaml ../data/linux/
"""

import logging
import os
import argparse
import sys
import codecs

import sigma.parser.exceptions

from sigma.backends import elasticsearch as sigma_elasticsearch
import sigma.configuration as sigma_configuration
from sigma.parser import collection as sigma_collection
logging.basicConfig(level=os.environ.get('LOGLEVEL', 'ERROR'))

def get_codepath():
    """Return the absolute path to where the tool is run from."""
    path = __file__
    if path.startswith(os.path.sep):
        return path

    dirname = os.path.dirname(path)
    for sys_path in sys.path:
        if sys_path.endswith(dirname):
            return sys_path
    return dirname

def run_verifier(rules_path, config_file_path):
    """Run an sigma parsing test on a given dir and returns results from the run.

    Args:
        rules_path: the path to the rules.
        config_file_path: the path to a config file that contains mapping data.
    Raises:
        IOError: if the path to either test or analyzer file does not exist
                 or if the analyzer module or class cannot be loaded.
    Returns:
        a tuple of lists:
            - sigma_verified_rules with rules that can be added
            - sigma_rules_with_problems with rules that should not be added
    """
    if not os.path.isdir(rules_path):
        raise IOError('Rules not found at path: {0:s}'.format(
            rules_path))
    if not os.path.isfile(config_file_path):
        raise IOError('Config file path not found at path: {0:s}'.format(
            config_file_path))

    sigma_config_path = config_file_path

    with open(sigma_config_path, 'r') as sigma_config_file:
        sigma_config_con = sigma_config_file.read()
    sigma_config = sigma_configuration.SigmaConfiguration(sigma_config_con)
    sigma_backend = sigma_elasticsearch.ElasticsearchQuerystringBackend(sigma_config, {})
    sigma_verified_rules = []
    sigma_rules_with_problems = []

    for dirpath, dirnames, files in os.walk(rules_path):

        if 'deprecated' in dirnames:
            dirnames.remove('deprecated')

        rule_extensions = ("yml","yaml")

        for rule_filename in files:
            if rule_filename.lower().endswith(rule_extensions):

                # if a sub dir is found, append it to be scanned for rules
                if os.path.isdir(os.path.join(rules_path, rule_filename)):
                    logging.debug(
                        'This is a directory, skipping: {0:s}'.format(
                            rule_filename))
                    continue

                tag_name, _, _ = rule_filename.rpartition('.')
                rule_file_path = os.path.join(dirpath, rule_filename)
                rule_file_path = os.path.abspath(rule_file_path)
                logging.debug('[sigma] Reading rules from {0:s}'.format(
                    rule_file_path))
                with codecs.open(rule_file_path, 'r',
                    encoding="utf-8") as rule_file:
                    try:
                        rule_file_content = rule_file.read()
                        parser = sigma_collection.SigmaCollectionParser(
                            rule_file_content, sigma_config, None)
                        parsed_sigma_rules = parser.generate(sigma_backend)
                    except (NotImplementedError) as exception:
                        logging.error(
                            '{0:s} Error NotImplementedError generating rule in file {1:s}: {2!s}'
                                .format(rule_filename,rule_file_path, exception))

                        sigma_rules_with_problems.append(rule_file_path)
                        continue
                    except (sigma.parser.exceptions.SigmaParseError, TypeError) as exception:
                        logging.error(
                            '{0:s} Error generating rule in file {1:s} '
                            'you should not use this rule in Timesketch: {2!s}'
                                .format(rule_filename,rule_file_path, exception))
                        sigma_rules_with_problems.append(rule_file_path)
                        continue
                    sigma_verified_rules.append(rule_file_path)
    return sigma_verified_rules,sigma_rules_with_problems

if __name__ == '__main__':
    code_path = get_codepath()
    # We want to ensure our mocked libraries get loaded first.
    sys.path.insert(0, code_path)

    description = (
        'Mock an sigma analyzer run. This tool is intended for developers '
        'of sigma rules as well as Timesketch server admins. '
        'The tool can also be used for automatic testing to make sure the '
        'rules are still working as intended.')
    epilog = (
        'Remember to feed the tool with proper rule data.'
    )

    arguments = argparse.ArgumentParser(
        description=description, allow_abbrev=True)
    arguments.add_argument(
        '--config_file', '--file', dest='config_file_path', action='store',
        default='', type=str, metavar='PATH_TO_TEST_FILE', help=(
            'Path to the file containing the config data to feed sigma '
        ))
    arguments.add_argument(
        'rules_path', action='store', default='', type=str,
        metavar='PATH_TO_RULES', help='Path to the rules to test.')

    try:
        options = arguments.parse_args()
    except UnicodeEncodeError:
        print(arguments.format_help())
        sys.exit(1)

    if not os.path.isfile(options.config_file_path):
        print('Config file not found.')
        sys.exit(1)

    if not os.path.isdir(options.rules_path):
        print('The path to the rules does not exist ({0:s})'.format(
            options.rules_path))
        sys.exit(1)

    sigma_verified_rules, sigma_rules_with_problems = run_verifier(
        rules_path=options.rules_path,
        config_file_path=options.config_file_path)

    print('### You should NOT import the following rules ###')
    for badrule in sigma_rules_with_problems:
        print(badrule)

    print('### You can import the following rules ###')
    for goodrule in sigma_verified_rules:
        print(goodrule)

<|endoftext|>


<commit_before>
"""
Wood author: thislight

Copyright 2016 thislight

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

Under License Apache v2, more information, see file 'LICENSE' in project root directory.
"""
import tornado.httpserver as _httpserver
import tornado.web as _web
import tornado.ioloop as _ioloop
from .timeit import timeit
import logging
import time
from .utils import functools

BASELOGTEMPLATE = '{method} {httpver} {path} {handler_name} {request_time}s'

class BaseTornadoView(_web.RequestHandler):
    def _get_info(self):
        _r = self.request
        return dict(
        method=_r.method,
        path=_r.path,
        httpver=_r.version,
        cliip=_r.remote_ip,
        p=_r.protocol,
        issec=True if _r.protocol.endswith('s') else False,
        host=_r.host,
        args=_r.arguments,
        request_time=_r.request_time(),
        handler_name=self.__name__,
        )

    def __log__(self):
        info = self._get_info()
        return BASELOGTEMPLATE.format(**info)

class RegisterAllow(object):
    pass

class OverrideObject(object):
    def override(self,name,back=False):
        def override(func):
            setattr(self.handler,name,func)
            if back: return func
        return override

class _PackedView(RegisterAllow,OverrideObject):
    def __init__(self,view,uri='/'):
        self._view = view
        self.__name__ = self._view.__name__ + '_Packed'
        self._uri = uri

    @property
    def get(self):
        return self.override('get',back=True)

    @property
    def post(self):
        return self.override('post',back=True)

    @property
    def head(self):
        return self.override('head',back=True)

    @property
    def put(self):
        return self.override('put',back=True)

    @property
    def delete(self):
        return self.override('delete',back=True)

    @property
    def patch(self):
        return self.override('patch',back=True)

    @property
    def options(self):
        return self.override('options',back=True)

    @property
    def handler(self):
        return self._view

    @handler.setter
    def handler(self,value):
        self._view = value

    @property
    def uri(self):
        return self._uri

def _make_empty_view(name='View',uri,*parents):
    """
    a help function for make a empty view.
    Return: _PackedView
    """
    view = type(name,(BaseTornadoView,*parent),{})
    _packed = _PackedView(view,uri=uri)
    return _packed

def _make_uri_tuple(uri,handler,kargs=None):
    t = [uri,handler]
    if kargs: t.append(kargs)
    return tuple(t)

def _print_and_log(logger=logging.getLogger(),*args):
    print(*args)
    logger.info(*args)

def base_log_function(o):
    if hasattr(o,'__log__'):#
        _print_and_log(o.__log__)
    elif isinstance(o,_web.ErrorHandler):
        _print_and_log(o)# TODO: More simple and useful infomation
    else:
        _print_and_log(str(o))

class Wood(object):
    def __init__(self,name=__name__,**config):
        self._app_settings = config
        self._name = name
        self.prepare_funcs = []
        if 'log_function' not in self._app_settings:
            self._app_settings['log_function'] = base_log_function
        self._bind_ports = []
        self._ui_mods = []

    @property
    def server(self):
        if not self._server:
            self._server = _httpserver.HTTPServer(self.application,xheaders=True)
        return self._server

    @property
    def application(self):
        if not self._app:
            self._app = _web.Application(**self._app_settings)
        return self._app

    def handler(self,uri,handler,host='',**kargs):
        self.application.add_handlers(host,[_make_uri_tuple(uri,handler,kargs)])

    def empty(self,uri,name,*parents):
        """
        Return: _PackedView
        """
        v = _make_empty_view(uri=uri,name=name,*parents)
        self.register(v)
        return v

    def route(self,uri,method='get',*parents,**kargs):
        """
        Route a function to uri.
        arg method: method for function
        Return: function
        """
        def route(f):
            """
            Return: function
            """
            view = self.empty(uri,f.__name__,*parents)
            method = method.lower()
            return view.override(method)(f,back=True)
        return route

    def register(self,view):
        if isinstance(view,RegisterAllow):
             self.handler(uri=view.uri,handler=view.handler)

    def register_all(self,g):
        for k in g:
            o = g[k]
            if isinstance(o,RegisterAllow):
                self.register(o)

    def _bind(self):
        """
        Bind all of ports in list self._bind_ports
        """
        for p in self._bind_ports:
            self.server.bind(p)

    def bind(self,port):
        self._bind_ports.append(port)

    def prepare(self,func):
        self.prepare_funcs.append(func)
        return func

    def call_prepare(self):
        for f in self.prepare_funcs:
            f(self)

    def _start(self):
        self.call_prepare()
        self.ioloop.start()

    def start(self,port=None,wokers=None):
        if port: self.bind(port)
        self._bind()# Bind before start server.
        if not wokers:
            self.server.start()
        else:
            self.server.start(wokers)
        self._start()

    @property
    def ioloop(self):
        return _ioloop.IOLoop.current()

# The end of the file
<commit_msg>ADD PackedUIModule & Remove import timeit & ADD auto make ui_module dict in app's settings.<commit_after>
"""
Wood author: thislight

Copyright 2016 thislight

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

Under License Apache v2, more information, see file 'LICENSE' in project root directory.
"""
import tornado.httpserver as _httpserver
import tornado.web as _web
import tornado.ioloop as _ioloop
import logging
import time
from .utils import functools

BASELOGTEMPLATE = '{method} {httpver} {path} {handler_name} {request_time}s'

class BaseTornadoView(_web.RequestHandler):
    def _get_info(self):
        _r = self.request
        return dict(
        method=_r.method,
        path=_r.path,
        httpver=_r.version,
        cliip=_r.remote_ip,
        p=_r.protocol,
        issec=True if _r.protocol.endswith('s') else False,
        host=_r.host,
        args=_r.arguments,
        request_time=_r.request_time(),
        handler_name=self.__name__,
        )

    def __log__(self):
        info = self._get_info()
        return BASELOGTEMPLATE.format(**info)

class RegisterAllow(object):
    pass

class OverrideObject(object):
    def override(self,name,back=False):
        def override(func):
            setattr(self.handler,name,func)
            if back: return func
        return override

class _PackedView(RegisterAllow,OverrideObject):
    def __init__(self,view,uri='/'):
        self._view = view
        self.__name__ = self._view.__name__ + '_Packed'
        self._uri = uri

    @property
    def get(self):
        return self.override('get',back=True)

    @property
    def post(self):
        return self.override('post',back=True)

    @property
    def head(self):
        return self.override('head',back=True)

    @property
    def put(self):
        return self.override('put',back=True)

    @property
    def delete(self):
        return self.override('delete',back=True)

    @property
    def patch(self):
        return self.override('patch',back=True)

    @property
    def options(self):
        return self.override('options',back=True)

    @property
    def handler(self):
        return self._view

    @handler.setter
    def handler(self,value):
        self._view = value

    @property
    def uri(self):
        return self._uri

class PackedUIModlue(_web.UIModule):
    def render(self,*args,**kargs):
        result = functools.call_or_not(self._dyncall,self,*args,**kargs)
        if result:
            self.write(result)

    def js(self,code):
        self._jscode = code
        return self

    def jsfile(self,*paths):
        if not self._jsfiles:
            self._jsfiles = []
        for v in paths:
            if v not in self._jsfiles:
                self._jsfiles.append(v)
        return self

    def css(self,code):
        self._csscode = code
        return self

    def cssfile(self,*paths):
        if not self._cssfiles:
            self._cssfiles = []
        for v in paths:
            if v not in self._cssfiles:
                self._cssfiles.append(v)
        return self

    def embedded_javascript(self):
        return self._jscode

    def embedded_css(self):
        return self._csscode

    def javascript_files(self):
        return self._jsfiles

    def css_files(self):
        return self._cssfiles

    def prepare(self,f):
        f(self)
        return f

    def dynrender(self,f):
        self._dyncall = f
        return f

def _make_empty_view(name='View',uri,*parents):
    """
    a help function for make a empty view.
    Return: _PackedView
    """
    view = type(name,(BaseTornadoView,*parent),{})
    _packed = _PackedView(view,uri=uri)
    return _packed

def _make_uri_tuple(uri,handler,kargs=None):
    t = [uri,handler]
    if kargs: t.append(kargs)
    return tuple(t)

def _print_and_log(logger=logging.getLogger(),*args):
    print(*args)
    logger.info(*args)

def base_log_function(o):
    if hasattr(o,'__log__'):#
        _print_and_log(o.__log__)
    elif isinstance(o,_web.ErrorHandler):
        _print_and_log(o)# TODO: More simple and useful infomation
    else:
        _print_and_log(str(o))

class Wood(object):
    def __init__(self,name=__name__,**config):
        self._app_settings = config
        self._name = name
        self.prepare_funcs = []
        if 'log_function' not in self._app_settings:
            self._app_settings['log_function'] = base_log_function
        self._bind_ports = []
        self._ui_mods = []
        if not self._app_settings['ui_modules']:
            self._app_settings['ui_modules'] = {}

    @property
    def server(self):
        if not self._server:
            self._server = _httpserver.HTTPServer(self.application,xheaders=True)
        return self._server

    @property
    def application(self):
        if not self._app:
            self._app = _web.Application(**self._app_settings)
        return self._app

    def handler(self,uri,handler,host='',**kargs):
        self.application.add_handlers(host,[_make_uri_tuple(uri,handler,kargs)])

    def empty(self,uri,name,*parents):
        """
        Return: _PackedView
        """
        v = _make_empty_view(uri=uri,name=name,*parents)
        self.register(v)
        return v

    def route(self,uri,method='get',*parents,**kargs):
        """
        Route a function to uri.
        arg method: method for function
        Return: function
        """
        def route(f):
            """
            Return: function
            """
            view = self.empty(uri,f.__name__,*parents)
            method = method.lower()
            return view.override(method)(f,back=True)
        return route

    def register(self,view):
        if isinstance(view,RegisterAllow):
             self.handler(uri=view.uri,handler=view.handler)

    def register_all(self,g):
        for k in g:
            o = g[k]
            if isinstance(o,RegisterAllow):
                self.register(o)

    def _bind(self):
        """
        Bind all of ports in list self._bind_ports
        """
        for p in self._bind_ports:
            self.server.bind(p)

    def bind(self,port):
        self._bind_ports.append(port)

    def prepare(self,func):
        self.prepare_funcs.append(func)
        return func

    def call_prepare(self):
        for f in self.prepare_funcs:
            f(self)

    def _start(self):
        self.call_prepare()
        self.ioloop.start()

    def start(self,port=None,wokers=None):
        if port: self.bind(port)
        self._bind()# Bind before start server.
        if not wokers:
            self.server.start()
        else:
            self.server.start(wokers)
        self._start()

    @property
    def ioloop(self):
        return _ioloop.IOLoop.current()

# The end of the file<|endoftext|>


<commit_before>
from __future__ import absolute_import

import argparse
from datetime import datetime
import boto3
import logging
import Queue
import sys
import time
import threading
import uuid

import strongfellowbtc.hex
from strongfellowbtc.protocol import ds256
import strongfellowbtc.zmq
from strongfellowbtc.logging import configure_logging

def k(region):
    return boto3.client('kinesis', region_name=region)

def _stream_name(region, env, host):
    return 'strongfellow-{region}-{env}-{host}'.format(region=region, env=env, host=host)

def little_endian_long(n):
    bs = bytearray(8)
    i = 0
    while n != 0:
        bs[i] = n & 0xff
        n = (n >> 8)
        i += 1
    return bytes(bs)

def create_stream(args=None):
    configure_logging()
    if args is None:
        args = sys.argv[1:]
    parser = argparse.ArgumentParser()
    parser.add_argument('--region', required=True)
    parser.add_argument('--env', required=True)
    parser.add_argument('--host', required=True)
    parser.add_argument('--shard-count', default='1', type=int)
    params = parser.parse_args(args)

    kinesis = k(params.region)

    stream_name = _stream_name(region=params.region, env=params.env, host=params.host)
    shard_count = params.shard_count

    logging.info('creating stream %s with shard count %d', stream_name, shard_count)
    response = kinesis.create_stream(
        StreamName=stream_name,
        ShardCount=shard_count
    )
    logging.info('success: created stream %s with shard count %d', stream_name, shard_count)

def stream_incoming_transactions(args=None):
    configure_logging()
    if args is None:
        args = sys.argv[1:]

    parser = argparse.ArgumentParser()
    parser.add_argument('--maxsize', type=int, default='300')
    parser.add_argument('--txport', type=int, default='28332')
    parser.add_argument('--region', required=True)
    parser.add_argument('--env', required=True)
    parser.add_argument('--host', required=True)
    args = parser.parse_args(args)

    q = Queue.Queue(maxsize=args.maxsize)

    def produce(q):
        with strongfellowbtc.zmq.socket(port=args.txport, topic='rawtx') as socket:
            while True:
                topic, tx = socket.recv_multipart()
                delta = datetime.now() - datetime(1970, 1, 1)
                ms = long(delta.total_seconds() * 1000)
                try:
                    q.put_nowait((ms, tx))
                except Queue.Full:
                    logging.exception('Queue is Full: we cant put %s' % strongfellowbtc.hex.little_endian_hex(ds256(tx)))

    def consume(q):
        kinesis = k(region=args.region)
        stream_name = _stream_name(region=args.region, env=args.env, host=args.host)
        while True:
            if q.empty():
               time.sleep(1)
               logging.info('no transactions, sleeping for a second')
            else:
                records = []
                n = q.qsize()
                logging.info('%d transactions equeued', n)
                while len(records) < n:
                    ms, tx = q.get_nowait()
                    record = {
                        'Data': little_endian_long(ms) + tx,
                        'PartitionKey': str(uuid.uuid4())
                    }
                    records.append(record)
                try:
                    response = kinesis.put_records(
                        Records=records,
                        StreamName=stream_name
                    )
                    logging.info('response was: %s', response)
                    logging.info('SUCCESS putting records')
                except:
                    logging.exception('problem putting records')
                    time.sleep(3)

    t1 = threading.Thread(target=produce, args=(q,))
    t2 = threading.Thread(target=consume, args=(q,))

    t1.start()
    t2.start()
    logging.info('join us, wont you?')
    t1.join()
    t2.join()

def test_get_records(args = None):
    configure_logging()
    if args is None:
        args = sys.argv[1:]
    parser = argparse.ArgumentParser()
    parser.add_argument('--region', required=True)
    parser.add_argument('--env', required=True)
    parser.add_argument('--host', required=True)
    params = parser.parse_args(args)

    kinesis = k(region=params.region)
    stream_name = _stream_name(region=params.region, env=params.env, host=params.host)
    shard_id = 'shardId-000000000000'
    shard_iterator = kinesis.get_shard_iterator(StreamName=stream_name, ShardId=shard_id, ShardIteratorType="LATEST")['ShardIterator']
    while True:
        response = kinesis.get_records(ShardIterator=shard_iterator, Limit=1000)
        shard_iterator = response['NextShardIterator']
        for record in response['Records']:
            print strongfellowbtc.hex.big_endian_hex(ds256(record['Data'][8:]))
        print response
        time.sleep(1)

<commit_msg>correct default port<commit_after>
from __future__ import absolute_import

import argparse
from datetime import datetime
import boto3
import logging
import Queue
import sys
import time
import threading
import uuid

import strongfellowbtc.hex
from strongfellowbtc.protocol import ds256
import strongfellowbtc.zmq
from strongfellowbtc.logging import configure_logging

def k(region):
    return boto3.client('kinesis', region_name=region)

def _stream_name(region, env, host):
    return 'strongfellow-{region}-{env}-{host}'.format(region=region, env=env, host=host)

def little_endian_long(n):
    bs = bytearray(8)
    i = 0
    while n != 0:
        bs[i] = n & 0xff
        n = (n >> 8)
        i += 1
    return bytes(bs)

def create_stream(args=None):
    configure_logging()
    if args is None:
        args = sys.argv[1:]
    parser = argparse.ArgumentParser()
    parser.add_argument('--region', required=True)
    parser.add_argument('--env', required=True)
    parser.add_argument('--host', required=True)
    parser.add_argument('--shard-count', default='1', type=int)
    params = parser.parse_args(args)

    kinesis = k(params.region)

    stream_name = _stream_name(region=params.region, env=params.env, host=params.host)
    shard_count = params.shard_count

    logging.info('creating stream %s with shard count %d', stream_name, shard_count)
    response = kinesis.create_stream(
        StreamName=stream_name,
        ShardCount=shard_count
    )
    logging.info('success: created stream %s with shard count %d', stream_name, shard_count)

def stream_incoming_transactions(args=None):
    configure_logging()
    if args is None:
        args = sys.argv[1:]

    parser = argparse.ArgumentParser()
    parser.add_argument('--maxsize', type=int, default='300')
    parser.add_argument('--txport', type=int, default='28334')
    parser.add_argument('--region', required=True)
    parser.add_argument('--env', required=True)
    parser.add_argument('--host', required=True)
    args = parser.parse_args(args)

    q = Queue.Queue(maxsize=args.maxsize)

    def produce(q):
        with strongfellowbtc.zmq.socket(port=args.txport, topic='rawtx') as socket:
            while True:
                topic, tx = socket.recv_multipart()
                delta = datetime.now() - datetime(1970, 1, 1)
                ms = long(delta.total_seconds() * 1000)
                try:
                    q.put_nowait((ms, tx))
                except Queue.Full:
                    logging.exception('Queue is Full: we cant put %s' % strongfellowbtc.hex.little_endian_hex(ds256(tx)))

    def consume(q):
        kinesis = k(region=args.region)
        stream_name = _stream_name(region=args.region, env=args.env, host=args.host)
        while True:
            if q.empty():
               time.sleep(1)
               logging.info('no transactions, sleeping for a second')
            else:
                records = []
                n = q.qsize()
                logging.info('%d transactions equeued', n)
                while len(records) < n:
                    ms, tx = q.get_nowait()
                    record = {
                        'Data': little_endian_long(ms) + tx,
                        'PartitionKey': str(uuid.uuid4())
                    }
                    records.append(record)
                try:
                    response = kinesis.put_records(
                        Records=records,
                        StreamName=stream_name
                    )
                    logging.info('response was: %s', response)
                    logging.info('SUCCESS putting records')
                except:
                    logging.exception('problem putting records')
                    time.sleep(3)

    t1 = threading.Thread(target=produce, args=(q,))
    t2 = threading.Thread(target=consume, args=(q,))

    t1.start()
    t2.start()
    logging.info('join us, wont you?')
    t1.join()
    t2.join()

def test_get_records(args = None):
    configure_logging()
    if args is None:
        args = sys.argv[1:]
    parser = argparse.ArgumentParser()
    parser.add_argument('--region', required=True)
    parser.add_argument('--env', required=True)
    parser.add_argument('--host', required=True)
    params = parser.parse_args(args)

    kinesis = k(region=params.region)
    stream_name = _stream_name(region=params.region, env=params.env, host=params.host)
    shard_id = 'shardId-000000000000'
    shard_iterator = kinesis.get_shard_iterator(StreamName=stream_name, ShardId=shard_id, ShardIteratorType="LATEST")['ShardIterator']
    while True:
        response = kinesis.get_records(ShardIterator=shard_iterator, Limit=1000)
        shard_iterator = response['NextShardIterator']
        for record in response['Records']:
            print strongfellowbtc.hex.big_endian_hex(ds256(record['Data'][8:]))
        print response
        time.sleep(1)

<|endoftext|>


<commit_before># encoding: utf-8
'''
Created on 2014/9/23

@author: kepler-moment
'''
from editDistance import editDistance
from BK_Tree import BK_Tree

class Node:
    """Used as node of Trie"""

    def __init__(self,numofchar):
        self._isWord = False
        self.NUMOFCHAR = numofchar
        self.value = None
        self.next = [None] * self.NUMOFCHAR

    def setWord(self,value):
        self._isWord = True
        self.value = value

    def isWord(self):
        return self._isWord

    def getValue(self):
        return self.value

class Trie:
    """Trie tree"""
    NUMOFLOW = 26
    def __init__(self,dict_map):
        #the root of word which is converted to lower
        self.rootOfLow = Node(26)
        #the root of work which has capical
        self.rootOfCapital = Node(self.NUMOFLOW * 2)
        for (key,value) in dict_map.items():
            self.insert(key,value)

    def hasCapital(self,key):
        for i in range(len(key)):
            if key[i].isupper():
                return True
        return False

    def insert(self,key,value):
        """insert key => value"""
        lenOfkey = len(key)

        """if there is capital in key,insert it into rootOfCapital"""

        if self.hasCapital(key):
            root = self.rootOfCapital
            for i in range(lenOfkey):
                if key[i].isupper():
                    tmp = ord(key[i]) - ord('A') + self.NUMOFLOW
                else:
                    tmp = ord(key[i]) - ord('a')
                #print "hasC %s " % key
                if root.next[tmp] is None:
                    newNode = Node(self.NUMOFLOW * 2)
                    root.next[tmp] = newNode
                root = root.next[tmp]
            root.setWord(value)
        else:
            lower_word = key.lower()
            root = self.rootOfLow
            for i in range(lenOfkey):
                tmp = ord(lower_word[i]) - ord('a')
                #print tmp
                #print lower_word
                if root.next[tmp] == None:
                    newNode = Node(self.NUMOFLOW)
                    root.next[tmp] = newNode
                root = root.next[tmp]
            root.setWord(value)

    def search(self,key):
        d = {}
        low_word = key.lower()
        root = self.rootOfLow
        for i in range(len(key)):
            tmp = ord(low_word[i]) - ord('a')
            if root.next[tmp] is None:
                break
            root = root.next[tmp]
        if root != None and root.isWord() is True:
            d[low_word] = root.getValue()
        if self.hasCapital(key):
            root = self.rootOfCapital
            for i in range(len(key)):
                if key[i].isupper():
                    tmp = ord(key[i]) - ord('A') + self.NUMOFLOW
                else:
                    tmp = ord(key[i]) - ord('a')
                if root.next[tmp] is None:
                    break
                root = root.next[tmp]
            if root != None and root.isWord() is True:
                d[key] = root.getValue()
        return d

class Dictionary:
    """"""
    def __init__(self,dict_map):
        """
        param dict_map is a map of all words which likes
        dict_map['english'] = 'chinese'
        """
        self.dict_trie = Trie(dict_map)
        self.dict_map = dict_map;

    def search(self,key):
        """search word in dict"""
        return self.dict_trie.search(key)

    def editDistanceTopK(self,key,k,distance):

        def cmp1(word1,word2):
            """"sort similarWord by distance"""
            if word1['distance'] < word2['distance']:
                return -1
            return 1

        assert k >= 1
        assert distance >= 0
        similarWord = []
        for (kt,va) in self.dict_map.items():
            d = editDistance(key,kt)
            if d <= distance:
                similarWord.append({})
                similarWord[len(similarWord) - 1] = {kt:va,"distance":d};
        similarWord.sort(cmp1)
        return similarWord[0:k]

def getDictMap(path):
    fp = open(path,"r")
    words = {}
    for line in fp.readlines():
        word = line.split(":",1)
        words[word[0]] = word[1]
        #print word[0]
    return words

if __name__=='__main__':
    #print timeit.timeit("search('Will')", "from __main__ import search", timeit.default_timer, 1)
    #print timeit.timeit("binary_search('Will')", "from __main__ import binary_search", timeit.default_timer, 1)
    #dictionary = Dictionary(getDictMap("./e-c"))
    bk = BK_Tree()
    bk.insert("656")
    bk.insert("67")
    bk.insert("9312")
    bk.insert("1178")
    bk.insert("38")
    print bk.topKSimilar("87", 5, 1)
    print bk.topKSimilar("6", 5, 3)
    <commit_msg>delete search.py<commit_after><|endoftext|>