<commit_before>import socket
import ssl

from frame import receive_frame
from handshake import ServerHandshake, ClientHandshake
from errors import SSLError

class websocket(object):
    """
    Implementation of web socket, upgrades a regular TCP socket to a websocket
    using the HTTP handshakes and frame (un)packing, as specified by RFC 6455.
    The API of a websocket is identical to that of a regular socket, as
    illustrated by the examples below.

    Server example:
    >>> import twspy, socket
    >>> sock = twspy.websocket()
    >>> sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    >>> sock.bind(('', 8000))
    >>> sock.listen()

    >>> client = sock.accept()
    >>> client.send(twspy.Frame(twspy.OPCODE_TEXT, 'Hello, Client!'))
    >>> frame = client.recv()

    Client example:
    >>> import twspy
    >>> sock = twspy.websocket(location='/my/path')
    >>> sock.connect(('', 8000))
    >>> sock.send(twspy.Frame(twspy.OPCODE_TEXT, 'Hello, Server!'))
    """
    def __init__(self, sock=None, protocols=[], extensions=[], origin=None,
                 trusted_origins=[], location='/', auth=None,
                 sfamily=socket.AF_INET, sproto=0):
        """
        Create a regular TCP socket of family `family` and protocol

        `sock` is an optional regular TCP socket to be used for sending binary
        data. If not specified, a new socket is created.

        `protocols` is a list of supported protocol names.

        `extensions` is a list of supported extensions (`Extension` instances).

        `origin` (for client sockets) is the value for the "Origin" header sent
        in a client handshake .

        `trusted_origins` (for servere sockets) is a list of expected values
        for the "Origin" header sent by a client. If the received Origin header
        has value not in this list, a HandshakeError is raised. If the list is
        empty (default), all origins are excepted.

        `location` is optional, used for the HTTP handshake. In a URL, this
        would show as ws://host[:port]/path.

        `auth` is optional, used for HTTP Basic or Digest authentication during
        the handshake. It must be specified as a (username, password) tuple.

        `sfamily` and `sproto` are used for the regular socket constructor.
        """
        self.protocols = protocols
        self.extensions = extensions
        self.origin = origin
        self.trusted_origins = trusted_origins
        self.location = location
        self.auth = auth
        self.sock = sock or socket.socket(sfamily, socket.SOCK_STREAM, sproto)
        self.secure = False
        self.handshake_sent = False
        self.hooks_send = []
        self.hooks_recv = []

    def bind(self, address):
        self.sock.bind(address)

    def listen(self, backlog):
        self.sock.listen(backlog)

    def accept(self):
        """
        Equivalent to socket.accept(), but transforms the socket into a
        websocket instance and sends a server handshake (after receiving a
        client handshake). Note that the handshake may raise a HandshakeError
        exception.
        """
        sock, address = self.sock.accept()
        wsock = websocket(sock)
        ServerHandshake(wsock).perform()
        wsock.handshake_sent = True
        return wsock, address

    def connect(self, address):
        """
        Equivalent to socket.connect(), but sends an client handshake request
        after connecting.

        `address` is a (host, port) tuple of the server to connect to.
        """
        self.sock.connect(address)
        ClientHandshake(self).perform()
        self.handshake_sent = True

    def send(self, *args):
        """
        Send a number of frames.
        """
        for frame in args:
            for hook in self.hooks_send:
                frame = hook(frame)

            #print 'send frame:', frame, 'to %s:%d' % self.sock.getpeername()
            self.sock.sendall(frame.pack())

    def recv(self):
        """
        Receive a single frames. This can be either a data frame or a control
        frame.
        """
        frame = receive_frame(self.sock)

        for hook in self.hooks_recv:
            frame = hook(frame)

        #print 'receive frame:', frame, 'from %s:%d' % self.sock.getpeername()
        return frame

    def recvn(self, n):
        """
        Receive exactly `n` frames. These can be either data frames or control
        frames, or a combination of both.
        """
        return [self.recv() for i in xrange(n)]

    def getpeername(self):
        return self.sock.getpeername()

    def getsockname(self):
        return self.sock.getsockname()

    def setsockopt(self, level, optname, value):
        self.sock.setsockopt(level, optname, value)

    def getsockopt(self, level, optname):
        return self.sock.getsockopt(level, optname)

    def close(self):
        self.sock.close()

    def enable_ssl(self, *args, **kwargs):
        """
        Transforms the regular socket.socket to an ssl.SSLSocket for secure
        connections. Any arguments are passed to ssl.wrap_socket:
        http://docs.python.org/dev/library/ssl.html#ssl.wrap_socket
        """
        if self.handshake_sent:
            raise SSLError('can only enable SSL before handshake')

        self.secure = True
        self.sock = ssl.wrap_socket(self.sock, *args, **kwargs)

    def add_hook(self, send=None, recv=None):
        if send:
            self.hooks_send.append(send)

        if recv:
            self.hooks_recv.insert(0, recv)
<commit_msg>Bugfix & added some docs<commit_after>import socket
import ssl

from frame import receive_frame
from handshake import ServerHandshake, ClientHandshake
from errors import SSLError

class websocket(object):
    """
    Implementation of web socket, upgrades a regular TCP socket to a websocket
    using the HTTP handshakes and frame (un)packing, as specified by RFC 6455.
    The API of a websocket is identical to that of a regular socket, as
    illustrated by the examples below.

    Server example:
    >>> import twspy, socket
    >>> sock = twspy.websocket()
    >>> sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    >>> sock.bind(('', 8000))
    >>> sock.listen()

    >>> client = sock.accept()
    >>> client.send(twspy.Frame(twspy.OPCODE_TEXT, 'Hello, Client!'))
    >>> frame = client.recv()

    Client example:
    >>> import twspy
    >>> sock = twspy.websocket(location='/my/path')
    >>> sock.connect(('', 8000))
    >>> sock.send(twspy.Frame(twspy.OPCODE_TEXT, 'Hello, Server!'))
    """
    def __init__(self, sock=None, protocols=[], extensions=[], origin=None,
                 trusted_origins=[], location='/', auth=None,
                 sfamily=socket.AF_INET, sproto=0):
        """
        Create a regular TCP socket of family `family` and protocol

        `sock` is an optional regular TCP socket to be used for sending binary
        data. If not specified, a new socket is created.

        `protocols` is a list of supported protocol names.

        `extensions` is a list of supported extensions (`Extension` instances).

        `origin` (for client sockets) is the value for the "Origin" header sent
        in a client handshake .

        `trusted_origins` (for servere sockets) is a list of expected values
        for the "Origin" header sent by a client. If the received Origin header
        has value not in this list, a HandshakeError is raised. If the list is
        empty (default), all origins are excepted.

        `location` is optional, used for the HTTP handshake. In a URL, this
        would show as ws://host[:port]/path.

        `auth` is optional, used for HTTP Basic or Digest authentication during
        the handshake. It must be specified as a (username, password) tuple.

        `sfamily` and `sproto` are used for the regular socket constructor.
        """
        self.protocols = protocols
        self.extensions = extensions
        self.origin = origin
        self.trusted_origins = trusted_origins
        self.location = location
        self.auth = auth
        self.sock = sock or socket.socket(sfamily, socket.SOCK_STREAM, sproto)
        self.secure = False
        self.handshake_sent = False
        self.hooks_send = []
        self.hooks_recv = []

    def bind(self, address):
        self.sock.bind(address)

    def listen(self, backlog):
        self.sock.listen(backlog)

    def accept(self):
        """
        Equivalent to socket.accept(), but transforms the socket into a
        websocket instance and sends a server handshake (after receiving a
        client handshake). Note that the handshake may raise a HandshakeError
        exception.
        """
        sock, address = self.sock.accept()
        wsock = websocket(sock)
        ServerHandshake(wsock).perform()
        wsock.handshake_sent = True
        return wsock, address

    def connect(self, address):
        """
        Equivalent to socket.connect(), but sends an client handshake request
        after connecting.

        `address` is a (host, port) tuple of the server to connect to.
        """
        self.sock.connect(address)
        ClientHandshake(self).perform()
        self.handshake_sent = True

    def send(self, *args):
        """
        Send a number of frames.
        """
        for frame in args:
            for hook in self.hooks_send:
                frame = hook(frame)

            #print 'send frame:', frame, 'to %s:%d' % self.sock.getpeername()
            self.sock.sendall(frame.pack())

    def recv(self):
        """
        Receive a single frames. This can be either a data frame or a control
        frame.
        """
        frame = receive_frame(self.sock)

        for hook in self.hooks_recv:
            frame = hook(frame)

        #print 'receive frame:', frame, 'from %s:%d' % self.sock.getpeername()
        return frame

    def recvn(self, n):
        """
        Receive exactly `n` frames. These can be either data frames or control
        frames, or a combination of both.
        """
        return [self.recv() for i in xrange(n)]

    def getpeername(self):
        return self.sock.getpeername()

    def getsockname(self):
        return self.sock.getsockname()

    def setsockopt(self, level, optname, value):
        self.sock.setsockopt(level, optname, value)

    def getsockopt(self, level, optname):
        return self.sock.getsockopt(level, optname)

    def close(self):
        self.sock.close()

    def enable_ssl(self, *args, **kwargs):
        """
        Transforms the regular socket.socket to an ssl.SSLSocket for secure
        connections. Any arguments are passed to ssl.wrap_socket:
        http://docs.python.org/dev/library/ssl.html#ssl.wrap_socket
        """
        if self.handshake_sent:
            raise SSLError('can only enable SSL before handshake')

        self.secure = True
        self.sock = ssl.wrap_socket(self.sock, *args, **kwargs)

    def add_hook(self, send=None, recv=None, prepend=False):
        """
        Add a pair of send and receive hooks that are called for each frame
        that is sent or received. A hook is a function that receives a single
        argument - a Frame instance - and returns a `Frame` instance as well.

        `prepend` is a flag indicating whether the `send` hook

        For example, the following code creates a `Frame` instance for data
        being sent and removes the instance for received data. This way, data
        can be sent and received as if on a regular socket.
        >>> import twspy
        >>> sock.add_hook(lambda data: tswpy.Frame(tswpy.OPCODE_TEXT, data),
        >>>               lambda f: f.data)

        To add base64 encoding to the example above:
        >>> import base64
        >>> sock.add_hook(base64.encodestring, base64.decodestring, True)

        Note that here `prepend=True`, so that data passed to `send()` is first
        encoded and then packed into a frame. Of course, one could also decide
        to add the base64 hook first, or to return a new `Frame` instance with
        base64-encoded data.
        """
        if send:
            self.hooks_send.insert(0 if prepend else -1, send)

        if recv:
            self.hooks_recv.insert(-1 if prepend else 0, recv)
<|endoftext|>


<commit_before>#########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  * See the License for the specific language governing permissions and
#  * limitations under the License.

import time

from cloudify import utils
from cloudify import amqp_client
from cloudify import constants
from cloudify.celery import celery as celery_client
from cloudify.decorators import operation
from cloudify.exceptions import NonRecoverableError

from windows_agent_installer import constants as win_constants
from windows_agent_installer import init_worker_installer
from windows_agent_installer import constants as win_const

# This is the folder under which the agent is
# extracted to inside the current directory.
# It is set in the packaging process so it
# must be hardcoded here.
AGENT_FOLDER_NAME = 'CloudifyAgent'

# This is where we download the agent to.
AGENT_EXEC_FILE_NAME = 'CloudifyAgent.exe'

# nssm will install celery and use this name to identify the service
AGENT_SERVICE_NAME = 'CloudifyAgent'

# location of the agent package on the management machine,
# relative to the file server root.
AGENT_PACKAGE_PATH = 'packages/agents/CloudifyWindowsAgent.exe'

# Path to the agent. We are using global (not user based) paths
# because of virtualenv relocation issues on windows.
RUNTIME_AGENT_PATH = 'C:\CloudifyAgent'

# Agent includes list, Mandatory
AGENT_INCLUDES = 'script_runner.tasks,windows_plugin_installer.tasks,'\
                 'cloudify.plugins.workflows'

def get_agent_package_url():
    return '{0}/{1}'.format(utils.get_manager_file_server_url(),
                            AGENT_PACKAGE_PATH)

def create_env_string(cloudify_agent):
    env = {
        constants.CELERY_WORK_DIR_PATH_KEY:
        RUNTIME_AGENT_PATH,
        constants.LOCAL_IP_KEY:
        cloudify_agent['host'],
        constants.MANAGER_IP_KEY:
        utils.get_manager_ip(),
        constants.MANAGER_FILE_SERVER_BLUEPRINTS_ROOT_URL_KEY:
        utils.get_manager_file_server_blueprints_root_url(),
        constants.MANAGER_FILE_SERVER_URL_KEY:
        utils.get_manager_file_server_url(),
        constants.MANAGER_REST_PORT_KEY:
        utils.get_manager_rest_service_port()
    }
    env_string = ''
    for key, value in env.iteritems():
        env_string = '{0} {1}={2}' \
            .format(env_string, key, value)
    return env_string.strip()

@operation
@init_worker_installer
def install(ctx, runner=None, cloudify_agent=None, **kwargs):

    """
    Installs the cloudify agent service on the machine.
    The agent installation consists of the following:

        1. Download and extract necessary files.
        2. Configure the agent service to auto start on vm launch.
        3. Configure the agent service to restart on failure.

    :param ctx: Invocation context - injected by the @operation
    :param runner: Injected by the @init_worker_installer
    :param cloudify_agent: Injected by the @init_worker_installer
    """

    if cloudify_agent.get('delete_amqp_queues'):
        _delete_amqp_queues(cloudify_agent['name'])

    ctx.logger.info('Installing agent {0}'.format(cloudify_agent['name']))

    agent_exec_path = 'C:\{0}'.format(AGENT_EXEC_FILE_NAME)

    runner.download(get_agent_package_url(), agent_exec_path)
    ctx.logger.debug('Extracting agent to C:\\ ...')

    runner.run('{0} -o{1} -y'.format(agent_exec_path, RUNTIME_AGENT_PATH),
               quiet=True)

    params = ('--broker=amqp://guest:guest@{0}:5672// '
              '-Ofair '
              '--events '
              '--app=cloudify '
              '-Q {1} '
              '--hostname={1} '
              '--logfile={2}\celery.log '
              '--pidfile={2}\celery.pid '
              '--autoscale={3},{4} '
              '--include={5} '
              '--without-gossip '
              '--without-mingle '
              .format(utils.get_manager_ip(),
                      cloudify_agent['name'],
                      RUNTIME_AGENT_PATH,
                      cloudify_agent[win_constants.MIN_WORKERS_KEY],
                      cloudify_agent[win_constants.MAX_WORKERS_KEY],
                      AGENT_INCLUDES))
    runner.run('{0}\\nssm\\nssm.exe install {1} {0}\Scripts\celeryd.exe {2}'
               .format(RUNTIME_AGENT_PATH, AGENT_SERVICE_NAME, params))
    env = create_env_string(cloudify_agent)
    runner.run('{0}\\nssm\\nssm.exe set {1} AppEnvironmentExtra {2}'
               .format(RUNTIME_AGENT_PATH, AGENT_SERVICE_NAME, env))
    runner.run('sc config {0} start= auto'.format(AGENT_SERVICE_NAME))
    runner.run(
        'sc failure {0} reset= {1} actions= restart/{2}'.format(
            AGENT_SERVICE_NAME,
            cloudify_agent['service'][
                win_const.SERVICE_FAILURE_RESET_TIMEOUT_KEY
            ],
            cloudify_agent['service'][
                win_const.SERVICE_FAILURE_RESTART_DELAY_KEY
            ]))

    ctx.logger.info('Creating parameters file from {0}'.format(params))
    runner.put(params, '{0}\AppParameters'.format(RUNTIME_AGENT_PATH))

@operation
@init_worker_installer
def start(ctx, runner=None, cloudify_agent=None, **kwargs):

    """
    Starts the cloudify agent service on the machine.

    :param ctx: Invocation context - injected by the @operation
    :param runner: Injected by the @init_worker_installer
    :param cloudify_agent: Injected by the @init_worker_installer
    """

    _start(cloudify_agent, ctx, runner)

@operation
@init_worker_installer
def stop(ctx, runner=None, cloudify_agent=None, **kwargs):

    """
    Stops the cloudify agent service on the machine.

    :param ctx: Invocation context - injected by the @operation
    :param runner: Injected by the @init_worker_installer
    :param cloudify_agent: Injected by the @init_worker_installer
    """

    _stop(cloudify_agent, ctx, runner)

@operation
@init_worker_installer
def restart(ctx, runner=None, cloudify_agent=None, **kwargs):

    """
    Restarts the cloudify agent service on the machine.

        1. Stop the service.
        2. Start the service.

    :param ctx: Invocation context - injected by the @operation
    :param runner: Injected by the @init_worker_installer
    :param cloudify_agent: Injected by the @init_worker_installer
    """

    ctx.logger.info('Restarting agent {0}'.format(cloudify_agent['name']))

    _stop(ctx=ctx, runner=runner, cloudify_agent=cloudify_agent)
    _start(ctx=ctx, runner=runner, cloudify_agent=cloudify_agent)

@operation
@init_worker_installer
def uninstall(ctx, runner=None, cloudify_agent=None, **kwargs):

    """
    Uninstalls the cloudify agent service from the machine.

        1. Remove the service from the registry.
        2. Delete related files..

    :param ctx: Invocation context - injected by the @operation
    :param runner: Injected by the @init_worker_installer
    :param cloudify_agent: Injected by the @init_worker_installer
    """

    ctx.logger.info('Uninstalling agent {0}'.format(cloudify_agent['name']))

    runner.run('{0} remove {1} confirm'.format('{0}\\nssm\\nssm.exe'
                                               .format(RUNTIME_AGENT_PATH),
                                               AGENT_SERVICE_NAME))

    runner.delete(path=RUNTIME_AGENT_PATH)
    runner.delete(path='C:\\{0}'.format(AGENT_EXEC_FILE_NAME))

def _delete_amqp_queues(worker_name):
    # FIXME: this function deletes amqp queues that will be used by worker.
    # The amqp queues used by celery worker are determined by worker name
    # and if there are multiple workers with same name celery gets confused.
    #
    # Currently the worker name is based solely on hostname, so it will be
    # re-used if vm gets re-created by auto-heal.
    # Deleting the queues is a workaround for celery problems this creates.
    # Having unique worker names is probably a better long-term strategy.
    client = amqp_client.create_client()
    try:
        channel = client.connection.channel()

        # celery worker queue
        channel.queue_delete(worker_name)

        # celery management queue
        channel.queue_delete('celery@{0}.celery.pidbox'.format(worker_name))
    finally:
        try:
            client.close()
        except Exception:
            pass

def _verify_no_celery_error(runner):
    # don't use os.path.join here since
    # since the manager is linux
    # and the agent is windows
    celery_error_out = '{0}\{1}'.format(RUNTIME_AGENT_PATH,
                                        win_constants.CELERY_ERROR_FILE)

    # this means the celery worker had an uncaught
    # exception and it wrote its content
    # to the file above because of our custom
    # exception handler (see celery.py)
    if runner.exists(celery_error_out):
        output = runner.get(celery_error_out)
        runner.delete(path=celery_error_out)
        raise NonRecoverableError(output)

def _wait_for_started(runner, cloudify_agent):
    _verify_no_celery_error(runner)
    worker_name = 'celery@{0}'.format(cloudify_agent['name'])
    wait_started_timeout = cloudify_agent[
        win_constants.AGENT_START_TIMEOUT_KEY
    ]
    timeout = time.time() + wait_started_timeout
    interval = cloudify_agent[win_constants.AGENT_START_INTERVAL_KEY]
    while time.time() < timeout:
        stats = get_worker_stats(worker_name)
        if stats:
            return
        time.sleep(interval)
    _verify_no_celery_error(runner)
    raise NonRecoverableError('Failed starting agent. waited for {0} seconds.'
                              .format(wait_started_timeout))

def _wait_for_stopped(runner, cloudify_agent):
    _verify_no_celery_error(runner)
    worker_name = 'celery@{0}'.format(cloudify_agent['name'])
    wait_started_timeout = cloudify_agent[
        win_constants.AGENT_STOP_TIMEOUT_KEY
    ]
    timeout = time.time() + wait_started_timeout
    interval = cloudify_agent[win_constants.AGENT_STOP_INTERVAL_KEY]
    while time.time() < timeout:
        stats = get_worker_stats(worker_name)
        if not stats:
            return
        time.sleep(interval)
    _verify_no_celery_error(runner)
    raise NonRecoverableError('Failed stopping agent. waited for {0} seconds.'
                              .format(wait_started_timeout))

def get_worker_stats(worker_name):
    inspect = celery_client.control.inspect(destination=[worker_name])
    stats = (inspect.stats() or {}).get(worker_name)
    return stats

def _stop(cloudify_agent, ctx, runner):
    ctx.logger.info('Stopping agent {0}'.format(cloudify_agent['name']))
    runner.run('sc stop {}'.format(AGENT_SERVICE_NAME))
    _wait_for_stopped(runner, cloudify_agent)
    runner.delete('{0}\celery.pid'.format(RUNTIME_AGENT_PATH),
                  ignore_missing=True)

def _start(cloudify_agent, ctx, runner):
    ctx.logger.info('Starting agent {0}'.format(cloudify_agent['name']))
    runner.run('sc start {}'.format(AGENT_SERVICE_NAME))
    ctx.logger.info('Waiting for {0} to start...'.format(AGENT_SERVICE_NAME))
    _wait_for_started(runner, cloudify_agent)
<commit_msg>CFY-2743 logging to multiple files in windows agents<commit_after>#########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  * See the License for the specific language governing permissions and
#  * limitations under the License.

import time

from cloudify import utils
from cloudify import amqp_client
from cloudify import constants
from cloudify.celery import celery as celery_client
from cloudify.decorators import operation
from cloudify.exceptions import NonRecoverableError

from windows_agent_installer import constants as win_constants
from windows_agent_installer import init_worker_installer
from windows_agent_installer import constants as win_const

# This is the folder under which the agent is
# extracted to inside the current directory.
# It is set in the packaging process so it
# must be hardcoded here.
AGENT_FOLDER_NAME = 'CloudifyAgent'

# This is where we download the agent to.
AGENT_EXEC_FILE_NAME = 'CloudifyAgent.exe'

# nssm will install celery and use this name to identify the service
AGENT_SERVICE_NAME = 'CloudifyAgent'

# location of the agent package on the management machine,
# relative to the file server root.
AGENT_PACKAGE_PATH = 'packages/agents/CloudifyWindowsAgent.exe'

# Path to the agent. We are using global (not user based) paths
# because of virtualenv relocation issues on windows.
RUNTIME_AGENT_PATH = 'C:\CloudifyAgent'

# Agent includes list, Mandatory
AGENT_INCLUDES = 'script_runner.tasks,windows_plugin_installer.tasks,'\
                 'cloudify.plugins.workflows'

def get_agent_package_url():
    return '{0}/{1}'.format(utils.get_manager_file_server_url(),
                            AGENT_PACKAGE_PATH)

def create_env_string(cloudify_agent):
    env = {
        constants.CELERY_WORK_DIR_PATH_KEY:
        RUNTIME_AGENT_PATH,
        constants.LOCAL_IP_KEY:
        cloudify_agent['host'],
        constants.MANAGER_IP_KEY:
        utils.get_manager_ip(),
        constants.MANAGER_FILE_SERVER_BLUEPRINTS_ROOT_URL_KEY:
        utils.get_manager_file_server_blueprints_root_url(),
        constants.MANAGER_FILE_SERVER_URL_KEY:
        utils.get_manager_file_server_url(),
        constants.MANAGER_REST_PORT_KEY:
        utils.get_manager_rest_service_port()
    }
    env_string = ''
    for key, value in env.iteritems():
        env_string = '{0} {1}={2}' \
            .format(env_string, key, value)
    return env_string.strip()

@operation
@init_worker_installer
def install(ctx, runner=None, cloudify_agent=None, **kwargs):

    """
    Installs the cloudify agent service on the machine.
    The agent installation consists of the following:

        1. Download and extract necessary files.
        2. Configure the agent service to auto start on vm launch.
        3. Configure the agent service to restart on failure.

    :param ctx: Invocation context - injected by the @operation
    :param runner: Injected by the @init_worker_installer
    :param cloudify_agent: Injected by the @init_worker_installer
    """

    if cloudify_agent.get('delete_amqp_queues'):
        _delete_amqp_queues(cloudify_agent['name'])

    ctx.logger.info('Installing agent {0}'.format(cloudify_agent['name']))

    agent_exec_path = 'C:\{0}'.format(AGENT_EXEC_FILE_NAME)

    runner.download(get_agent_package_url(), agent_exec_path)
    ctx.logger.debug('Extracting agent to C:\\ ...')

    runner.run('{0} -o{1} -y'.format(agent_exec_path, RUNTIME_AGENT_PATH),
               quiet=True)

    params = ('--broker=amqp://guest:guest@{0}:5672// '
              '-Ofair '
              '--events '
              '--app=cloudify '
              '-Q {1} '
              '--hostname={1} '
              '--logfile={2}\celery{{0}}.log '
              '--pidfile={2}\celery.pid '
              '--autoscale={3},{4} '
              '--include={5} '
              '--without-gossip '
              '--without-mingle '
              .format(utils.get_manager_ip(),
                      cloudify_agent['name'],
                      RUNTIME_AGENT_PATH,
                      cloudify_agent[win_constants.MIN_WORKERS_KEY],
                      cloudify_agent[win_constants.MAX_WORKERS_KEY],
                      AGENT_INCLUDES))
    runner.run('{0}\\nssm\\nssm.exe install {1} {0}\Scripts\celeryd.exe {2}'
               .format(RUNTIME_AGENT_PATH, AGENT_SERVICE_NAME, params))
    env = create_env_string(cloudify_agent)
    runner.run('{0}\\nssm\\nssm.exe set {1} AppEnvironmentExtra {2}'
               .format(RUNTIME_AGENT_PATH, AGENT_SERVICE_NAME, env))
    runner.run('sc config {0} start= auto'.format(AGENT_SERVICE_NAME))
    runner.run(
        'sc failure {0} reset= {1} actions= restart/{2}'.format(
            AGENT_SERVICE_NAME,
            cloudify_agent['service'][
                win_const.SERVICE_FAILURE_RESET_TIMEOUT_KEY
            ],
            cloudify_agent['service'][
                win_const.SERVICE_FAILURE_RESTART_DELAY_KEY
            ]))

    ctx.logger.info('Creating parameters file from {0}'.format(params))
    runner.put(params, '{0}\AppParameters'.format(RUNTIME_AGENT_PATH))

@operation
@init_worker_installer
def start(ctx, runner=None, cloudify_agent=None, **kwargs):

    """
    Starts the cloudify agent service on the machine.

    :param ctx: Invocation context - injected by the @operation
    :param runner: Injected by the @init_worker_installer
    :param cloudify_agent: Injected by the @init_worker_installer
    """

    _start(cloudify_agent, ctx, runner)

@operation
@init_worker_installer
def stop(ctx, runner=None, cloudify_agent=None, **kwargs):

    """
    Stops the cloudify agent service on the machine.

    :param ctx: Invocation context - injected by the @operation
    :param runner: Injected by the @init_worker_installer
    :param cloudify_agent: Injected by the @init_worker_installer
    """

    _stop(cloudify_agent, ctx, runner)

@operation
@init_worker_installer
def restart(ctx, runner=None, cloudify_agent=None, **kwargs):

    """
    Restarts the cloudify agent service on the machine.

        1. Stop the service.
        2. Start the service.

    :param ctx: Invocation context - injected by the @operation
    :param runner: Injected by the @init_worker_installer
    :param cloudify_agent: Injected by the @init_worker_installer
    """

    ctx.logger.info('Restarting agent {0}'.format(cloudify_agent['name']))

    _stop(ctx=ctx, runner=runner, cloudify_agent=cloudify_agent)
    _start(ctx=ctx, runner=runner, cloudify_agent=cloudify_agent)

@operation
@init_worker_installer
def uninstall(ctx, runner=None, cloudify_agent=None, **kwargs):

    """
    Uninstalls the cloudify agent service from the machine.

        1. Remove the service from the registry.
        2. Delete related files..

    :param ctx: Invocation context - injected by the @operation
    :param runner: Injected by the @init_worker_installer
    :param cloudify_agent: Injected by the @init_worker_installer
    """

    ctx.logger.info('Uninstalling agent {0}'.format(cloudify_agent['name']))

    runner.run('{0} remove {1} confirm'.format('{0}\\nssm\\nssm.exe'
                                               .format(RUNTIME_AGENT_PATH),
                                               AGENT_SERVICE_NAME))

    runner.delete(path=RUNTIME_AGENT_PATH)
    runner.delete(path='C:\\{0}'.format(AGENT_EXEC_FILE_NAME))

def _delete_amqp_queues(worker_name):
    # FIXME: this function deletes amqp queues that will be used by worker.
    # The amqp queues used by celery worker are determined by worker name
    # and if there are multiple workers with same name celery gets confused.
    #
    # Currently the worker name is based solely on hostname, so it will be
    # re-used if vm gets re-created by auto-heal.
    # Deleting the queues is a workaround for celery problems this creates.
    # Having unique worker names is probably a better long-term strategy.
    client = amqp_client.create_client()
    try:
        channel = client.connection.channel()

        # celery worker queue
        channel.queue_delete(worker_name)

        # celery management queue
        channel.queue_delete('celery@{0}.celery.pidbox'.format(worker_name))
    finally:
        try:
            client.close()
        except Exception:
            pass

def _verify_no_celery_error(runner):
    # don't use os.path.join here since
    # since the manager is linux
    # and the agent is windows
    celery_error_out = '{0}\{1}'.format(RUNTIME_AGENT_PATH,
                                        win_constants.CELERY_ERROR_FILE)

    # this means the celery worker had an uncaught
    # exception and it wrote its content
    # to the file above because of our custom
    # exception handler (see celery.py)
    if runner.exists(celery_error_out):
        output = runner.get(celery_error_out)
        runner.delete(path=celery_error_out)
        raise NonRecoverableError(output)

def _wait_for_started(runner, cloudify_agent):
    _verify_no_celery_error(runner)
    worker_name = 'celery@{0}'.format(cloudify_agent['name'])
    wait_started_timeout = cloudify_agent[
        win_constants.AGENT_START_TIMEOUT_KEY
    ]
    timeout = time.time() + wait_started_timeout
    interval = cloudify_agent[win_constants.AGENT_START_INTERVAL_KEY]
    while time.time() < timeout:
        stats = get_worker_stats(worker_name)
        if stats:
            return
        time.sleep(interval)
    _verify_no_celery_error(runner)
    raise NonRecoverableError('Failed starting agent. waited for {0} seconds.'
                              .format(wait_started_timeout))

def _wait_for_stopped(runner, cloudify_agent):
    _verify_no_celery_error(runner)
    worker_name = 'celery@{0}'.format(cloudify_agent['name'])
    wait_started_timeout = cloudify_agent[
        win_constants.AGENT_STOP_TIMEOUT_KEY
    ]
    timeout = time.time() + wait_started_timeout
    interval = cloudify_agent[win_constants.AGENT_STOP_INTERVAL_KEY]
    while time.time() < timeout:
        stats = get_worker_stats(worker_name)
        if not stats:
            return
        time.sleep(interval)
    _verify_no_celery_error(runner)
    raise NonRecoverableError('Failed stopping agent. waited for {0} seconds.'
                              .format(wait_started_timeout))

def get_worker_stats(worker_name):
    inspect = celery_client.control.inspect(destination=[worker_name])
    stats = (inspect.stats() or {}).get(worker_name)
    return stats

def _stop(cloudify_agent, ctx, runner):
    ctx.logger.info('Stopping agent {0}'.format(cloudify_agent['name']))
    runner.run('sc stop {}'.format(AGENT_SERVICE_NAME))
    _wait_for_stopped(runner, cloudify_agent)
    runner.delete('{0}\celery.pid'.format(RUNTIME_AGENT_PATH),
                  ignore_missing=True)

def _start(cloudify_agent, ctx, runner):
    ctx.logger.info('Starting agent {0}'.format(cloudify_agent['name']))
    runner.run('sc start {}'.format(AGENT_SERVICE_NAME))
    ctx.logger.info('Waiting for {0} to start...'.format(AGENT_SERVICE_NAME))
    _wait_for_started(runner, cloudify_agent)
<|endoftext|>


<commit_before>import os
import glob
import re

class WhiteList(dict):
    def __init__(self, basedir):
    	searchdir = os.path.join(basedir, "whitelists/whitelist_*.ini")
        fpaths = glob.glob(searchdir)
        for fpath in fpaths:
            t = os.path.splitext(os.path.split(fpath)[1])[0].split('_',1)[1]
            patterns = [line.strip() for line in open(fpath)]
            self[t]  = [re.compile(p) for p in patterns]<commit_msg>Using 'with' instead of 'open'<commit_after>import os
import glob
import re

class WhiteList(dict):
    def __init__(self, basedir):
    	searchdir = os.path.join(basedir, "whitelists/whitelist_*.ini")
        fpaths = glob.glob(searchdir)
        for fpath in fpaths:
            t = os.path.splitext(os.path.split(fpath)[1])[0].split('_',1)[1]
            with open(fpath) as f:
                patterns = [line.strip() for line in f]
                self[t]  = [re.compile(p) for p in patterns]
<|endoftext|>


<commit_before># Author: Andreas Christian Mueller <amueller@ais.uni-bonn.de>
# (c) 2012
#
# License: MIT

import random

from PIL import Image
from PIL import ImageDraw
from PIL import ImageFont

import numpy as np
from query_integral_image import query_integral_image

FONT_PATH = "/usr/share/fonts/truetype/droid/DroidSansMono.ttf"

def make_wordcloud(words, counts, fname, font_path=None, width=400, height=200,
                   margin=5):
    """Build word cloud using word counts, store in image.

    Parameters
    ----------
    words : numpy array of strings
        Words that will be drawn in the image.

    counts : numpy array of word counts
        Word counts or weighting of words. Determines the size of the word in
        the final image.
        Will be normalized to lie between zero and one.

    font_path : sting
        Font path to the font that will be used.
        Defaults to DroidSansMono path.

    fname : sting
        Output filename. Extension determins image type
        (written with PIL).

    width : int (default=400)
        Width of the word cloud image.

    height : int (default=200)
        Height of the word cloud image.

    Notes
    -----
    Larger Images with make the code significantly slower.
    If you need a large image, you can try running the algorithm at a lower
    resolution and then drawing the result at the desired resolution.

    In the current form it actually just uses the rank of the counts,
    i.e. the relative differences don't matter.
    Play with setting the font_size in the main loop vor differnt styles.

    Colors are used completely at random. Currently the colors are sampled
    from HSV space with a fixed S and V.
    Adjusting the percentages at the very end gives differnt color ranges.
    Obviously you can also set all at random - haven't tried that.

    """
    if len(counts) <= 0:
        print("We need at least 1 word to plot a word cloud, got %d."
              % len(counts))

    if font_path is None:
        font_path = FONT_PATH

    # normalize counts
    counts = counts / float(counts.max())
    # sort words by counts
    inds = np.argsort(counts)[::-1]
    counts = counts[inds]
    words = words[inds]
    # create image
    img_grey = Image.new("L", (width, height))
    draw = ImageDraw.Draw(img_grey)
    integral = np.zeros((height, width), dtype=np.uint32)
    img_array = np.asarray(img_grey)
    font_sizes, positions, orientations = [], [], []
    # intitiallize font size "large enough"
    font_size = 1000
    # start drawing grey image
    for word, count in zip(words, counts):
        # alternative way to set the font size
        #font_size = min(font_size, int(100 * np.log(count + 100)))
        while True:
            # try to find a position
            font = ImageFont.truetype(font_path, font_size)
            # transpose font optionally
            orientation = random.choice([None, Image.ROTATE_90])
            transposed_font = ImageFont.TransposedFont(font,
                                                       orientation=orientation)
            draw.setfont(transposed_font)
            # get size of resulting text
            box_size = draw.textsize(word)
            # find possible places using integral image:
            result = query_integral_image(integral, box_size[1] + margin,
                                          box_size[0] + margin)
            if result is not None or font_size == 0:
                break
            # if we didn't find a place, make font smaller
            font_size -= 1

        if font_size == 0:
            # we were unable to draw any more
            break

        x, y = np.array(result) + margin // 2
        # actually draw the text
        draw.text((y, x), word, fill="white")
        positions.append((x, y))
        orientations.append(orientation)
        font_sizes.append(font_size)
        # recompute integral image
        img_array = np.asarray(img_grey)
        # recompute bottom right
        # the order of the cumsum's is important for speed ?!
        partial_integral = np.cumsum(np.cumsum(img_array[x:, y:], axis=1),
                                     axis=0)
        # paste recomputed part into old image
        # if x or y is zero it is a bit annoying
        if x > 0:
            if y > 0:
                partial_integral += (integral[x - 1, y:]
                                     - integral[x - 1, y - 1])
            else:
                partial_integral += integral[x - 1, y:]
        if y > 0:
            partial_integral += integral[x:, y - 1][:, np.newaxis]

        integral[x:, y:] = partial_integral

    # redraw in color
    img = Image.new("RGB", (width, height))
    draw = ImageDraw.Draw(img)
    everything = zip(words, font_sizes, positions, orientations)
    for word, font_size, position, orientation in everything:
        font = ImageFont.truetype(font_path, font_size)
        # transpose font optionally
        transposed_font = ImageFont.TransposedFont(font,
                                                   orientation=orientation)
        draw.setfont(transposed_font)
        draw.text((position[1], position[0]), word,
                  fill="hsl(%d" % random.randint(0, 255) + ", 80%, 50%)")
    img.show()
    img.save(fname)

if __name__ == "__main__":

    import os
    import sys
    from sklearn.feature_extraction.text import CountVectorizer

    sources = ([arg for arg in sys.argv[1:] if os.path.exists(arg)]
               or ["constitution.txt"])
    lines = []
    for s in sources:
        with open(s) as f:
            lines.extend(f.readlines())
    text = "".join(lines)

    cv = CountVectorizer(min_df=1, charset_error="ignore",
                         stop_words="english", max_features=200)
    counts = cv.fit_transform([text]).toarray().ravel()
    words = np.array(cv.get_feature_names())
    # throw away some words, normalize
    words = words[counts > 1]
    counts = counts[counts > 1]
    output_filename = (os.path.splitext(os.path.basename(sources[0]))[0]
                       + "_.png")
    counts = make_wordcloud(words, counts, output_filename)
<commit_msg>Read from stdin when '-' passed<commit_after># Author: Andreas Christian Mueller <amueller@ais.uni-bonn.de>
# (c) 2012
#
# License: MIT

import random
import re

from PIL import Image
from PIL import ImageDraw
from PIL import ImageFont

import numpy as np
from query_integral_image import query_integral_image

FONT_PATH = "/usr/share/fonts/truetype/droid/DroidSansMono.ttf"

def make_wordcloud(words, counts, fname, font_path=None, width=400, height=200,
                   margin=5):
    """Build word cloud using word counts, store in image.

    Parameters
    ----------
    words : numpy array of strings
        Words that will be drawn in the image.

    counts : numpy array of word counts
        Word counts or weighting of words. Determines the size of the word in
        the final image.
        Will be normalized to lie between zero and one.

    font_path : sting
        Font path to the font that will be used.
        Defaults to DroidSansMono path.

    fname : sting
        Output filename. Extension determins image type
        (written with PIL).

    width : int (default=400)
        Width of the word cloud image.

    height : int (default=200)
        Height of the word cloud image.

    Notes
    -----
    Larger Images with make the code significantly slower.
    If you need a large image, you can try running the algorithm at a lower
    resolution and then drawing the result at the desired resolution.

    In the current form it actually just uses the rank of the counts,
    i.e. the relative differences don't matter.
    Play with setting the font_size in the main loop vor differnt styles.

    Colors are used completely at random. Currently the colors are sampled
    from HSV space with a fixed S and V.
    Adjusting the percentages at the very end gives differnt color ranges.
    Obviously you can also set all at random - haven't tried that.

    """
    if len(counts) <= 0:
        print("We need at least 1 word to plot a word cloud, got %d."
              % len(counts))

    if font_path is None:
        font_path = FONT_PATH

    # normalize counts
    counts = counts / float(counts.max())
    # sort words by counts
    inds = np.argsort(counts)[::-1]
    counts = counts[inds]
    words = words[inds]
    # create image
    img_grey = Image.new("L", (width, height))
    draw = ImageDraw.Draw(img_grey)
    integral = np.zeros((height, width), dtype=np.uint32)
    img_array = np.asarray(img_grey)
    font_sizes, positions, orientations = [], [], []
    # intitiallize font size "large enough"
    font_size = 1000
    # start drawing grey image
    for word, count in zip(words, counts):
        # alternative way to set the font size
        #font_size = min(font_size, int(100 * np.log(count + 100)))
        while True:
            # try to find a position
            font = ImageFont.truetype(font_path, font_size)
            # transpose font optionally
            orientation = random.choice([None, Image.ROTATE_90])
            transposed_font = ImageFont.TransposedFont(font,
                                                       orientation=orientation)
            draw.setfont(transposed_font)
            # get size of resulting text
            box_size = draw.textsize(word)
            # find possible places using integral image:
            result = query_integral_image(integral, box_size[1] + margin,
                                          box_size[0] + margin)
            if result is not None or font_size == 0:
                break
            # if we didn't find a place, make font smaller
            font_size -= 1

        if font_size == 0:
            # we were unable to draw any more
            break

        x, y = np.array(result) + margin // 2
        # actually draw the text
        draw.text((y, x), word, fill="white")
        positions.append((x, y))
        orientations.append(orientation)
        font_sizes.append(font_size)
        # recompute integral image
        img_array = np.asarray(img_grey)
        # recompute bottom right
        # the order of the cumsum's is important for speed ?!
        partial_integral = np.cumsum(np.cumsum(img_array[x:, y:], axis=1),
                                     axis=0)
        # paste recomputed part into old image
        # if x or y is zero it is a bit annoying
        if x > 0:
            if y > 0:
                partial_integral += (integral[x - 1, y:]
                                     - integral[x - 1, y - 1])
            else:
                partial_integral += integral[x - 1, y:]
        if y > 0:
            partial_integral += integral[x:, y - 1][:, np.newaxis]

        integral[x:, y:] = partial_integral

    # redraw in color
    img = Image.new("RGB", (width, height))
    draw = ImageDraw.Draw(img)
    everything = zip(words, font_sizes, positions, orientations)
    for word, font_size, position, orientation in everything:
        font = ImageFont.truetype(font_path, font_size)
        # transpose font optionally
        transposed_font = ImageFont.TransposedFont(font,
                                                   orientation=orientation)
        draw.setfont(transposed_font)
        draw.text((position[1], position[0]), word,
                  fill="hsl(%d" % random.randint(0, 255) + ", 80%, 50%)")
    img.show()
    img.save(fname)

if __name__ == "__main__":

    import os
    import sys
    from sklearn.feature_extraction.text import CountVectorizer

    if "-" in sys.argv:
        lines = sys.stdin.readlines()
        sources = ['stdin']
    else:
        sources = ([arg for arg in sys.argv[1:] if os.path.exists(arg)]
                   or ["constitution.txt"])
        lines = []
        for s in sources:
            with open(s) as f:
                lines.extend(f.readlines())
    text = "".join(lines)

    cv = CountVectorizer(min_df=1, charset_error="ignore",
                         stop_words="english", max_features=200)
    counts = cv.fit_transform([text]).toarray().ravel()
    words = np.array(cv.get_feature_names())
    # throw away some words, normalize
    words = words[counts > 1]
    counts = counts[counts > 1]
    output_filename = (os.path.splitext(os.path.basename(sources[0]))[0]
                       + "_.png")
    counts = make_wordcloud(words, counts, output_filename)
<|endoftext|>


<commit_before>import sys, random, cryptomath_module as cryptoMath

SYMBOLS = r""" !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~"""

def main():
    message = input("Enter message: ")
    key = int(input("Enter key [2000 - 9000]: "))
    mode = input("Encrypt/Decrypt [E/D]: ")

    if mode.lower().startswith("e"):
        mode = "encrypt"
        translated = encryptMessage(key, message)
    elif mode.lower().startswith("d"):
        mode = "decrypt"
        translated = decryptMessage(key, message)
    print("\n%sed text: \n%s" % (mode.title(), translated))

def getKeyParts(key):
    keyA = key // len(SYMBOLS)
    keyB = key % len(SYMBOLS)
    return (keyA, keyB)

def checkKeys(keyA, keyB, mode):
    if keyA == 1 and mode == "encrypt":
        sys.exit(
            "The affine cipher becomes weak when key A is set to 1. Choose different key"
        )
    if keyB == 0 and mode == "encrypt":
        sys.exit(
            "The affine cipher becomes weak when key A is set to 1. Choose different key"
        )
    if keyA < 0 or keyB < 0 or keyB > len(SYMBOLS) - 1:
        sys.exit(
            "Key A must be greater than 0 and key B must be between 0 and %s."
            % (len(SYMBOLS) - 1)
        )
    if cryptoMath.gcd(keyA, len(SYMBOLS)) != 1:
        sys.exit(
            "Key A %s and the symbol set size %s are not relatively prime. Choose a different key."
            % (keyA, len(SYMBOLS))
        )

def encryptMessage(key, message):
    """
    >>> encryptMessage(4545, 'The affine cipher is a type of monoalphabetic substitution cipher.')
    'VL}p MM{I}p~{HL}Gp{vp pFsH}pxMpyxIx JHL O}F{~pvuOvF{FuF{xIp~{HL}Gi'
    """
    keyA, keyB = getKeyParts(key)
    checkKeys(keyA, keyB, "encrypt")
    cipherText = ""
    for symbol in message:
        if symbol in SYMBOLS:
            symIndex = SYMBOLS.find(symbol)
            cipherText += SYMBOLS[(symIndex * keyA + keyB) % len(SYMBOLS)]
        else:
            cipherText += symbol
    return cipherText

def decryptMessage(key, message):
    """
    >>> decryptMessage(4545, 'VL}p MM{I}p~{HL}Gp{vp pFsH}pxMpyxIx JHL O}F{~pvuOvF{FuF{xIp~{HL}Gi')
    'The affine cipher is a type of monoalphabetic substitution cipher.'
    """
    keyA, keyB = getKeyParts(key)
    checkKeys(keyA, keyB, "decrypt")
    plainText = ""
    modInverseOfkeyA = cryptoMath.findModInverse(keyA, len(SYMBOLS))
    for symbol in message:
        if symbol in SYMBOLS:
            symIndex = SYMBOLS.find(symbol)
            plainText += SYMBOLS[(symIndex - keyB) * modInverseOfkeyA % len(SYMBOLS)]
        else:
            plainText += symbol
    return plainText

def getRandomKey():
    while True:
        keyA = random.randint(2, len(SYMBOLS))
        keyB = random.randint(2, len(SYMBOLS))
        if cryptoMath.gcd(keyA, len(SYMBOLS)) == 1:
            return keyA * len(SYMBOLS) + keyB

if __name__ == "__main__":
    import doctest

    doctest.testmod()
    main()
<commit_msg>Fixed warning string for Key B = 0 (#1639)<commit_after>import random
import sys

import cryptomath_module as cryptomath

SYMBOLS = (r""" !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`"""
           r"""abcdefghijklmnopqrstuvwxyz{|}~""")

def main():
    """
    >>> key = get_random_key()
    >>> msg = "This is a test!"
    >>> decrypt_message(key, encrypt_message(key, msg)) == msg
    True
    """
    message = input("Enter message: ").strip()
    key = int(input("Enter key [2000 - 9000]: ").strip())
    mode = input("Encrypt/Decrypt [E/D]: ").strip().lower()

    if mode.startswith("e"):
        mode = "encrypt"
        translated = encrypt_message(key, message)
    elif mode.startswith("d"):
        mode = "decrypt"
        translated = decrypt_message(key, message)
    print(f"\n{mode.title()}ed text: \n{translated}")

def check_keys(keyA, keyB, mode):
    if mode == "encrypt":
        if keyA == 1:
            sys.exit(
                "The affine cipher becomes weak when key "
                "A is set to 1. Choose different key"
            )
        if keyB == 0:
            sys.exit(
                "The affine cipher becomes weak when key "
                "B is set to 0. Choose different key"
            )
    if keyA < 0 or keyB < 0 or keyB > len(SYMBOLS) - 1:
        sys.exit(
            "Key A must be greater than 0 and key B must "
            f"be between 0 and {len(SYMBOLS) - 1}."
        )
    if cryptomath.gcd(keyA, len(SYMBOLS)) != 1:
        sys.exit(
            f"Key A {keyA} and the symbol set size {len(SYMBOLS)} "
            "are not relatively prime. Choose a different key."
        )

def encrypt_message(key: int, message: str) -> str:
    """
    >>> encrypt_message(4545, 'The affine cipher is a type of monoalphabetic substitution cipher.')
    'VL}p MM{I}p~{HL}Gp{vp pFsH}pxMpyxIx JHL O}F{~pvuOvF{FuF{xIp~{HL}Gi'
    """
    keyA, keyB = divmod(key, len(SYMBOLS))
    check_keys(keyA, keyB, "encrypt")
    cipherText = ""
    for symbol in message:
        if symbol in SYMBOLS:
            symIndex = SYMBOLS.find(symbol)
            cipherText += SYMBOLS[(symIndex * keyA + keyB) % len(SYMBOLS)]
        else:
            cipherText += symbol
    return cipherText

def decrypt_message(key: int, message: str) -> str:
    """
    >>> decrypt_message(4545, 'VL}p MM{I}p~{HL}Gp{vp pFsH}pxMpyxIx JHL O}F{~pvuOvF{FuF{xIp~{HL}Gi')
    'The affine cipher is a type of monoalphabetic substitution cipher.'
    """
    keyA, keyB = divmod(key, len(SYMBOLS))
    check_keys(keyA, keyB, "decrypt")
    plainText = ""
    modInverseOfkeyA = cryptomath.findModInverse(keyA, len(SYMBOLS))
    for symbol in message:
        if symbol in SYMBOLS:
            symIndex = SYMBOLS.find(symbol)
            plainText += SYMBOLS[(symIndex - keyB) * modInverseOfkeyA % len(SYMBOLS)]
        else:
            plainText += symbol
    return plainText

def get_random_key():
    while True:
        keyA = random.randint(2, len(SYMBOLS))
        keyB = random.randint(2, len(SYMBOLS))
        if cryptomath.gcd(keyA, len(SYMBOLS)) == 1:
            return keyA * len(SYMBOLS) + keyB

if __name__ == "__main__":
    import doctest

    doctest.testmod()
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python3
import config

import logging

logging.basicConfig(level=logging.DEBUG)

def RaiseRedFlags(reason):
	logging.getLogger('redflag').critical(reason)
	return reason

from bitcoin.node import BitcoinLink
UpstreamBitcoind = BitcoinLink( config.UpstreamBitcoindNode, config.UpstreamNetworkId )

from bitcoin.script import BitcoinScript
from bitcoin.txn import Txn
from base58 import b58decode
from struct import pack
import subprocess
from time import time

def makeCoinbaseTxn(coinbaseValue, useCoinbaser = True):
	t = Txn.new()

	if useCoinbaser and hasattr(config, 'CoinbaserCmd'):
		coinbased = 0
		try:
			cmd = config.CoinbaserCmd
			cmd = cmd.replace('%d', str(coinbaseValue))
			p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
			nout = int(p.stdout.readline())
			for i in range(nout):
				amount = int(p.stdout.readline())
				addr = p.stdout.readline().rstrip(b'\n').decode('utf8')
				pkScript = BitcoinScript.toAddress(addr)
				t.addOutput(amount, pkScript)
				coinbased += amount
		except:
			coinbased = coinbaseValue + 1
		if coinbased >= coinbaseValue:
			logging.getLogger('makeCoinbaseTxn').error('Coinbaser failed!')
			t.outputs = []
		else:
			coinbaseValue -= coinbased

	pkScript = BitcoinScript.toAddress(config.TrackerAddr)
	t.addOutput(coinbaseValue, pkScript)

	# TODO
	# TODO: red flag on dupe coinbase
	return t

from util import Bits2Target

workLog = {}
networkTarget = None

def blockChanged():
	global MM, networkTarget
	networkTarget = Bits2Target(MM.currentBlock[1])
	workLog.clear()

from merklemaker import merkleMaker
MM = merkleMaker()
MM.__dict__.update(config.__dict__)
MM.clearCoinbaseTxn = makeCoinbaseTxn(5000000000, False)  # FIXME
MM.clearCoinbaseTxn.assemble()
MM.makeCoinbaseTxn = makeCoinbaseTxn
MM.onBlockChange = blockChanged
MM._THISISUGLY = UpstreamBitcoind
MM.start()

from binascii import b2a_hex
from struct import pack, unpack
from time import time
from util import RejectedShare, dblsha, hash2int

db = None
if hasattr(config, 'DbOptions'):
	import psycopg2
	db = psycopg2.connect(**config.DbOptions)

def getBlockHeader(username):
	MRD = MM.getMRD()
	(merkleRoot, merkleTree, coinbase, prevBlock, bits, rollPrevBlk) = MRD
	timestamp = pack('<L', int(time()))
	hdr = b'\1\0\0\0' + prevBlock + merkleRoot + timestamp + bits + b'iolE'
	workLog.setdefault(username, {})[merkleRoot] = MRD
	return hdr

def YN(b):
	return 'Y' if b else 'N'

def logShare(share):
	if db is None:
		return
	dbc = db.cursor()
	rem_host = 'TODO'
	username = share['username']
	reason = share.get('rejectReason', None)
	solution = share['data']
	solution = b2a_hex(solution).decode('utf8')
	stmt = "insert into shares (rem_host, username, our_result, upstream_result, reason, solution) values (%s, %s, %s, %s, %s, decode(%s, 'hex'))"
	params = (rem_host, username, YN(not reason), None, reason, solution)
	print((stmt,params))
	dbc.execute(stmt, params)
	db.commit()

def checkShare(share):
	data = share['data']
	(prevBlock, bits) = MM.currentBlock
	sharePrevBlock = data[4:36]
	if sharePrevBlock != prevBlock:
		if sharePrevBlock == MM.lastBlock[0]:
			raise RejectedShare('stale-prevblk')
		raise RejectedShare('bad-prevblk')

	shareMerkleRoot = data[36:68]
	# TODO: use userid
	username = share['username']
	if username not in workLog:
		raise RejectedShare('unknown-user')
	MWL = workLog[username]
	if shareMerkleRoot not in MWL:
		raise RejectedShare('unknown-work')
	MRD = MWL[shareMerkleRoot]
	share['MRD'] = MRD

	shareTimestamp = unpack('<L', data[68:72])[0]
	shareTime = share['time'] = time()
	if shareTimestamp < shareTime - 300:
		raise RejectedShare('time-too-old')
	if shareTimestamp > shareTime + 7200:
		raise RejectedShare('time-too-new')
	if data[72:76] != bits:
		raise RejectedShare('bad-diffbits')
	if data[:4] != b'\1\0\0\0':
		raise RejectedShare('bad-version')

	blkhash = dblsha(data)
	if blkhash[28:] != b'\0\0\0\0':
		raise RejectedShare('H-not-zero')
	blkhashn = hash2int(blkhash)

	global networkTarget
	logfunc = getattr(checkShare.logger, 'info' if blkhashn <= networkTarget else 'debug')
	logfunc('BLKHASH: %64x' % (blkhashn,))
	logfunc(' TARGET: %64x' % (networkTarget,))

	logShare(share)

	if blkhashn <= networkTarget:
		logfunc("Submitting upstream")
		txlist = MRD[1].data
		t = txlist[0]
		t.setCoinbase(MRD[2])
		t.assemble()
		UpstreamBitcoind.submitBlock(data, txlist)
checkShare.logger = logging.getLogger('checkShare')

def receiveShare(share):
	# TODO: username => userid
	checkShare(share)
	# TODO

def newBlockNotification(signum, frame):
	MM.updateMerkleTree()
	# TODO: Force RESPOND TO LONGPOLLS?
	pass

from signal import signal, SIGUSR1
signal(SIGUSR1, newBlockNotification)

from jsonrpcserver import JSONRPCServer
import interactivemode

if __name__ == "__main__":
	server = JSONRPCServer(('', 8444))
	server.getBlockHeader = getBlockHeader
	server.receiveShare = receiveShare
	server.RaiseRedFlags = RaiseRedFlags
	server.serve_forever()
<commit_msg>Log upstream_result='Y' when it meets the upstream target and gets submitted (even though we don't get a confirmation)<commit_after>#!/usr/bin/python3
import config

import logging

logging.basicConfig(level=logging.DEBUG)

def RaiseRedFlags(reason):
	logging.getLogger('redflag').critical(reason)
	return reason

from bitcoin.node import BitcoinLink
UpstreamBitcoind = BitcoinLink( config.UpstreamBitcoindNode, config.UpstreamNetworkId )

from bitcoin.script import BitcoinScript
from bitcoin.txn import Txn
from base58 import b58decode
from struct import pack
import subprocess
from time import time

def makeCoinbaseTxn(coinbaseValue, useCoinbaser = True):
	t = Txn.new()

	if useCoinbaser and hasattr(config, 'CoinbaserCmd'):
		coinbased = 0
		try:
			cmd = config.CoinbaserCmd
			cmd = cmd.replace('%d', str(coinbaseValue))
			p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
			nout = int(p.stdout.readline())
			for i in range(nout):
				amount = int(p.stdout.readline())
				addr = p.stdout.readline().rstrip(b'\n').decode('utf8')
				pkScript = BitcoinScript.toAddress(addr)
				t.addOutput(amount, pkScript)
				coinbased += amount
		except:
			coinbased = coinbaseValue + 1
		if coinbased >= coinbaseValue:
			logging.getLogger('makeCoinbaseTxn').error('Coinbaser failed!')
			t.outputs = []
		else:
			coinbaseValue -= coinbased

	pkScript = BitcoinScript.toAddress(config.TrackerAddr)
	t.addOutput(coinbaseValue, pkScript)

	# TODO
	# TODO: red flag on dupe coinbase
	return t

from util import Bits2Target

workLog = {}
networkTarget = None

def blockChanged():
	global MM, networkTarget
	networkTarget = Bits2Target(MM.currentBlock[1])
	workLog.clear()

from merklemaker import merkleMaker
MM = merkleMaker()
MM.__dict__.update(config.__dict__)
MM.clearCoinbaseTxn = makeCoinbaseTxn(5000000000, False)  # FIXME
MM.clearCoinbaseTxn.assemble()
MM.makeCoinbaseTxn = makeCoinbaseTxn
MM.onBlockChange = blockChanged
MM._THISISUGLY = UpstreamBitcoind
MM.start()

from binascii import b2a_hex
from struct import pack, unpack
from time import time
from util import RejectedShare, dblsha, hash2int

db = None
if hasattr(config, 'DbOptions'):
	import psycopg2
	db = psycopg2.connect(**config.DbOptions)

def getBlockHeader(username):
	MRD = MM.getMRD()
	(merkleRoot, merkleTree, coinbase, prevBlock, bits, rollPrevBlk) = MRD
	timestamp = pack('<L', int(time()))
	hdr = b'\1\0\0\0' + prevBlock + merkleRoot + timestamp + bits + b'iolE'
	workLog.setdefault(username, {})[merkleRoot] = MRD
	return hdr

def YN(b):
	if b is None:
		return None
	return 'Y' if b else 'N'

def logShare(share):
	if db is None:
		return
	dbc = db.cursor()
	rem_host = 'TODO'
	username = share['username']
	reason = share.get('rejectReason', None)
	upstreamResult = share.get('upstreamResult', None)
	solution = share['data']
	solution = b2a_hex(solution).decode('utf8')
	stmt = "insert into shares (rem_host, username, our_result, upstream_result, reason, solution) values (%s, %s, %s, %s, %s, decode(%s, 'hex'))"
	params = (rem_host, username, YN(not reason), YN(upstreamResult), reason, solution)
	print((stmt,params))
	dbc.execute(stmt, params)
	db.commit()

def checkShare(share):
	data = share['data']
	(prevBlock, bits) = MM.currentBlock
	sharePrevBlock = data[4:36]
	if sharePrevBlock != prevBlock:
		if sharePrevBlock == MM.lastBlock[0]:
			raise RejectedShare('stale-prevblk')
		raise RejectedShare('bad-prevblk')

	shareMerkleRoot = data[36:68]
	# TODO: use userid
	username = share['username']
	if username not in workLog:
		raise RejectedShare('unknown-user')
	MWL = workLog[username]
	if shareMerkleRoot not in MWL:
		raise RejectedShare('unknown-work')
	MRD = MWL[shareMerkleRoot]
	share['MRD'] = MRD

	shareTimestamp = unpack('<L', data[68:72])[0]
	shareTime = share['time'] = time()
	if shareTimestamp < shareTime - 300:
		raise RejectedShare('time-too-old')
	if shareTimestamp > shareTime + 7200:
		raise RejectedShare('time-too-new')
	if data[72:76] != bits:
		raise RejectedShare('bad-diffbits')
	if data[:4] != b'\1\0\0\0':
		raise RejectedShare('bad-version')

	blkhash = dblsha(data)
	if blkhash[28:] != b'\0\0\0\0':
		raise RejectedShare('H-not-zero')
	blkhashn = hash2int(blkhash)

	global networkTarget
	logfunc = getattr(checkShare.logger, 'info' if blkhashn <= networkTarget else 'debug')
	logfunc('BLKHASH: %64x' % (blkhashn,))
	logfunc(' TARGET: %64x' % (networkTarget,))

	if blkhashn <= networkTarget:
		logfunc("Submitting upstream")
		txlist = MRD[1].data
		t = txlist[0]
		t.setCoinbase(MRD[2])
		t.assemble()
		UpstreamBitcoind.submitBlock(data, txlist)
		share['upstreamResult'] = True

	logShare(share)
checkShare.logger = logging.getLogger('checkShare')

def receiveShare(share):
	# TODO: username => userid
	checkShare(share)
	# TODO

def newBlockNotification(signum, frame):
	MM.updateMerkleTree()
	# TODO: Force RESPOND TO LONGPOLLS?
	pass

from signal import signal, SIGUSR1
signal(SIGUSR1, newBlockNotification)

from jsonrpcserver import JSONRPCServer
import interactivemode

if __name__ == "__main__":
	server = JSONRPCServer(('', 8444))
	server.getBlockHeader = getBlockHeader
	server.receiveShare = receiveShare
	server.RaiseRedFlags = RaiseRedFlags
	server.serve_forever()
<|endoftext|>


<commit_before># Module:   wsgi
# Date:     6th November 2008
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""WSGI Components

This module implements WSGI Components.
"""

from traceback import format_exc

from circuits.core import listener, Component

import webob
from utils import quoteHTML
from headers import Headers
from dispatchers import Dispatcher
from events import Request, Response, HTTPError
from constants import RESPONSES, DEFAULT_ERROR_MESSAGE

class Application(Component):

   headerNames = {
         "HTTP_CGI_AUTHORIZATION": "Authorization",
         "CONTENT_LENGTH": "Content-Length",
         "CONTENT_TYPE": "Content-Type",
         "REMOTE_HOST": "Remote-Host",
         "REMOTE_ADDR": "Remote-Addr",}

   def __init__(self, *args, **kwargs):
      super(Application, self).__init__(*args, **kwargs)

      self += Dispatcher()

   def translateHeaders(self, environ):
      for cgiName in environ:
         # We assume all incoming header keys are uppercase already.
         if cgiName in self.headerNames:
            yield self.headerNames[cgiName], environ[cgiName]
         elif cgiName[:5] == "HTTP_":
            # Hackish attempt at recovering original header names.
            translatedHeader = cgiName[5:].replace("_", "-")
            yield translatedHeader, environ[cgiName]

   def getRequestResponse(self, environ):
      env = environ.get

      headers = Headers(list(self.translateHeaders(environ)))

      request = webob.Request(
            env("REQUEST_METHOD"),
            env("PATH_INFO"),
            env("SERVER_PROTOCOL"),
            env("QUERY_STRING"))

      request.headers = headers
      request.script_name = env("SCRIPT_NAME")
      request.wsgi_environ = environ
      request.body = env("wsgi.input")

      response = webob.Response(None)
      response.gzip = "gzip" in request.headers.get("Accept-Encoding", "")

      return request, response

   def setError(self, response, code, message=None, traceback=None):
      try:
         short, long = RESPONSES[code]
      except KeyError:
         short, long = "???", "???"

      if message is None:
         message = short

      explain = long

      content = DEFAULT_ERROR_MESSAGE % {
         "code": code,
         "message": quoteHTML(message),
         "explain": explain,
         "traceback": traceback or ""}

      response.body = content
      response.status = "%s %s" % (code, message)
      response.headers.add_header("Connection", "close")

   def __call__(self, environ, start_response):
      request, response = self.getRequestResponse(environ)

      try:
         self.send(Request(request, response), "request", self.channel)
      except:
         error = HTTPError(request, response, 500, error=format_exc())
         self.send(error, "httperror", self.channel)
      finally:
         body = response.process()
         start_response(response.status, response.headers.items())
         return [body]

class Middleware(Component):

   request = None
   response = None

   def __init__(self, app, path=None):
      super(Middleware, self).__init__(channel=path)

      self.app = app

   def environ(self):
      environ = {}
      req = self.request
      env = environ.__setitem__

      env("REQUEST_METHOD", req.method)
      env("PATH_INFO", req.path)
      env("SERVER_PROTOCOL", req.server_protocol)
      env("QUERY_STRING", req.qs)
      env("SCRIPT_NAME", req.script_name)
      env("wsgi.input", req.body)

      return environ

   def start_response(self, status, headers):
      self.response.stats = status
      for header in headers:
         self.response.headers.add_header(*header)

   @listener("request", type="filter")
   def onREQUEST(self, request, response, *args, **kwargs):
      self.request = request
      self.response = response

      response.body = "".join(self.app(self.environ(), self.start_response))

      self.send(Response(response), "response")

class Filter(Component):

   @listener("response", type="filter")
   def onRESPONSE(self, request, response):
      self.request = request
      self.response = response

      try:
         response.body = self.process()
      finally:
         del self.request
         del self.response
<commit_msg>lib.web.wsgi: Fixed import of HTTPError<commit_after># Module:   wsgi
# Date:     6th November 2008
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""WSGI Components

This module implements WSGI Components.
"""

from traceback import format_exc

from circuits.core import listener, Component

import webob
from utils import quoteHTML
from headers import Headers
from errors import HTTPError
from dispatchers import Dispatcher
from events import Request, Response
from constants import RESPONSES, DEFAULT_ERROR_MESSAGE

class Application(Component):

   headerNames = {
         "HTTP_CGI_AUTHORIZATION": "Authorization",
         "CONTENT_LENGTH": "Content-Length",
         "CONTENT_TYPE": "Content-Type",
         "REMOTE_HOST": "Remote-Host",
         "REMOTE_ADDR": "Remote-Addr",}

   def __init__(self, *args, **kwargs):
      super(Application, self).__init__(*args, **kwargs)

      self += Dispatcher()

   def translateHeaders(self, environ):
      for cgiName in environ:
         # We assume all incoming header keys are uppercase already.
         if cgiName in self.headerNames:
            yield self.headerNames[cgiName], environ[cgiName]
         elif cgiName[:5] == "HTTP_":
            # Hackish attempt at recovering original header names.
            translatedHeader = cgiName[5:].replace("_", "-")
            yield translatedHeader, environ[cgiName]

   def getRequestResponse(self, environ):
      env = environ.get

      headers = Headers(list(self.translateHeaders(environ)))

      request = webob.Request(
            env("REQUEST_METHOD"),
            env("PATH_INFO"),
            env("SERVER_PROTOCOL"),
            env("QUERY_STRING"))

      request.headers = headers
      request.script_name = env("SCRIPT_NAME")
      request.wsgi_environ = environ
      request.body = env("wsgi.input")

      response = webob.Response(None)
      response.gzip = "gzip" in request.headers.get("Accept-Encoding", "")

      return request, response

   def setError(self, response, code, message=None, traceback=None):
      try:
         short, long = RESPONSES[code]
      except KeyError:
         short, long = "???", "???"

      if message is None:
         message = short

      explain = long

      content = DEFAULT_ERROR_MESSAGE % {
         "code": code,
         "message": quoteHTML(message),
         "explain": explain,
         "traceback": traceback or ""}

      response.body = content
      response.status = "%s %s" % (code, message)
      response.headers.add_header("Connection", "close")

   def __call__(self, environ, start_response):
      request, response = self.getRequestResponse(environ)

      try:
         self.send(Request(request, response), "request", self.channel)
      except:
         error = HTTPError(request, response, 500, error=format_exc())
         self.send(error, "httperror", self.channel)
      finally:
         body = response.process()
         start_response(response.status, response.headers.items())
         return [body]

class Middleware(Component):

   request = None
   response = None

   def __init__(self, app, path=None):
      super(Middleware, self).__init__(channel=path)

      self.app = app

   def environ(self):
      environ = {}
      req = self.request
      env = environ.__setitem__

      env("REQUEST_METHOD", req.method)
      env("PATH_INFO", req.path)
      env("SERVER_PROTOCOL", req.server_protocol)
      env("QUERY_STRING", req.qs)
      env("SCRIPT_NAME", req.script_name)
      env("wsgi.input", req.body)

      return environ

   def start_response(self, status, headers):
      self.response.stats = status
      for header in headers:
         self.response.headers.add_header(*header)

   @listener("request", type="filter")
   def onREQUEST(self, request, response, *args, **kwargs):
      self.request = request
      self.response = response

      response.body = "".join(self.app(self.environ(), self.start_response))

      self.send(Response(response), "response")

class Filter(Component):

   @listener("response", type="filter")
   def onRESPONSE(self, request, response):
      self.request = request
      self.response = response

      try:
         response.body = self.process()
      finally:
         del self.request
         del self.response
<|endoftext|>


<commit_before>{
  'includes': [ 'common.gypi' ],
  'targets': [
    {
      'target_name': 'route',
      'product_name': 'route',
      'type': 'static_library',
      'sources': [
       'route.cc'
      ],
    },
    {
      'target_name': 'test',
      'type': 'executable',
      'sources': [
        'route.cc', 'test.cc',
      ],
      'dependencies': [
        'route'
      ],
    },
  ],
}

<commit_msg>adds header to sources<commit_after>{
  'includes': [ 'common.gypi' ],
  'targets': [
    {
      'target_name': 'route',
      'product_name': 'route',
      'type': 'static_library',
      'sources': [
       'route.h',
       'route.cc',
      ],
    },
    {
      'target_name': 'test',
      'type': 'executable',
      'sources': [
        'route.cc', 'test.cc',
      ],
      'dependencies': [
        'route'
      ],
    },
  ],
}

<|endoftext|>


<commit_before># Module:   wsgi
# Date:     6th November 2008
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""WSGI Components

This module implements WSGI Components.
"""

from traceback import format_exc

from circuits import handler, Component

import webob
from events import Request
from utils import quoteHTML
from headers import Headers
from errors import HTTPError
from dispatchers import Dispatcher
from constants import RESPONSES, DEFAULT_ERROR_MESSAGE

class Application(Component):

    headerNames = {
            "HTTP_CGI_AUTHORIZATION": "Authorization",
            "CONTENT_LENGTH": "Content-Length",
            "CONTENT_TYPE": "Content-Type",
            "REMOTE_HOST": "Remote-Host",
            "REMOTE_ADDR": "Remote-Addr",
            }

    def __init__(self, *args, **kwargs):
        super(Application, self).__init__(*args, **kwargs)

        self.dispatcher = Dispatcher(**kwargs)
        self.manager += self.dispatcher

    def registered(self, component, manager):
        manager += self.dispatcher

    def translateHeaders(self, environ):
        for cgiName in environ:
            # We assume all incoming header keys are uppercase already.
            if cgiName in self.headerNames:
                yield self.headerNames[cgiName], environ[cgiName]
            elif cgiName[:5] == "HTTP_":
                # Hackish attempt at recovering original header names.
                translatedHeader = cgiName[5:].replace("_", "-")
                yield translatedHeader, environ[cgiName]

    def getRequestResponse(self, environ):
        env = environ.get

        headers = Headers(list(self.translateHeaders(environ)))

        request = webob.Request(
                env("REQUEST_METHOD"),
                env("PATH_INFO"),
                env("SERVER_PROTOCOL"),
                env("QUERY_STRING"))

        request.headers = headers
        request.script_name = env("SCRIPT_NAME")
        request.wsgi_environ = environ
        request.body = env("wsgi.input")

        response = webob.Response(None)
        response.gzip = "gzip" in request.headers.get("Accept-Encoding", "")

        return request, response

    def setError(self, response, status, message=None, traceback=None):
        try:
            short, long = RESPONSES[status]
        except KeyError:
            short, long = "???", "???"

        if message is None:
            message = short

        explain = long

        content = DEFAULT_ERROR_MESSAGE % {
            "status": status,
            "message": quoteHTML(message),
            "traceback": traceback or ""}

        response.body = content
        response.status = "%s %s" % (status, message)
        response.headers.add_header("Connection", "close")

    def __call__(self, environ, start_response):
        request, response = self.getRequestResponse(environ)

        try:
            self.send(Request(request, response), "request", self.channel)
        except:
            error = HTTPError(request, response, 500, error=format_exc())
            self.send(error, "httperror", self.channel)
        finally:
            body = response.process()
            start_response(response.status, response.headers.items())
            return [body]

class Middleware(Component):

    def __init__(self, app, path=None):
        super(Middleware, self).__init__(channel=path)

        self.app = app
        self.request = self.response = None

    def environ(self):
        environ = {}
        req = self.request
        env = environ.__setitem__

        env("REQUEST_METHOD", req.method)
        env("PATH_INFO", req.path)
        env("SERVER_PROTOCOL", req.server_protocol)
        env("QUERY_STRING", req.qs)
        env("SCRIPT_NAME", req.script_name)
        env("wsgi.input", req.body)

        return environ

    def start_response(self, status, headers):
        self.response.stats = status
        for header in headers:
            self.response.headers.add_header(*header)

    @handler("request", filter=True)
    def onREQUEST(self, request, response, *args, **kwargs):
        self.request = request
        self.response = response

        return "".join(self.app(self.environ(), self.start_response))

class Filter(Component):

    @handler("response", filter=True)
    def onRESPONSE(self, request, response):
        self.request = request
        self.response = response

        try:
            response.body = self.process()
        finally:
            del self.request
            del self.response
<commit_msg>lib.web.wsgI: Added in extra imports we need.<commit_after># Module:   wsgi
# Date:     6th November 2008
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""WSGI Components

This module implements WSGI Components.
"""

from traceback import format_exc

from circuits import handler, Component

import webob
from headers import Headers
from errors import HTTPError
from utils import quoteHTML, url
from dispatchers import Dispatcher
from events import Request, Response
from constants import RESPONSES, DEFAULT_ERROR_MESSAGE, SERVER_VERSION

class Application(Component):

    headerNames = {
            "HTTP_CGI_AUTHORIZATION": "Authorization",
            "CONTENT_LENGTH": "Content-Length",
            "CONTENT_TYPE": "Content-Type",
            "REMOTE_HOST": "Remote-Host",
            "REMOTE_ADDR": "Remote-Addr",
            }

    def __init__(self, *args, **kwargs):
        super(Application, self).__init__(*args, **kwargs)

        self.dispatcher = Dispatcher(**kwargs)
        self.manager += self.dispatcher

    def registered(self, component, manager):
        manager += self.dispatcher

    def translateHeaders(self, environ):
        for cgiName in environ:
            # We assume all incoming header keys are uppercase already.
            if cgiName in self.headerNames:
                yield self.headerNames[cgiName], environ[cgiName]
            elif cgiName[:5] == "HTTP_":
                # Hackish attempt at recovering original header names.
                translatedHeader = cgiName[5:].replace("_", "-")
                yield translatedHeader, environ[cgiName]

    def getRequestResponse(self, environ):
        env = environ.get

        headers = Headers(list(self.translateHeaders(environ)))

        request = webob.Request(
                env("REQUEST_METHOD"),
                env("PATH_INFO"),
                env("SERVER_PROTOCOL"),
                env("QUERY_STRING"))

        request.headers = headers
        request.script_name = env("SCRIPT_NAME")
        request.wsgi_environ = environ
        request.body = env("wsgi.input")

        response = webob.Response(None)
        response.gzip = "gzip" in request.headers.get("Accept-Encoding", "")

        return request, response

    def setError(self, response, status, message=None, traceback=None):
        try:
            short, long = RESPONSES[status]
        except KeyError:
            short, long = "???", "???"

        if message is None:
            message = short

        explain = long

        content = DEFAULT_ERROR_MESSAGE % {
            "status": status,
            "message": quoteHTML(message),
            "traceback": traceback or ""}

        response.body = content
        response.status = "%s %s" % (status, message)
        response.headers.add_header("Connection", "close")

    def __call__(self, environ, start_response):
        request, response = self.getRequestResponse(environ)

        try:
            self.send(Request(request, response), "request", self.channel)
        except:
            error = HTTPError(request, response, 500, error=format_exc())
            self.send(error, "httperror", self.channel)
        finally:
            body = response.process()
            start_response(response.status, response.headers.items())
            return [body]

class Middleware(Component):

    def __init__(self, app, path=None):
        super(Middleware, self).__init__(channel=path)

        self.app = app
        self.request = self.response = None

    def environ(self):
        environ = {}
        req = self.request
        env = environ.__setitem__

        env("REQUEST_METHOD", req.method)
        env("PATH_INFO", req.path)
        env("SERVER_PROTOCOL", req.server_protocol)
        env("QUERY_STRING", req.qs)
        env("SCRIPT_NAME", req.script_name)
        env("wsgi.input", req.body)

        return environ

    def start_response(self, status, headers):
        self.response.stats = status
        for header in headers:
            self.response.headers.add_header(*header)

    @handler("request", filter=True)
    def onREQUEST(self, request, response, *args, **kwargs):
        self.request = request
        self.response = response

        return "".join(self.app(self.environ(), self.start_response))

class Filter(Component):

    @handler("response", filter=True)
    def onRESPONSE(self, request, response):
        self.request = request
        self.response = response

        try:
            response.body = self.process()
        finally:
            del self.request
            del self.response
<|endoftext|>


<commit_before>import tensorflow as tf
import numpy as np

class LMSystem(object):

    def __init__(self, _parms):
        self.parms = _parms
        # tf Graph
        # input
        self.x = tf.placeholder("float", [_parms.n_steps, None, _parms.n_input])
        # desired output
        self.y = tf.placeholder("float", [None, _parms.n_classes])
        # Tensorflow LSTM cell requires 2x n_hidden length (state & cell)
        self.i_state = tf.placeholder("float", [None, 2 * _parms.n_hidden])

        # Define weights
        self.weights = {
            'hidden': tf.Variable(tf.random_normal([_parms.n_input, _parms.n_hidden])),  # Hidden layer weights
            'out': tf.Variable(tf.random_normal([_parms.n_hidden, _parms.n_classes]))
        }
        self.biases = {
            'hidden': tf.Variable(tf.random_normal([_parms.n_hidden])),
            'out': tf.Variable(tf.random_normal([_parms.n_classes]))
        }

        # Define the RNN for the language model
        def lm_rnn(_x, _i_state, _weights, _biases, s_parms):
            # Reformat _x from [ ] to [n_steps*batch_size x n_input]
            xin = tf.reshape(_x, [-1, _parms.n_input])

            # Linear activation
            xin = tf.matmul(xin, _weights['hidden']) + _biases['hidden']

            # Split data because rnn cell needs a list of inputs for the RNN inner loop
            xin = tf.split(0, s_parms.n_steps, xin)  # n_steps * (batch_size, n_hidden)

            # Define a lstm cell with tensorflow
            lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(s_parms.n_hidden, forget_bias=0.95)

            # Get lstm cell output
            # outputs - a list of n_step matrix of shape [? x n_hidden]
            # states - a list of n_step vectors of size [2*n_hidden]
            outputs, states = tf.nn.rnn(lstm_cell, xin, initial_state=_i_state)

            # Linear activation
            # Get inner loop last output
            logits = tf.matmul(outputs[-1], _weights['out']) + _biases['out']

            return logits

        # predict the input x using the state and weights
        self.predictions = lm_rnn(self.x, self.i_state, self.weights, self.biases, self.parms)

        # Define loss and optimizer
        self.cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(self.predictions, self.y))  # Softmax loss
        self.optimizer = tf.train.AdamOptimizer(learning_rate=self.parms.learning_rate).minimize(self.cost)

        # Evaluate model
        self.correct_predictions = tf.equal(tf.argmax(self.predictions, 1), tf.argmax(self.y, 1))
        self.accuracy = tf.reduce_mean(tf.cast(self.correct_predictions, tf.float32))

        # Initializing the variables
        self.init = tf.initialize_all_variables()

    def train(self, lang_data, _model_file_name):
        saver = tf.train.Saver()
        # Launch the graph
        with tf.Session() as sess:
            sess.run(self.init)
            step = 1
            model_count = 0
            # Keep training until reach max iterations
            while step * self.parms.batch_size < self.parms.training_cycles:
                # batch_xs - list (of length batch_size) of strings each of length n_input,
                # batch_ys - list (of length batch_size) of language ids (strings, example: 'bg','ep',...)
                batch_xs, batch_ys = lang_data.get_next_batch_one_hot(self.parms.batch_size, self.parms.n_steps)

                # Fit training using batch data
                sess.run(self.optimizer, feed_dict={self.x: batch_xs, self.y: batch_ys,
                                                    self.i_state: np.zeros((self.parms.batch_size,
                                                                            2 * self.parms.n_hidden))})
                if step % self.parms.display_step == 0:
                    # Calculate batch accuracy
                    acc = sess.run(self.accuracy, feed_dict={self.x: batch_xs, self.y: batch_ys,
                                                             self.i_state: np.zeros((self.parms.batch_size,
                                                                                2 * self.parms.n_hidden))})
                    # Calculate batch loss
                    loss = sess.run(self.cost, feed_dict={self.x: batch_xs, self.y: batch_ys,
                                                          self.i_state: np.zeros((self.parms.batch_size,
                                                                                2 * self.parms.n_hidden))})
                    # Read a new batch for validation
                    batch_xs, batch_ys = lang_data.get_next_batch_one_hot(self.parms.batch_size, self.parms.n_steps)

                    acc_val = sess.run(self.accuracy, feed_dict={self.x: batch_xs, self.y: batch_ys,
                                                                 self.i_state: np.zeros((self.parms.batch_size,
                                                                                2 * self.parms.n_hidden))})
                    # Calculate batch loss
                    loss_val = sess.run(self.cost, feed_dict={self.x: batch_xs, self.y: batch_ys,
                                                              self.i_state: np.zeros((self.parms.batch_size,
                                                                                2 * self.parms.n_hidden))})
                    print("Iter " + str(step*self.parms.batch_size) + ", Minibatch Loss= " +
                          "{:.6f}".format(loss) +
                          ", Training Accuracy= " + "{:.5f}".format(acc) + ", Validation Loss=" +
                          "{:.6f}".format(loss_val) + ", Validation Accuracy= " + "{:.5f}".format(acc_val))

                    # Save model periodically
                    if step % self.parms.model_step == 0:
                        model_count += 1
                        model_save_name = _model_file_name + '_' + str(model_count)
                        saver.save(sess, model_save_name)
                        print('model saved:', model_save_name)

                step += 1
            print("Optimization Finished!")
            saver.save(sess, _model_file_name + '_final')

    def test(self, _test_data, _test_label,  _model_file_name):
            saver = tf.train.Saver()
            test_len = len(_test_label)
            # Calculate accuracy
            with tf.Session() as sess:
                saver.restore(sess, _model_file_name)
                test_batch_size = 1000
                acc_sum = 0
                b_count = int(test_len/test_batch_size)

                for i in range(b_count):
                    test_batch_x = _test_data[:, i*test_batch_size:(i+1)*test_batch_size, :]
                    test_batch_y = _test_label[i*test_batch_size:(i+1)*test_batch_size]

                    acc_sum += sess.run(self.accuracy,
                                        feed_dict={self.x: test_batch_x, self.y: test_batch_y,
                                                   self.i_state: np.zeros((test_batch_size, 2 * self.parms.n_hidden))})
                acc_avg = acc_sum / b_count
                print("Testing Accuracy:", acc_avg)
<commit_msg>changed save model naming method. now models are 1,2,3,...<commit_after>"""
  A class for training and testing Recurrent Neural Network language identification models.
"""
import tensorflow as tf
import numpy as np

class LMSystem(object):

    def __init__(self, _parms):
        self.parms = _parms
        # tf Graph
        # input
        self.x = tf.placeholder("float", [_parms.n_steps, None, _parms.n_input])
        # desired output
        self.y = tf.placeholder("float", [None, _parms.n_classes])
        # Tensorflow LSTM cell requires 2x n_hidden length (state & cell)
        self.i_state = tf.placeholder("float", [None, 2 * _parms.n_hidden])

        # Define weights
        self.weights = {
            'hidden': tf.Variable(tf.random_normal([_parms.n_input, _parms.n_hidden])),  # Hidden layer weights
            'out': tf.Variable(tf.random_normal([_parms.n_hidden, _parms.n_classes]))
        }
        self.biases = {
            'hidden': tf.Variable(tf.random_normal([_parms.n_hidden])),
            'out': tf.Variable(tf.random_normal([_parms.n_classes]))
        }

        # Define the RNN for the language model
        def lm_rnn(_x, _i_state, _weights, _biases, s_parms):
            # Reformat _x from [ ] to [n_steps*batch_size x n_input]
            xin = tf.reshape(_x, [-1, _parms.n_input])

            # Linear activation
            xin = tf.matmul(xin, _weights['hidden']) + _biases['hidden']

            # Split data because rnn cell needs a list of inputs for the RNN inner loop
            xin = tf.split(0, s_parms.n_steps, xin)  # n_steps * (batch_size, n_hidden)

            # Define a lstm cell with tensorflow
            lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(s_parms.n_hidden, forget_bias=0.95)

            # Get lstm cell output
            # outputs - a list of n_step matrix of shape [? x n_hidden]
            # states - a list of n_step vectors of size [2*n_hidden]
            outputs, states = tf.nn.rnn(lstm_cell, xin, initial_state=_i_state)

            # Linear activation
            # Get inner loop last output
            logits = tf.matmul(outputs[-1], _weights['out']) + _biases['out']

            return logits

        # predict the input x using the state and weights
        self.predictions = lm_rnn(self.x, self.i_state, self.weights, self.biases, self.parms)

        # Define loss and optimizer
        self.cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(self.predictions, self.y))  # Softmax loss
        self.optimizer = tf.train.AdamOptimizer(learning_rate=self.parms.learning_rate).minimize(self.cost)

        # Evaluate model
        self.correct_predictions = tf.equal(tf.argmax(self.predictions, 1), tf.argmax(self.y, 1))
        self.accuracy = tf.reduce_mean(tf.cast(self.correct_predictions, tf.float32))

        # Initializing the variables
        self.init = tf.initialize_all_variables()

    def train(self, lang_data, _model_file_name):
        """ Train the model on a data set
        :param lang_data: a LanguageSource object for the training data.
        :param _model_file_name: the base name for the output model file.
        :return: nothing
        """
        saver = tf.train.Saver()
        # Launch the graph
        with tf.Session() as sess:
            sess.run(self.init)
            step = 1
            model_count = 0  # name models with this variable
            # Keep training until reach max iterations
            while step * self.parms.batch_size < self.parms.training_cycles:
                # batch_xs - list (of length batch_size) of strings each of length n_input,
                # batch_ys - list (of length batch_size) of language ids (strings, example: 'bg','ep',...)
                batch_xs, batch_ys = lang_data.get_next_batch_one_hot(self.parms.batch_size, self.parms.n_steps)

                # Fit training using batch data
                sess.run(self.optimizer, feed_dict={self.x: batch_xs, self.y: batch_ys,
                                                    self.i_state: np.zeros((self.parms.batch_size,
                                                                            2 * self.parms.n_hidden))})
                if step % self.parms.display_step == 0:
                    # Calculate batch accuracy
                    acc = sess.run(self.accuracy, feed_dict={self.x: batch_xs, self.y: batch_ys,
                                                             self.i_state: np.zeros((self.parms.batch_size,
                                                                                2 * self.parms.n_hidden))})
                    # Calculate batch loss
                    loss = sess.run(self.cost, feed_dict={self.x: batch_xs, self.y: batch_ys,
                                                          self.i_state: np.zeros((self.parms.batch_size,
                                                                                2 * self.parms.n_hidden))})
                    # Read a new batch for validation
                    batch_xs, batch_ys = lang_data.get_next_batch_one_hot(self.parms.batch_size, self.parms.n_steps)

                    acc_val = sess.run(self.accuracy, feed_dict={self.x: batch_xs, self.y: batch_ys,
                                                                 self.i_state: np.zeros((self.parms.batch_size,
                                                                                2 * self.parms.n_hidden))})
                    # Calculate batch loss
                    loss_val = sess.run(self.cost, feed_dict={self.x: batch_xs, self.y: batch_ys,
                                                              self.i_state: np.zeros((self.parms.batch_size,
                                                                                2 * self.parms.n_hidden))})
                    print("Iter " + str(step*self.parms.batch_size) + ", Minibatch Loss= " +
                          "{:.6f}".format(loss) +
                          ", Training Accuracy= " + "{:.5f}".format(acc) + ", Validation Loss=" +
                          "{:.6f}".format(loss_val) + ", Validation Accuracy= " + "{:.5f}".format(acc_val))

                    # Save model periodically
                    if step % self.parms.model_step == 0:
                        model_count += 1
                        model_save_name = _model_file_name + '_' + str(model_count)
                        saver.save(sess, model_save_name)
                        print('model saved:', model_save_name)

                step += 1
            print("Optimization Finished!")
            saver.save(sess, _model_file_name + '_final')

    def test(self, _test_data, _test_label,  _model_file_name):
        """ Test a model on a test data.
        :param _test_data: a numpy array of test data
        :param _test_label: class labels for the test data
        :param _model_file_name: the name of the model file to test with.
        :return: nothing.
        """
        saver = tf.train.Saver()
        test_len = len(_test_label)
        # Calculate accuracy
        with tf.Session() as sess:
            saver.restore(sess, _model_file_name)
            test_batch_size = 1000
            acc_sum = 0
            b_count = int(test_len/test_batch_size)

            for i in range(b_count):
                test_batch_x = _test_data[:, i*test_batch_size:(i+1)*test_batch_size, :]
                test_batch_y = _test_label[i*test_batch_size:(i+1)*test_batch_size]

                acc_sum += sess.run(self.accuracy,
                                    feed_dict={self.x: test_batch_x, self.y: test_batch_y,
                                               self.i_state: np.zeros((test_batch_size, 2 * self.parms.n_hidden))})
            acc_avg = acc_sum / b_count
            print("Testing Accuracy:", acc_avg)
<|endoftext|>