<commit_before>"""
    flaskext.frozen
    ~~~~~~~~~~~~~~~

    Frozen-Flask freezes a Flask application into a set of static files.
    The result can be hosted without any server-side software other than a
    traditional web server.

    :copyright: (c) 2010 by Simon Sapin.
    :license: BSD, see LICENSE for more details.
"""

from __future__ import with_statement

import os.path
import mimetypes
import urlparse
import urllib
import warnings
from unicodedata import normalize

from werkzeug.exceptions import HTTPException
from flask import Flask, Module, url_for, request, send_from_directory

try:
    from collections import Mapping
    def is_mapping(obj):
        return isinstance(obj, Mapping)
except ImportError:
    # Python 2.5, no Abstract Base Classes. Default to duck-typing.
    def is_mapping(obj):
        return hasattr(obj, 'keys')

__all__ = ['Freezer']

class MissingURLGeneratorWarning(Warning):
    pass

class Freezer(object):
    """
    :param app: your application or None if you use :meth:`init_app`
    :type app: Flask instance
    :param with_static_files: Whether to automatically generate URLs
                              for static files.
    :type with_static_files boolean:
    :param with_no_argument_rules: Whether to automatically generate URLs
                                   for URL rules that take no arguments.
    :type with_no_argument_rules boolean:
    """
    def __init__(self, app=None, with_static_files=True,
                 with_no_argument_rules=True):
        self.url_generators = []
        if with_static_files:
            self.register_generator(self.static_files_urls)
        if with_no_argument_rules:
            self.register_generator(self.no_argument_rules_urls)
        self.init_app(app)

    def init_app(self, app):
        """
        Allow to register an app after the Freezer initialization.

        :param app: your Flask application
        """
        self.app = app
        if app:
            app.config.setdefault('FREEZER_DESTINATION', 'build')
            app.config.setdefault('FREEZER_BASE_URL', 'http://localhost/')
            app.config.setdefault('FREEZER_REMOVE_EXTRA_FILES', True)

    def register_generator(self, function):
        """Register a function as an URL generator.

        The function should return an iterable of URL paths or
        ``(endpoint, values)`` tuples to be used as
        ``url_for(endpoint, **values)``.

        :Returns: the function, so that it can be used as a decorator
        """
        self.url_generators.append(function)
        # Allow use as a decorator
        return function

    @property
    def root(self):
        """The build destination."""
        # unicode() will raise if the path is not ASCII or already unicode.
        return os.path.join(
            unicode(self.app.root_path),
            unicode(self.app.config['FREEZER_DESTINATION'])
        )

    def freeze(self):
        """Clean the destination and build all URLs from generators."""
        remove_extra = self.app.config['FREEZER_REMOVE_EXTRA_FILES']
        if not os.path.isdir(self.root):
            os.makedirs(self.root)
        previous_files = set(
            # See https://github.com/SimonSapin/Frozen-Flask/issues/5
            normalize('NFC', os.path.join(self.root, *name.split('/')))
            for name in walk_directory(self.root)
        )
        seen_urls = set()
        seen_endpoints = set()
        built_files = set()
        for url, endpoint in self._generate_all_urls():
            seen_endpoints.add(endpoint)
            if url in seen_urls:
                # Don't build the same URL more than once
                continue
            seen_urls.add(url)
            new_filename = self._build_one(url)
            built_files.add(normalize('NFC', new_filename))
        self._check_endpoints(seen_endpoints)
        if remove_extra:
            # Remove files from the previous build that are not here anymore.
            for extra_file in previous_files - built_files:
                os.remove(extra_file)
                parent = os.path.dirname(extra_file)
                if not os.listdir(parent):
                    # The directory is now empty, remove it.
                    os.removedirs(parent)
        return seen_urls

    def all_urls(self):
        """
        Run all generators and yield URLs relative to the app root.
        May be useful for testing URL generators.
        """
        for url, _endpoint in self._generate_all_urls():
            yield url

    def _generate_all_urls(self):
        """
        Run all generators and yield (url, enpoint) tuples.
        """
        base_url = self.app.config['FREEZER_BASE_URL']
        script_name = urlparse.urlsplit(base_url).path.rstrip('/')
        url_encoding = self.app.url_map.charset
        # A request context is required to use url_for
        with self.app.test_request_context(base_url=script_name):
            for generator in self.url_generators:
                for generated in generator():
                    if isinstance(generated, basestring):
                        url = generated
                        endpoint = None
                    else:
                        if is_mapping(generated):
                            values = generated
                            # The endpoint defaults to the name of the
                            # generator function, just like with Flask views.
                            endpoint = generator.__name__
                        else:
                            # Assume a tuple.
                            endpoint, values = generated
                        url = url_for(endpoint, **values)
                        assert url.startswith(script_name), (
                            'url_for returned an URL %r not starting with '
                            'script_name %r. Bug in Werkzeug?'
                            % (url, script_name)
                        )
                        url = url[len(script_name):]
                    # flask.url_for "quotes" URLs, eg. a space becomes %20
                    url = urllib.unquote(url)
                    parsed_url = urlparse.urlsplit(url)
                    if parsed_url.scheme or parsed_url.netloc:
                        raise ValueError('External URLs not supported: ' + url)

                    if not isinstance(url, unicode):
                        url = url.decode(url_encoding)
                    yield url, endpoint

    def _check_endpoints(self, seen_endpoints):
        """
        Warn if some of the app's enpoints are not in seen_endpoints.
        """
        all_endpoints = set(
            rule.endpoint for rule in self.app.url_map.iter_rules())
        not_generated_endpoints = all_endpoints - seen_endpoints

        if self.static_files_urls in self.url_generators:
            # Special case: do not warn when there is no static file
            not_generated_endpoints -= set(self._static_rules_endpoints())

        if not_generated_endpoints:
            warnings.warn(
                'Nothing frozen for endpoints %s. Did you forget an URL '
                'generator?' % ', '.join(
                    unicode(e) for e in not_generated_endpoints),
                MissingURLGeneratorWarning,
                stacklevel=3)

    def _build_one(self, url):
        """Get the given ``url`` from the app and write the matching file.
        """
        client = self.app.test_client()
        base_url = self.app.config['FREEZER_BASE_URL']
        response = client.get(url, follow_redirects=True, base_url=base_url)
        # The client follows redirects by itself
        # Any other status code is probably an error
        if not(response.status_code == 200):
            raise ValueError('Unexpected status %r on URL %s' \
                % (response.status, url))

        destination_path = url + 'index.html' if url.endswith('/') else url

        # Most web servers guess the mime type of static files by their
        # filename.  Check that this guess is consistent with the actual
        # Content-Type header we got from the app.
        basename = destination_path.rsplit('/', 1)[-1]
        guessed_type, guessed_encoding = mimetypes.guess_type(basename)
        if not guessed_type:
            # Used by most server when they can not determine the type
            guessed_type = 'application/octet-stream'
        if not guessed_type == response.mimetype:
            raise ValueError(
                'Filename extension of %r (type %s) does not match Content-'
                'Type: %s' % (basename, guessed_type, response.content_type))

        # Remove the initial slash that should always be there
        assert destination_path[0] == '/'
        destination_path = destination_path[1:]

        filename = os.path.join(self.root, *destination_path.split('/'))
        dirname = os.path.dirname(filename)
        if not os.path.isdir(dirname):
            os.makedirs(dirname)
        content = response.data
        if os.path.isfile(filename):
            with open(filename, 'rb') as fd:
                previous_content = fd.read()
        else:
            previous_content = None
        if content != previous_content:
            # Do not overwrite when content hasn't changed to help rsync
            # by keeping the modification date.
            with open(filename, 'wb') as fd:
                fd.write(content)
        return filename

    def serve(self, **options):
        """Run an HTTP server on the result of the build.

        :param options: passed to ``app.run()``.
        """
        self.make_static_app().run(**options)

    def make_static_app(self):
        """Return a Flask application serving the build destination."""
        root = os.path.join(
            self.app.root_path,
            self.app.config['FREEZER_DESTINATION']
        )

        def dispatch_request():
            try:
                # request.path is unicode, but the freezer previously wrote
                # to UTF-8 filenames, so encode to UTF-8 now.
                path = request.path.encode('utf8')
                if path.endswith('/'):
                    path += 'index.html'
                assert path.startswith('/')
                # Disable etags because of a Flask bug:
                # https://github.com/mitsuhiko/flask/pull/237
                # They are useless for tests anyway.
                return send_from_directory(root, path[1:])
            except HTTPException, e:
                # eg. NotFound
                return e

        app = Flask(__name__)
        # Do not use the URL map
        app.dispatch_request = dispatch_request
        return app

    def _static_rules_endpoints(self):
        """
        Yield the 'static' URL rules for the app and all modules.
        """
        send_static_file = unwrap_method(Flask.send_static_file)
        # Assumption about a Flask internal detail:
        # Flask and Module inherit the same method.
        # This will break loudly if the assumption isn't valid anymore in
        # a future version of Flask
        assert unwrap_method(Module.send_static_file) is send_static_file

        for rule in self.app.url_map.iter_rules():
            view = self.app.view_functions[rule.endpoint]
            if unwrap_method(view) is send_static_file:
                yield rule.endpoint

    def static_files_urls(self):
        """URL generator for static files for app and all registered modules.
        """
        for endpoint in self._static_rules_endpoints():
            view = self.app.view_functions[endpoint]
            app_or_module = method_self(view)
            root = app_or_module.static_folder
            if root is None or not os.path.isdir(root):
                # No 'static' directory for this app/module.
                continue
            for filename in walk_directory(root):
                yield endpoint, {'filename': filename}

    def no_argument_rules_urls(self):
        """URL generator for URL rules that take no arguments."""
        for rule in self.app.url_map.iter_rules():
            if not rule.arguments:
                yield rule.endpoint, {}

def walk_directory(root):
    """
    Recursively walk the `root` directory and yield slash-separated paths
    relative to the root.

    Used to implement the URL genertor for static files.
    """
    for name in os.listdir(root):
        full_name = os.path.join(root, name)
        if os.path.isdir(full_name):
            for filename in walk_directory(full_name):
                yield name + '/' + filename
        elif os.path.isfile(full_name):
            yield name

def unwrap_method(method):
    """Return the function object for the given method object."""
    try:
        # Python 2
        return method.im_func
    except AttributeError:
        try:
            # Python 3
            return method.__func__
        except AttributeError:
            # Not a method.
            return method

def method_self(method):
    """Return the instance a bound method is attached to."""
    try:
        # Python 2
        return method.im_self
    except AttributeError:
        # Python 3
        return method.__self__
<commit_msg>Factorize common code in _build_one and make_static_app.<commit_after>"""
    flaskext.frozen
    ~~~~~~~~~~~~~~~

    Frozen-Flask freezes a Flask application into a set of static files.
    The result can be hosted without any server-side software other than a
    traditional web server.

    :copyright: (c) 2010 by Simon Sapin.
    :license: BSD, see LICENSE for more details.
"""

from __future__ import with_statement

import os.path
import mimetypes
import urlparse
import urllib
import warnings
from unicodedata import normalize

from werkzeug.exceptions import HTTPException
from flask import Flask, Module, url_for, request, send_from_directory

try:
    from collections import Mapping
    def is_mapping(obj):
        return isinstance(obj, Mapping)
except ImportError:
    # Python 2.5, no Abstract Base Classes. Default to duck-typing.
    def is_mapping(obj):
        return hasattr(obj, 'keys')

__all__ = ['Freezer']

class MissingURLGeneratorWarning(Warning):
    pass

class Freezer(object):
    """
    :param app: your application or None if you use :meth:`init_app`
    :type app: Flask instance
    :param with_static_files: Whether to automatically generate URLs
                              for static files.
    :type with_static_files boolean:
    :param with_no_argument_rules: Whether to automatically generate URLs
                                   for URL rules that take no arguments.
    :type with_no_argument_rules boolean:
    """
    def __init__(self, app=None, with_static_files=True,
                 with_no_argument_rules=True):
        self.url_generators = []
        if with_static_files:
            self.register_generator(self.static_files_urls)
        if with_no_argument_rules:
            self.register_generator(self.no_argument_rules_urls)
        self.init_app(app)

    def init_app(self, app):
        """
        Allow to register an app after the Freezer initialization.

        :param app: your Flask application
        """
        self.app = app
        if app:
            app.config.setdefault('FREEZER_DESTINATION', 'build')
            app.config.setdefault('FREEZER_BASE_URL', 'http://localhost/')
            app.config.setdefault('FREEZER_REMOVE_EXTRA_FILES', True)

    def register_generator(self, function):
        """Register a function as an URL generator.

        The function should return an iterable of URL paths or
        ``(endpoint, values)`` tuples to be used as
        ``url_for(endpoint, **values)``.

        :Returns: the function, so that it can be used as a decorator
        """
        self.url_generators.append(function)
        # Allow use as a decorator
        return function

    @property
    def root(self):
        """The build destination."""
        # unicode() will raise if the path is not ASCII or already unicode.
        return os.path.join(
            unicode(self.app.root_path),
            unicode(self.app.config['FREEZER_DESTINATION'])
        )

    def freeze(self):
        """Clean the destination and build all URLs from generators."""
        remove_extra = self.app.config['FREEZER_REMOVE_EXTRA_FILES']
        if not os.path.isdir(self.root):
            os.makedirs(self.root)
        previous_files = set(
            # See https://github.com/SimonSapin/Frozen-Flask/issues/5
            normalize('NFC', os.path.join(self.root, *name.split('/')))
            for name in walk_directory(self.root)
        )
        seen_urls = set()
        seen_endpoints = set()
        built_files = set()
        for url, endpoint in self._generate_all_urls():
            seen_endpoints.add(endpoint)
            if url in seen_urls:
                # Don't build the same URL more than once
                continue
            seen_urls.add(url)
            new_filename = self._build_one(url)
            built_files.add(normalize('NFC', new_filename))
        self._check_endpoints(seen_endpoints)
        if remove_extra:
            # Remove files from the previous build that are not here anymore.
            for extra_file in previous_files - built_files:
                os.remove(extra_file)
                parent = os.path.dirname(extra_file)
                if not os.listdir(parent):
                    # The directory is now empty, remove it.
                    os.removedirs(parent)
        return seen_urls

    def all_urls(self):
        """
        Run all generators and yield URLs relative to the app root.
        May be useful for testing URL generators.
        """
        for url, _endpoint in self._generate_all_urls():
            yield url

    def _generate_all_urls(self):
        """
        Run all generators and yield (url, enpoint) tuples.
        """
        base_url = self.app.config['FREEZER_BASE_URL']
        script_name = urlparse.urlsplit(base_url).path.rstrip('/')
        url_encoding = self.app.url_map.charset
        # A request context is required to use url_for
        with self.app.test_request_context(base_url=script_name):
            for generator in self.url_generators:
                for generated in generator():
                    if isinstance(generated, basestring):
                        url = generated
                        endpoint = None
                    else:
                        if is_mapping(generated):
                            values = generated
                            # The endpoint defaults to the name of the
                            # generator function, just like with Flask views.
                            endpoint = generator.__name__
                        else:
                            # Assume a tuple.
                            endpoint, values = generated
                        url = url_for(endpoint, **values)
                        assert url.startswith(script_name), (
                            'url_for returned an URL %r not starting with '
                            'script_name %r. Bug in Werkzeug?'
                            % (url, script_name)
                        )
                        url = url[len(script_name):]
                    # flask.url_for "quotes" URLs, eg. a space becomes %20
                    url = urllib.unquote(url)
                    parsed_url = urlparse.urlsplit(url)
                    if parsed_url.scheme or parsed_url.netloc:
                        raise ValueError('External URLs not supported: ' + url)

                    if not isinstance(url, unicode):
                        url = url.decode(url_encoding)
                    yield url, endpoint

    def _check_endpoints(self, seen_endpoints):
        """
        Warn if some of the app's enpoints are not in seen_endpoints.
        """
        all_endpoints = set(
            rule.endpoint for rule in self.app.url_map.iter_rules())
        not_generated_endpoints = all_endpoints - seen_endpoints

        if self.static_files_urls in self.url_generators:
            # Special case: do not warn when there is no static file
            not_generated_endpoints -= set(self._static_rules_endpoints())

        if not_generated_endpoints:
            warnings.warn(
                'Nothing frozen for endpoints %s. Did you forget an URL '
                'generator?' % ', '.join(
                    unicode(e) for e in not_generated_endpoints),
                MissingURLGeneratorWarning,
                stacklevel=3)

    def _build_one(self, url):
        """Get the given ``url`` from the app and write the matching file.
        """
        client = self.app.test_client()
        base_url = self.app.config['FREEZER_BASE_URL']
        response = client.get(url, follow_redirects=True, base_url=base_url)
        # The client follows redirects by itself
        # Any other status code is probably an error
        if not(response.status_code == 200):
            raise ValueError('Unexpected status %r on URL %s' \
                % (response.status, url))

        destination_path = self.urlpath_to_filepath(url)
        filename = os.path.join(self.root, *destination_path.split('/'))

        # Most web servers guess the mime type of static files by their
        # filename.  Check that this guess is consistent with the actual
        # Content-Type header we got from the app.
        basename = os.path.basename(filename)
        guessed_type, guessed_encoding = mimetypes.guess_type(basename)
        if not guessed_type:
            # Used by most server when they can not determine the type
            guessed_type = 'application/octet-stream'
        if not guessed_type == response.mimetype:
            raise ValueError(
                'Filename extension of %r (type %s) does not match Content-'
                'Type: %s' % (basename, guessed_type, response.content_type))

        # Create directories as needed
        dirname = os.path.dirname(filename)
        if not os.path.isdir(dirname):
            os.makedirs(dirname)

        # Write the file, but only if its content has changed
        content = response.data
        if os.path.isfile(filename):
            with open(filename, 'rb') as fd:
                previous_content = fd.read()
        else:
            previous_content = None
        if content != previous_content:
            # Do not overwrite when content hasn't changed to help rsync
            # by keeping the modification date.
            with open(filename, 'wb') as fd:
                fd.write(content)

        return filename

    def urlpath_to_filepath(self, path):
        """
        Convert a URL path like /admin/ to a file path like admin/index.html
        """
        if path.endswith('/'):
            path += 'index.html'
        # Remove the initial slash that should always be there
        assert path.startswith('/')
        return path[1:]

    def serve(self, **options):
        """Run an HTTP server on the result of the build.

        :param options: passed to ``app.run()``.
        """
        self.make_static_app().run(**options)

    def make_static_app(self):
        """Return a Flask application serving the build destination."""
        root = os.path.join(
            self.app.root_path,
            self.app.config['FREEZER_DESTINATION']
        )

        def dispatch_request():
            filename = self.urlpath_to_filepath(request.path)
            return send_from_directory(root, filename)

        app = Flask(__name__)
        # Do not use the URL map
        app.dispatch_request = dispatch_request
        return app

    def _static_rules_endpoints(self):
        """
        Yield the 'static' URL rules for the app and all modules.
        """
        send_static_file = unwrap_method(Flask.send_static_file)
        # Assumption about a Flask internal detail:
        # Flask and Module inherit the same method.
        # This will break loudly if the assumption isn't valid anymore in
        # a future version of Flask
        assert unwrap_method(Module.send_static_file) is send_static_file

        for rule in self.app.url_map.iter_rules():
            view = self.app.view_functions[rule.endpoint]
            if unwrap_method(view) is send_static_file:
                yield rule.endpoint

    def static_files_urls(self):
        """URL generator for static files for app and all registered modules.
        """
        for endpoint in self._static_rules_endpoints():
            view = self.app.view_functions[endpoint]
            app_or_module = method_self(view)
            root = app_or_module.static_folder
            if root is None or not os.path.isdir(root):
                # No 'static' directory for this app/module.
                continue
            for filename in walk_directory(root):
                yield endpoint, {'filename': filename}

    def no_argument_rules_urls(self):
        """URL generator for URL rules that take no arguments."""
        for rule in self.app.url_map.iter_rules():
            if not rule.arguments:
                yield rule.endpoint, {}

def walk_directory(root):
    """
    Recursively walk the `root` directory and yield slash-separated paths
    relative to the root.

    Used to implement the URL genertor for static files.
    """
    for name in os.listdir(root):
        full_name = os.path.join(root, name)
        if os.path.isdir(full_name):
            for filename in walk_directory(full_name):
                yield name + '/' + filename
        elif os.path.isfile(full_name):
            yield name

def unwrap_method(method):
    """Return the function object for the given method object."""
    try:
        # Python 2
        return method.im_func
    except AttributeError:
        try:
            # Python 3
            return method.__func__
        except AttributeError:
            # Not a method.
            return method

def method_self(method):
    """Return the instance a bound method is attached to."""
    try:
        # Python 2
        return method.im_self
    except AttributeError:
        # Python 3
        return method.__self__
<|endoftext|>


<commit_before>"""Quick script to read inputs.
"""

if __name__ == '__main__':
    # Read the number of inputs
    num_inputs = int(input("How many inputs? "))
    assert num_inputs > 3, "At least 3 please."
    print("Enter your {} inputs in the following form: inp1 inp2 ... inp{}".format(
        num_inputs, num_inputs))
    a = list(map(int, input().split()))
    assert(len(a) == num_inputs)
    print(a)
<commit_msg>Fix bug with input number validation<commit_after>"""Quick script to read inputs.
"""

if __name__ == '__main__':
    # Read the number of inputs
    num_inputs = int(input("How many inputs? "))
    assert num_inputs >= 3, "At least 3 please."
    print("Enter your {} inputs in the following form: inp1 inp2 ... inp{}".format(
        num_inputs, num_inputs))
    a = list(map(int, input().split()))
    assert(len(a) == num_inputs)
    print(a)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# -----------------------------------------------------------------------------
# Copyright (c) 2017 The Regents of the University of California
#
# This file is part of kevlar (http://github.com/standage/kevlar) and is
# licensed under the MIT license: see LICENSE.
# -----------------------------------------------------------------------------

import pytest
import kevlar

def test_help(capsys):
    with pytest.raises(SystemExit):
        kevlar.cli.parser().parse_args(['-h'])
    out, err = capsys.readouterr()
    msg = 'show this help message and exit'
    assert msg in out or msg in err

def test_version(capsys):
    with pytest.raises(SystemExit):
        kevlar.cli.parser().parse_args(['-v'])
    out, err = capsys.readouterr()
    assert kevlar.__version__ in out

@pytest.mark.parametrize('subcommand', [s for s in kevlar.cli.mains])
def test_help_sub(subcommand, capsys):
    with pytest.raises(SystemExit):
        kevlar.cli.parser().parse_args([subcommand, '-h'])
    out, err = capsys.readouterr()
    assert subcommand in out
    assert 'show this help message and exit' in out
<commit_msg>Py2/Py3 stdout/stderr incompatibility?<commit_after>#!/usr/bin/env python
#
# -----------------------------------------------------------------------------
# Copyright (c) 2017 The Regents of the University of California
#
# This file is part of kevlar (http://github.com/standage/kevlar) and is
# licensed under the MIT license: see LICENSE.
# -----------------------------------------------------------------------------

import pytest
import kevlar

def test_help(capsys):
    with pytest.raises(SystemExit):
        kevlar.cli.parser().parse_args(['-h'])
    out, err = capsys.readouterr()
    assert 'show this help message and exit' in out

def test_version(capsys):
    with pytest.raises(SystemExit):
        kevlar.cli.parser().parse_args(['-v'])
    out, err = capsys.readouterr()
    assert kevlar.__version__ in out or kevlar.__version__ in err

@pytest.mark.parametrize('subcommand', [s for s in kevlar.cli.mains])
def test_help_sub(subcommand, capsys):
    with pytest.raises(SystemExit):
        kevlar.cli.parser().parse_args([subcommand, '-h'])
    out, err = capsys.readouterr()
    assert subcommand in out
    assert 'show this help message and exit' in out
<|endoftext|>


<commit_before>import os
import sys
import types
import inspect
import hashlib
import tempfile
import subprocess

from . import Parser
from .stdlib import get_full_std_lib  # noqa
from .modules import create_js_module

class JSString(str):
    """ A subclass of string, so we can add attributes to JS string objects.
    """
    pass

def py2js(ob=None, new_name=None, **parser_options):
    """ Convert Python to JavaScript.

    Parameters:
        ob (str, module, function, class): The code, function or class
            to transpile.
        new_name (str, optional): If given, renames the function or
            class. This argument is ignored if ob is a string.
        parser_options: Additional options for the parser. See Parser class
            for details.

    Returns:
        jscode (str): The JavaScript code as a special str object that
        has a ``meta`` attribute that contains the following fields:

        * filename (str): the name of the file that defines the object.
        * linenr (int): the starting linenr for the object definition.
        * pycode (str): the Python code used to generate the JS.
        * pyhash (str): a hash of the Python code.
        * vars_defined (set): names defined in the toplevel namespace.
        * vars_unknown (set): names used in the code but not defined in it.
        * vars_global (set): names explicitly declared global.
        * std_functions (set): stdlib functions used in this code.
        * std_method (set): stdlib methods used in this code.

    Notes:
        The Python source code for a class is acquired by name.
        Therefore one should avoid decorating classes in modules where
        multiple classes with the same name are defined. This is a
        consequence of classes not having a corresponding code object (in
        contrast to functions).

    """

    def py2js_(ob):
        if isinstance(ob, str):
            thetype = 'str'
            pycode = ob
            filename = None
            linenr = 0
        elif isinstance(ob, types.ModuleType) and hasattr(ob, '__file__'):
            thetype = 'str'
            filename = inspect.getsourcefile(ob)
            linenr = 0
            pycode = open(filename, 'rb').read().decode()
            if pycode.startswith('# -*- coding:'):
                pycode = '\n' + pycode.split('\n', 1)[-1]
        elif isinstance(ob, (type, types.FunctionType, types.MethodType)):
            thetype = 'class' if isinstance(ob, type) else 'def'
            # Get code
            try:
                filename = inspect.getsourcefile(ob)
                lines, linenr = inspect.getsourcelines(ob)
            except Exception as err:
                raise ValueError('Could not get source code for object %r: %s' %
                                 (ob, err))
            if getattr(ob, '__name__', '') in ('', '<lambda>'):
                raise ValueError('py2js() got anonymous function from '
                                 '"%s", line %i, %r.' % (filename, linenr, ob))
            # Normalize indentation, based on first line
            indent = len(lines[0]) - len(lines[0].lstrip())
            for i in range(len(lines)):
                line = lines[i]
                line_indent = len(line) - len(line.lstrip())
                if line_indent < indent and line.strip():
                    assert line.lstrip().startswith('#')  # only possible for comments
                    lines[i] = indent * ' ' + line.lstrip()
                else:
                    lines[i] = line[indent:]
            # Skip any decorators
            while not lines[0].lstrip().startswith(thetype):
                lines.pop(0)
            # join lines and rename
            pycode = ''.join(lines)
        else:
            raise ValueError('py2js() only accepts non-builtin modules, '
                             'classes and functions.')

        # Get hash, in case we ever want to cache JS accross sessions
        h = hashlib.sha256('pyscript version 1'.encode())
        h.update(pycode.encode())
        hash = h.digest()

        # Get JS code
        if filename:
            p = Parser(pycode, (filename, linenr), **parser_options)
        else:
            p = Parser(pycode, **parser_options)
        jscode = p.dump()
        if new_name and thetype in ('class', 'def'):
            jscode = js_rename(jscode, ob.__name__, new_name)

        # todo: now that we have so much info in the meta, maybe we should
        # use use py2js everywhere where we now use Parser and move its docs here.

        # Wrap in JSString
        jscode = JSString(jscode)
        jscode.meta = {}
        jscode.meta['filename'] = filename
        jscode.meta['linenr'] = linenr
        jscode.meta['pycode'] = pycode
        jscode.meta['pyhash'] = hash
        jscode.meta['vars_defined'] = set(n for n in p.vars if p.vars[n])
        jscode.meta['vars_unknown'] = set(n for n in p.vars if not p.vars[n])
        jscode.meta['vars_global'] = set(n for n in p.vars if p.vars[n] is False)
        jscode.meta['std_functions'] = p._std_functions
        jscode.meta['std_methods'] = p._std_methods

        return jscode

    if ob is None:
        return py2js_  # uses as a decorator with some options set
    return py2js_(ob)

def js_rename(jscode, cur_name, new_name):
    """ Rename a function or class in a JavaScript code string.

    Parameters:
        jscode (str): the JavaScript source code
        cur_name (str): the current name
        new_name (str): the name to replace the current name with

    Returns:
        jscode (str): the modified JavaScript source code
    """
    jscode = jscode.replace('function flx_%s' % cur_name,
                            'function flx_%s' % (new_name.split('.')[-1]), 1)
    jscode = jscode.replace('%s = function' % cur_name,
                            '%s = function' % (new_name), 1)
    jscode = jscode.replace('%s.prototype' % cur_name,
                            '%s.prototype' % new_name)
    jscode = jscode.replace('_class_name = "%s"' % cur_name,
                            '_class_name = "%s"' % new_name)
    if '.' in new_name:
        jscode = jscode.replace('var %s;\n' % cur_name, '', 1)
    else:
        jscode = jscode.replace('var %s;\n' % cur_name,
                                'var %s;\n' % new_name, 1)
    return jscode

NODE_EXE = None
def get_node_exe():
    """ Small utility that provides the node exe. The first time this
    is called both 'nodejs' and 'node' are tried. To override the
    executable path, set the ``FLEXX_NODE_EXE`` environment variable.
    """
    # This makes things work on Ubuntu's nodejs as well as other node
    # implementations, and allows users to set the node exe if necessary
    global NODE_EXE
    NODE_EXE = os.getenv('FLEXX_NODE_EXE') or NODE_EXE
    if NODE_EXE is None:
        NODE_EXE = 'nodejs'
        try:
            subprocess.check_output([NODE_EXE, '-v'])
        except Exception:  # pragma: no cover
            NODE_EXE = 'node'
    return NODE_EXE

_eval_count = 0

def evaljs(jscode, whitespace=True, print_result=True):
    """ Evaluate JavaScript code in Node.js.

    parameters:
        jscode (str): the JavaScript code to evaluate.
        whitespace (bool): if whitespace is False, the whitespace
            is removed from the result. Default True.
        print_result (bool): whether to print the result of the evaluation.
            Default True. If False, larger pieces of code can be evaluated
            because we can use file-mode.

    returns:
        result (str): the last result as a string.
    """
    global _eval_count

    # Prepare command
    if len(jscode) > 2**14:
        if print_result:
            # Strictly speaking, this is a limitation of Windows, but come-on!
            raise RuntimeError('evaljs() wont send more than 16 kB of code '
                               'over the command line, but cannot use a file '
                               'unless print_result is False.')
        _eval_count += 1
        fname = 'pyscript_%i_%i.js' % (os.getpid(), _eval_count)
        filename = os.path.join(tempfile.gettempdir(), fname)
        with open(filename, 'wb') as f:
            f.write(jscode.encode())
        cmd = [get_node_exe(), '--use_strict', filename]
    else:
        filename = None
        p_or_e = ['-p', '-e'] if print_result else ['-e']
        cmd = [get_node_exe(), '--use_strict'] + p_or_e + [jscode]
        if sys.version_info[0] < 3:
            cmd = [c.encode('raw_unicode_escape') for c in cmd]

    # Call node
    try:
        res = subprocess.check_output(cmd)
    except Exception as err:
        if hasattr(err, 'output'):
            err = err.output.decode()
        else:
            err = str(err)
        err = err[:200] + '...' if len(err) > 200 else err
        raise Exception(err)
    finally:
        if filename is not None:
            try:
                os.remove(filename)
            except Exception:
                pass

    # Process result
    res = res.decode().rstrip()
    if print_result and res.endswith('undefined'):
        res = res[:-9].rstrip()
    if not whitespace:
        res = res.replace('\n', '').replace('\t', '').replace(' ', '')
    return res

def evalpy(pycode, whitespace=True):
    """ Evaluate PyScript code in Node.js (after translating to JS).

    parameters:
        pycode (str): the PyScript code to evaluate.
        whitespace (bool): if whitespace is False, the whitespace is
            removed from the result. Default True.

    returns:
        result (str): the last result as a string.
    """
    # delibirate numpy doc style to see if napoleon handles it the same
    return evaljs(py2js(pycode), whitespace)

def script2js(filename, namespace=None, target=None, module_type='umd',
              **parser_options):
    """ Export a .py file to a .js file.

    Parameters:
      filename (str): the filename of the .py file to transpile.
      namespace (str): the namespace for this module. (optional)
      target (str): the filename of the resulting .js file. If not given
        or None, will use the ``filename``, but with a ``.js`` extension.
      module_type (str): the type of module to produce (if namespace is given),
        can be 'hidden', 'simple', 'amd', 'umd', default 'umd'.
      parser_options: additional options for the parser. See Parser class
        for details.
    """
    # Import
    assert filename.endswith('.py')
    pycode = open(filename, 'rb').read().decode()
    # Convert
    parser = Parser(pycode, filename, **parser_options)
    jscode = '/* Do not edit, autogenerated by flexx.pyscript */\n\n' + parser.dump()
    # Wrap in module
    if namespace:
        exports = [name for name in parser.vars
                   if not (parser.vars[name] or name.startswith('_'))]
        jscode = create_js_module(namespace, jscode, [], exports, module_type)
    # Export
    if target is None:
        dirname, fname = os.path.split(filename)
        filename2 = os.path.join(dirname, fname[:-3] + '.js')
    else:
        filename2 = target
    with open(filename2, 'wb') as f:
        f.write(jscode.encode())
<commit_msg>eval_js includes stderr (for error logging) and raises runtimeerror on fail<commit_after>import os
import sys
import types
import inspect
import hashlib
import tempfile
import subprocess

from . import Parser
from .stdlib import get_full_std_lib  # noqa
from .modules import create_js_module

class JSString(str):
    """ A subclass of string, so we can add attributes to JS string objects.
    """
    pass

def py2js(ob=None, new_name=None, **parser_options):
    """ Convert Python to JavaScript.

    Parameters:
        ob (str, module, function, class): The code, function or class
            to transpile.
        new_name (str, optional): If given, renames the function or
            class. This argument is ignored if ob is a string.
        parser_options: Additional options for the parser. See Parser class
            for details.

    Returns:
        jscode (str): The JavaScript code as a special str object that
        has a ``meta`` attribute that contains the following fields:

        * filename (str): the name of the file that defines the object.
        * linenr (int): the starting linenr for the object definition.
        * pycode (str): the Python code used to generate the JS.
        * pyhash (str): a hash of the Python code.
        * vars_defined (set): names defined in the toplevel namespace.
        * vars_unknown (set): names used in the code but not defined in it.
        * vars_global (set): names explicitly declared global.
        * std_functions (set): stdlib functions used in this code.
        * std_method (set): stdlib methods used in this code.

    Notes:
        The Python source code for a class is acquired by name.
        Therefore one should avoid decorating classes in modules where
        multiple classes with the same name are defined. This is a
        consequence of classes not having a corresponding code object (in
        contrast to functions).

    """

    def py2js_(ob):
        if isinstance(ob, str):
            thetype = 'str'
            pycode = ob
            filename = None
            linenr = 0
        elif isinstance(ob, types.ModuleType) and hasattr(ob, '__file__'):
            thetype = 'str'
            filename = inspect.getsourcefile(ob)
            linenr = 0
            pycode = open(filename, 'rb').read().decode()
            if pycode.startswith('# -*- coding:'):
                pycode = '\n' + pycode.split('\n', 1)[-1]
        elif isinstance(ob, (type, types.FunctionType, types.MethodType)):
            thetype = 'class' if isinstance(ob, type) else 'def'
            # Get code
            try:
                filename = inspect.getsourcefile(ob)
                lines, linenr = inspect.getsourcelines(ob)
            except Exception as err:
                raise ValueError('Could not get source code for object %r: %s' %
                                 (ob, err))
            if getattr(ob, '__name__', '') in ('', '<lambda>'):
                raise ValueError('py2js() got anonymous function from '
                                 '"%s", line %i, %r.' % (filename, linenr, ob))
            # Normalize indentation, based on first line
            indent = len(lines[0]) - len(lines[0].lstrip())
            for i in range(len(lines)):
                line = lines[i]
                line_indent = len(line) - len(line.lstrip())
                if line_indent < indent and line.strip():
                    assert line.lstrip().startswith('#')  # only possible for comments
                    lines[i] = indent * ' ' + line.lstrip()
                else:
                    lines[i] = line[indent:]
            # Skip any decorators
            while not lines[0].lstrip().startswith(thetype):
                lines.pop(0)
            # join lines and rename
            pycode = ''.join(lines)
        else:
            raise ValueError('py2js() only accepts non-builtin modules, '
                             'classes and functions.')

        # Get hash, in case we ever want to cache JS accross sessions
        h = hashlib.sha256('pyscript version 1'.encode())
        h.update(pycode.encode())
        hash = h.digest()

        # Get JS code
        if filename:
            p = Parser(pycode, (filename, linenr), **parser_options)
        else:
            p = Parser(pycode, **parser_options)
        jscode = p.dump()
        if new_name and thetype in ('class', 'def'):
            jscode = js_rename(jscode, ob.__name__, new_name)

        # todo: now that we have so much info in the meta, maybe we should
        # use use py2js everywhere where we now use Parser and move its docs here.

        # Wrap in JSString
        jscode = JSString(jscode)
        jscode.meta = {}
        jscode.meta['filename'] = filename
        jscode.meta['linenr'] = linenr
        jscode.meta['pycode'] = pycode
        jscode.meta['pyhash'] = hash
        jscode.meta['vars_defined'] = set(n for n in p.vars if p.vars[n])
        jscode.meta['vars_unknown'] = set(n for n in p.vars if not p.vars[n])
        jscode.meta['vars_global'] = set(n for n in p.vars if p.vars[n] is False)
        jscode.meta['std_functions'] = p._std_functions
        jscode.meta['std_methods'] = p._std_methods

        return jscode

    if ob is None:
        return py2js_  # uses as a decorator with some options set
    return py2js_(ob)

def js_rename(jscode, cur_name, new_name):
    """ Rename a function or class in a JavaScript code string.

    Parameters:
        jscode (str): the JavaScript source code
        cur_name (str): the current name
        new_name (str): the name to replace the current name with

    Returns:
        jscode (str): the modified JavaScript source code
    """
    jscode = jscode.replace('function flx_%s' % cur_name,
                            'function flx_%s' % (new_name.split('.')[-1]), 1)
    jscode = jscode.replace('%s = function' % cur_name,
                            '%s = function' % (new_name), 1)
    jscode = jscode.replace('%s.prototype' % cur_name,
                            '%s.prototype' % new_name)
    jscode = jscode.replace('_class_name = "%s"' % cur_name,
                            '_class_name = "%s"' % new_name)
    if '.' in new_name:
        jscode = jscode.replace('var %s;\n' % cur_name, '', 1)
    else:
        jscode = jscode.replace('var %s;\n' % cur_name,
                                'var %s;\n' % new_name, 1)
    return jscode

NODE_EXE = None
def get_node_exe():
    """ Small utility that provides the node exe. The first time this
    is called both 'nodejs' and 'node' are tried. To override the
    executable path, set the ``FLEXX_NODE_EXE`` environment variable.
    """
    # This makes things work on Ubuntu's nodejs as well as other node
    # implementations, and allows users to set the node exe if necessary
    global NODE_EXE
    NODE_EXE = os.getenv('FLEXX_NODE_EXE') or NODE_EXE
    if NODE_EXE is None:
        NODE_EXE = 'nodejs'
        try:
            subprocess.check_output([NODE_EXE, '-v'])
        except Exception:  # pragma: no cover
            NODE_EXE = 'node'
    return NODE_EXE

_eval_count = 0

def evaljs(jscode, whitespace=True, print_result=True):
    """ Evaluate JavaScript code in Node.js.

    parameters:
        jscode (str): the JavaScript code to evaluate.
        whitespace (bool): if whitespace is False, the whitespace
            is removed from the result. Default True.
        print_result (bool): whether to print the result of the evaluation.
            Default True. If False, larger pieces of code can be evaluated
            because we can use file-mode.

    returns:
        result (str): the last result as a string.
    """
    global _eval_count

    # Prepare command
    if len(jscode) > 2**14:
        if print_result:
            # Strictly speaking, this is a limitation of Windows, but come-on!
            raise RuntimeError('evaljs() wont send more than 16 kB of code '
                               'over the command line, but cannot use a file '
                               'unless print_result is False.')
        _eval_count += 1
        fname = 'pyscript_%i_%i.js' % (os.getpid(), _eval_count)
        filename = os.path.join(tempfile.gettempdir(), fname)
        with open(filename, 'wb') as f:
            f.write(jscode.encode())
        cmd = [get_node_exe(), '--use_strict', filename]
    else:
        filename = None
        p_or_e = ['-p', '-e'] if print_result else ['-e']
        cmd = [get_node_exe(), '--use_strict'] + p_or_e + [jscode]
        if sys.version_info[0] < 3:
            cmd = [c.encode('raw_unicode_escape') for c in cmd]

    # Call node
    try:
        res = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    except Exception as err:
        if hasattr(err, 'output'):
            err = err.output.decode()
        else:
            err = str(err)
        err = err[:200] + '...' if len(err) > 200 else err
        raise RuntimeError(err)
    finally:
        if filename is not None:
            try:
                os.remove(filename)
            except Exception:
                pass

    # Process result
    res = res.decode().rstrip()
    if print_result and res.endswith('undefined'):
        res = res[:-9].rstrip()
    if not whitespace:
        res = res.replace('\n', '').replace('\t', '').replace(' ', '')
    return res

def evalpy(pycode, whitespace=True):
    """ Evaluate PyScript code in Node.js (after translating to JS).

    parameters:
        pycode (str): the PyScript code to evaluate.
        whitespace (bool): if whitespace is False, the whitespace is
            removed from the result. Default True.

    returns:
        result (str): the last result as a string.
    """
    # delibirate numpy doc style to see if napoleon handles it the same
    return evaljs(py2js(pycode), whitespace)

def script2js(filename, namespace=None, target=None, module_type='umd',
              **parser_options):
    """ Export a .py file to a .js file.

    Parameters:
      filename (str): the filename of the .py file to transpile.
      namespace (str): the namespace for this module. (optional)
      target (str): the filename of the resulting .js file. If not given
        or None, will use the ``filename``, but with a ``.js`` extension.
      module_type (str): the type of module to produce (if namespace is given),
        can be 'hidden', 'simple', 'amd', 'umd', default 'umd'.
      parser_options: additional options for the parser. See Parser class
        for details.
    """
    # Import
    assert filename.endswith('.py')
    pycode = open(filename, 'rb').read().decode()
    # Convert
    parser = Parser(pycode, filename, **parser_options)
    jscode = '/* Do not edit, autogenerated by flexx.pyscript */\n\n' + parser.dump()
    # Wrap in module
    if namespace:
        exports = [name for name in parser.vars
                   if not (parser.vars[name] or name.startswith('_'))]
        jscode = create_js_module(namespace, jscode, [], exports, module_type)
    # Export
    if target is None:
        dirname, fname = os.path.split(filename)
        filename2 = os.path.join(dirname, fname[:-3] + '.js')
    else:
        filename2 = target
    with open(filename2, 'wb') as f:
        f.write(jscode.encode())
<|endoftext|>


<commit_before>from __future__ import print_function
import logging
import backoff
from throttle import throttle
from config import __packagename__

logging.getLogger('backoff').addHandler(logging.StreamHandler())
logging.getLogger('backoff').setLevel(logging.DEBUG)

_global_config = None
def get_max_tries():
    return _global_config.retry + 1
def get_delay_sec():
    return _global_config.throttling

class Resiliently(object):
    def __init__(self, config):
        global _global_config
        _global_config = config
        self._config = config

    @backoff.on_exception(backoff.expo, Exception, max_tries=get_max_tries)
    @throttle(delay_sec=get_delay_sec)
    def call(self, func, *args, **kwargs):
        return func(*args, **kwargs)
        # return self._throttle(self._retry, func, *args, **kwargs)

    def _throttle(self, func, *args, **kwargs):
        return throttle(delay_sec=self._config.throttling)(func)(*args, **kwargs)

    def _retry(self, func, *args, **kwargs):
        # We +1 this because backoff retries UP to and not including max_retries
        max_tries = self._config.retry + 1
        return backoff.on_exception(backoff.expo, Exception, max_tries=max_tries)(func)(*args, **kwargs)

<commit_msg>Tests failing but improved throttle design<commit_after>from __future__ import print_function
import logging
import backoff
from functools import partial
from throttle import throttle

logging.getLogger('backoff').addHandler(logging.StreamHandler())
logging.getLogger('backoff').setLevel(logging.DEBUG)

class Resiliently(object):
    def __init__(self, config):
        self._config = config
        # self._throttle_bind = throttle

    # @backoff.on_exception(backoff.expo, Exception, max_tries=get_max_tries)
    # @throttle(delay_sec=get_delay_sec)
    def call(self, func, *args, **kwargs):
        # return func(*args, **kwargs)
        return self._throttle(self._retry, func, *args, **kwargs)

    def _throttle(self, func, *args, **kwargs):
        return throttle(delay_sec=self._config.throttling)(func)(*args, **kwargs)

    def _retry(self, func, *args, **kwargs):
        # We +1 this because backoff retries UP to and not including max_retries
        max_tries = self._config.retry + 1
        return backoff.on_exception(backoff.expo, Exception, max_tries=max_tries)(func)(*args, **kwargs)

<|endoftext|>


<commit_before>from flow.brokers.amqp.channel_facade import ChannelFacade
from flow.protocol.exceptions import InvalidMessageException
from flow import interfaces
from injector import inject
from twisted.internet import defer
from flow.util.exit import exit_process
from flow.exit_codes import EXECUTE_SYSTEM_FAILURE
from flow.util.defer import add_callback_and_default_errback

import logging

LOG = logging.getLogger(__name__)

@inject(channel=ChannelFacade)
class AmqpBroker(interfaces.IBroker):
    def publish(self, exchange_name, routing_key, message):
        LOG.debug("Publishing to exchange (%s) with routing_key (%s) "
                "the message (%s)", exchange_name, routing_key, message)

        encoded_message = message.encode()

        return self.channel.basic_publish(
                exchange_name=exchange_name,
                routing_key=routing_key,
                encoded_message=encoded_message)

    def register_handler(self, handler):
        LOG.debug("Registering handler on queue '%s'.", handler.queue_name)
        connect_deferred = self.channel.connect()
        add_callback_and_default_errback(connect_deferred, self._start_handler,
                handler)
        return connect_deferred

    def declare_queue(self, *args, **kwargs):
        return self.channel.declare_queue(*args, **kwargs)

    def _start_handler(self, _cb_channel, handler):
        queue_name = handler.queue_name
        consume_deferred = self.channel.basic_consume(queue=queue_name)
        add_callback_and_default_errback(consume_deferred, self._begin_get_loop,
                handler)
        return _cb_channel

    def _begin_get_loop(self, consume_info, handler):
        queue, consumer_tag = consume_info
        queue_name = handler.queue_name
        LOG.debug('Beginning consumption on queue (%s)', queue_name)

        self._get_message_from_queue(queue=queue, handler=handler)
        return consume_info

    def _get_message_from_queue(self, queue, handler):
        deferred = queue.get()
        deferred.addCallback(self._on_message_recieved, queue, handler)
        deferred.addErrback(self._on_get_failed, handler)
        return deferred

    def _on_get_failed(self, reason, handler):
        LOG.critical("Get from queue '%s' failed: %s", handler.queue_name,
                reason)
        exit_process(EXECUTE_SYSTEM_FAILURE)

    def _on_ack_reject_failed(self, error):
        LOG.critical('Failed to ack or reject:\n%s', error.getTrackback())
        exit_process(EXECUTE_SYSTEM_FAILURE)

    def _on_message_recieved(self, get_info, queue, handler):
        (channel, basic_deliver, properties, encoded_message) = get_info

        message_class = handler.message_class
        try:
            message = message_class.decode(encoded_message)
            deferred = handler(message)
        except InvalidMessageException as e:
            LOG.exception('Invalid message.  message = %s', encoded_message)
            deferred = defer.fail(e)

        receive_tag = basic_deliver.delivery_tag
        deferred.addCallbacks(self._ack, self._reject,
                callbackArgs=(receive_tag,),
                errbackArgs=(receive_tag,))
        deferred.addErrback(self._on_ack_reject_failed)

        self._get_message_from_queue(queue, handler)
        return get_info

    def _ack(self, confirm_info, receive_tag):
        LOG.debug('Acking message (%s)', receive_tag)
        self.channel.basic_ack(receive_tag)
        return confirm_info

    def _reject(self, reason, receive_tag):
        LOG.error('Rejecting message (%s) due to error: %s', receive_tag,
                reason.getTraceback())
        self.channel.basic_reject(receive_tag)
        return reason
<commit_msg>fix typo<commit_after>from flow.brokers.amqp.channel_facade import ChannelFacade
from flow.protocol.exceptions import InvalidMessageException
from flow import interfaces
from injector import inject
from twisted.internet import defer
from flow.util.exit import exit_process
from flow.exit_codes import EXECUTE_SYSTEM_FAILURE
from flow.util.defer import add_callback_and_default_errback

import logging

LOG = logging.getLogger(__name__)

@inject(channel=ChannelFacade)
class AmqpBroker(interfaces.IBroker):
    def publish(self, exchange_name, routing_key, message):
        LOG.debug("Publishing to exchange (%s) with routing_key (%s) "
                "the message (%s)", exchange_name, routing_key, message)

        encoded_message = message.encode()

        return self.channel.basic_publish(
                exchange_name=exchange_name,
                routing_key=routing_key,
                encoded_message=encoded_message)

    def register_handler(self, handler):
        LOG.debug("Registering handler on queue '%s'.", handler.queue_name)
        connect_deferred = self.channel.connect()
        add_callback_and_default_errback(connect_deferred, self._start_handler,
                handler)
        return connect_deferred

    def declare_queue(self, *args, **kwargs):
        return self.channel.declare_queue(*args, **kwargs)

    def _start_handler(self, _cb_channel, handler):
        queue_name = handler.queue_name
        consume_deferred = self.channel.basic_consume(queue=queue_name)
        add_callback_and_default_errback(consume_deferred, self._begin_get_loop,
                handler)
        return _cb_channel

    def _begin_get_loop(self, consume_info, handler):
        queue, consumer_tag = consume_info
        queue_name = handler.queue_name
        LOG.debug('Beginning consumption on queue (%s)', queue_name)

        self._get_message_from_queue(queue=queue, handler=handler)
        return consume_info

    def _get_message_from_queue(self, queue, handler):
        deferred = queue.get()
        deferred.addCallback(self._on_message_recieved, queue, handler)
        deferred.addErrback(self._on_get_failed, handler)
        return deferred

    def _on_get_failed(self, reason, handler):
        LOG.critical("Get from queue '%s' failed: %s", handler.queue_name,
                reason)
        exit_process(EXECUTE_SYSTEM_FAILURE)

    def _on_ack_reject_failed(self, error):
        LOG.critical('Failed to ack or reject:\n%s', error.getTraceback())
        exit_process(EXECUTE_SYSTEM_FAILURE)

    def _on_message_recieved(self, get_info, queue, handler):
        (channel, basic_deliver, properties, encoded_message) = get_info

        message_class = handler.message_class
        try:
            message = message_class.decode(encoded_message)
            deferred = handler(message)
        except InvalidMessageException as e:
            LOG.exception('Invalid message.  message = %s', encoded_message)
            deferred = defer.fail(e)

        receive_tag = basic_deliver.delivery_tag
        deferred.addCallbacks(self._ack, self._reject,
                callbackArgs=(receive_tag,),
                errbackArgs=(receive_tag,))
        deferred.addErrback(self._on_ack_reject_failed)

        self._get_message_from_queue(queue, handler)
        return get_info

    def _ack(self, confirm_info, receive_tag):
        LOG.debug('Acking message (%s)', receive_tag)
        self.channel.basic_ack(receive_tag)
        return confirm_info

    def _reject(self, reason, receive_tag):
        LOG.error('Rejecting message (%s) due to error: %s', receive_tag,
                reason.getTraceback())
        self.channel.basic_reject(receive_tag)
        return reason
<|endoftext|>


<commit_before>'''
Storage
=======

.. versionadded:: 1.7.0

.. warning::

    This module is still experimental, and the API is subject to change in a
    future version.

Usage
-----

The idea behind the Storage module is to be able to load/store any number of
key/value pairs via an indexed key. The default model is abstract so you
cannot use it directly. We provide some implementations such as:

- :class:`kivy.storage.dictstore.DictStore`: use a python dict as a store
- :class:`kivy.storage.jsonstore.JsonStore`: use a JSON file as a store
- :class:`kivy.storage.redistore.RedisStore`: use a `Redis <http://redis.io>`_
  database with `redis-py <https://github.com/andymccurdy/redis-py>`_

Examples
--------

For example, let's use a JsonStore::

    from kivy.storage.jsonstore import JsonStore

    store = JsonStore('hello.json')

    # put some values
    store.put('tito', name='Mathieu', org='kivy')
    store.put('tshirtman', name='Gabriel', age=27)

    # using the same index key erases all previously added key/value pairs
    store.put('tito', name='Mathieu', age=30)

    # get a value using a index key and key
    print('tito is', store.get('tito')['age'])

    # or guess the key/entry for a part of the key
    for item in store.find(name='Gabriel'):
        print('tshirtmans index key is', item[0])
        print('his key value pairs are', str(item[1]))

Because the data is persistant, you can check later to see if the key exists::

    from kivy.storage.jsonstore import JsonStore

    store = JsonStore('hello.json')
    if store.exists('tito'):
        print('tite exists:', store.get('tito'))
        store.delete('tito')

Synchronous / Asynchronous API
------------------------------

All the standard methods (:meth:`~AbstractStore.get`,
:meth:`~AbstractStore.put` , :meth:`~AbstractStore.exists`,
:meth:`~AbstractStore.delete`, :meth:`~AbstractStore.find`) have an
asynchronous version.

For example, the *get* method has a `callback` parameter. If set, the
`callback` will be used to return the result to the user when available:
the request will be asynchronous.  If the `callback` is None, then the
request will be synchronous and the result will be returned directly.

Without callback (Synchronous API)::

    entry = mystore.get('tito')
    print('tito =', entry)

With callback (Asynchronous API)::

    def my_callback(store, key, entry):
        print('the key', key, 'have', entry)
    mystore.get('plop', callback=my_callback)

The callback signature is (for almost all methods) `callback(store, key,
result)`::

#. `store` is the `Store` instance currently used.
#. `key` is the key to search for.
#. `entry` is the result of the lookup for the `key`.

Synchronous container type
--------------------------

The storage API emulates the container type for the synchronous API::

    store = JsonStore('hello.json')

    # original: store.get('tito')
    store['tito']

    # original: store.put('tito', name='Mathieu')
    store['tito'] = {'name': 'Mathieu'}

    # original: store.delete('tito')
    del store['tito']

    # original: store.count()
    len(store)

    # original: store.exists('tito')
    'tito' in store

    # original: for key in store.keys()
    for key in store:
        pass

'''

from kivy.clock import Clock
from kivy.event import EventDispatcher
from functools import partial

class AbstractStore(EventDispatcher):
    '''Abstract class used to implement a Store
    '''

    def __init__(self, **kwargs):
        super(AbstractStore, self).__init__(**kwargs)
        self.store_load()

    def exists(self, key):
        '''Check if a key exists in the store.
        '''
        return self.store_exists(key)

    def async_exists(self, callback, key):
        '''Asynchronous version of :meth:`exists`.

        :Callback arguments:
            `store`: :class:`AbstractStore` instance
                Store instance
            `key`: string
                Name of the key to search for
            `result`: boo
                Result of the query, None if any error
        '''
        self._schedule(self.store_exists_async,
                       key=key, callback=callback)

    def get(self, key):
        '''Get the key/value pairs stored at `key`. If the key is not found, a
        `KeyError` exception will be thrown.
        '''
        return self.store_get(key)

    def async_get(self, callback, key):
        '''Asynchronous version of :meth:`get`.

        :Callback arguments:
            `store`: :class:`AbstractStore` instance
                Store instance
            `key`: string
                Name of the key to search for
            `result`: dict
                Result of the query, None if any error
        '''
        self._schedule(self.store_get_async, key=key, callback=callback)

    def put(self, key, **values):
        '''Put new key/value pairs (given in *values*) into the storage. Any
        existing key/value pairs will be removed.
        '''
        need_sync = self.store_put(key, values)
        if need_sync:
            self.store_sync()
        return need_sync

    def async_put(self, callback, key, **values):
        '''Asynchronous version of :meth:`put`.

        :Callback arguments:
            `store`: :class:`AbstractStore` instance
                Store instance
            `key`: string
                Name of the key to search for
            `result`: bool
                Indicate True if the storage has been updated, or False if
                nothing has been done (no changes). None if any error.
        '''
        self._schedule(self.store_put_async,
                       key=key, value=values, callback=callback)

    def delete(self, key):
        '''Delete a key from the storage. If the key is not found, a `KeyError`
        exception will be thrown.'''
        need_sync = self.store_delete(key)
        if need_sync:
            self.store_sync()
        return need_sync

    def async_delete(self, callback, key):
        '''Asynchronous version of :meth:`delete`.

        :Callback arguments:
            `store`: :class:`AbstractStore` instance
                Store instance
            `key`: string
                Name of the key to search for
            `result`: bool
                Indicate True if the storage has been updated, or False if
                nothing has been done (no changes). None if any error.
        '''
        self._schedule(self.store_delete_async, key=key,
                       callback=callback)

    def find(self, **filters):
        '''Return all the entries matching the filters. The entries are
        returned through a generator as a list of (key, entry) pairs
        where *entry* is a dict of key/value pairs ::

            for key, entry in store.find(name='Mathieu'):
                print('key:', key, ', entry:', entry)

        Because it's a generator, you cannot directly use it as a list. You can
        do::

            # get all the (key, entry) availables
            entries = list(store.find(name='Mathieu'))
            # get only the entry from (key, entry)
            entries = list((x[1] for x in store.find(name='Mathieu')))
        '''
        return self.store_find(filters)

    def async_find(self, callback, **filters):
        '''Asynchronous version of :meth:`find`.

        The callback will be called for each entry in the result.

        :Callback arguments:
            `store`: :class:`AbstractStore` instance
                Store instance
            `key`: string
                Name of the key to search for, or None if we reach the end of
                the results
            `result`: bool
                Indicate True if the storage has been updated, or False if
                nothing has been done (no changes). None if any error.
        '''
        self._schedule(self.store_find_async,
                       callback=callback, filters=filters)

    def keys(self):
        '''Return a list of all the keys in the storage.
        '''
        return self.store_keys()

    def async_keys(self, callback):
        '''Asynchronously return all the keys in the storage.
        '''
        self._schedule(self.store_keys_async, callback=callback)

    def count(self):
        '''Return the number of entries in the storage.
        '''
        return self.store_count()

    def async_count(self, callback):
        '''Asynchronously return the number of entries in the storage.
        '''
        self._schedule(self.store_count_async, callback=callback)

    def clear(self):
        '''Wipe the whole storage.
        '''
        return self.store_clear()

    def async_clear(self, callback):
        '''Asynchronous version of :meth:`clear`.
        '''
        self._schedule(self.store_clear_async, callback=callback)

    #
    # Operators
    #

    def __setitem__(self, key, values):
        if not isinstance(values, dict):
            raise Exception('Only dict are accepted for the store[key] = dict')
        self.put(key, **values)

    def __getitem__(self, key):
        return self.get(key)

    def __delitem__(self, key):
        return self.keys()

    def __contains__(self, key):
        return self.exists(key)

    def __len__(self):
        return self.count()

    def __iter__(self):
        for key in self.keys():
            yield key

    #
    # Used for implementation
    #

    def store_load(self):
        pass

    def store_sync(self):
        pass

    def store_get(self, key):
        raise NotImplemented()

    def store_put(self, key, value):
        raise NotImplemented()

    def store_exists(self, key):
        raise NotImplemented()

    def store_delete(self, key):
        raise NotImplemented()

    def store_find(self, filters):
        return []

    def store_keys(self):
        return []

    def store_count(self):
        return len(self.store_keys())

    def store_clear(self):
        for key in self.store_keys():
            self.store_delete(key)

    def store_get_async(self, key, callback):
        try:
            value = self.store_get(key)
            callback(self, key, value)
        except KeyError:
            callback(self, key, None)

    def store_put_async(self, key, value, callback):
        try:
            value = self.store_put(key, value)
            callback(self, key, value)
        except:
            callback(self, key, None)

    def store_exists_async(self, key, callback):
        try:
            value = self.store_exists(key)
            callback(self, key, value)
        except:
            callback(self, key, None)

    def store_delete_async(self, key, callback):
        try:
            value = self.store_delete(key)
            callback(self, key, value)
        except:
            callback(self, key, None)

    def store_find_async(self, filters, callback):
        for key, entry in self.store_find(filters):
            callback(self, filters, key, entry)
        callback(self, filters, None, None)

    def store_count_async(self, callback):
        try:
            value = self.store_count()
            callback(self, value)
        except:
            callback(self, 0)

    def store_keys_async(self, callback):
        try:
            keys = self.store_keys()
            callback(self, keys)
        except:
            callback(self, [])

    def store_clear_async(self, callback):
        self.store_clear()
        callback(self)

    #
    # Privates
    #

    def _schedule(self, callback, **kwargs):
        # XXX not entirely sure about the best value (0 or -1).
        Clock.schedule_once(partial(callback, **kwargs), 0)
<commit_msg>fix bad params for async storage<commit_after>'''
Storage
=======

.. versionadded:: 1.7.0

.. warning::

    This module is still experimental, and the API is subject to change in a
    future version.

Usage
-----

The idea behind the Storage module is to be able to load/store any number of
key/value pairs via an indexed key. The default model is abstract so you
cannot use it directly. We provide some implementations such as:

- :class:`kivy.storage.dictstore.DictStore`: use a python dict as a store
- :class:`kivy.storage.jsonstore.JsonStore`: use a JSON file as a store
- :class:`kivy.storage.redistore.RedisStore`: use a `Redis <http://redis.io>`_
  database with `redis-py <https://github.com/andymccurdy/redis-py>`_

Examples
--------

For example, let's use a JsonStore::

    from kivy.storage.jsonstore import JsonStore

    store = JsonStore('hello.json')

    # put some values
    store.put('tito', name='Mathieu', org='kivy')
    store.put('tshirtman', name='Gabriel', age=27)

    # using the same index key erases all previously added key/value pairs
    store.put('tito', name='Mathieu', age=30)

    # get a value using a index key and key
    print('tito is', store.get('tito')['age'])

    # or guess the key/entry for a part of the key
    for item in store.find(name='Gabriel'):
        print('tshirtmans index key is', item[0])
        print('his key value pairs are', str(item[1]))

Because the data is persistant, you can check later to see if the key exists::

    from kivy.storage.jsonstore import JsonStore

    store = JsonStore('hello.json')
    if store.exists('tito'):
        print('tite exists:', store.get('tito'))
        store.delete('tito')

Synchronous / Asynchronous API
------------------------------

All the standard methods (:meth:`~AbstractStore.get`,
:meth:`~AbstractStore.put` , :meth:`~AbstractStore.exists`,
:meth:`~AbstractStore.delete`, :meth:`~AbstractStore.find`) have an
asynchronous version.

For example, the *get* method has a `callback` parameter. If set, the
`callback` will be used to return the result to the user when available:
the request will be asynchronous.  If the `callback` is None, then the
request will be synchronous and the result will be returned directly.

Without callback (Synchronous API)::

    entry = mystore.get('tito')
    print('tito =', entry)

With callback (Asynchronous API)::

    def my_callback(store, key, entry):
        print('the key', key, 'have', entry)
    mystore.get('plop', callback=my_callback)

The callback signature is (for almost all methods) `callback(store, key,
result)`::

#. `store` is the `Store` instance currently used.
#. `key` is the key to search for.
#. `entry` is the result of the lookup for the `key`.

Synchronous container type
--------------------------

The storage API emulates the container type for the synchronous API::

    store = JsonStore('hello.json')

    # original: store.get('tito')
    store['tito']

    # original: store.put('tito', name='Mathieu')
    store['tito'] = {'name': 'Mathieu'}

    # original: store.delete('tito')
    del store['tito']

    # original: store.count()
    len(store)

    # original: store.exists('tito')
    'tito' in store

    # original: for key in store.keys()
    for key in store:
        pass

'''

from kivy.clock import Clock
from kivy.event import EventDispatcher
from functools import partial

class AbstractStore(EventDispatcher):
    '''Abstract class used to implement a Store
    '''

    def __init__(self, **kwargs):
        super(AbstractStore, self).__init__(**kwargs)
        self.store_load()

    def exists(self, key):
        '''Check if a key exists in the store.
        '''
        return self.store_exists(key)

    def async_exists(self, callback, key):
        '''Asynchronous version of :meth:`exists`.

        :Callback arguments:
            `store`: :class:`AbstractStore` instance
                Store instance
            `key`: string
                Name of the key to search for
            `result`: boo
                Result of the query, None if any error
        '''
        self._schedule(self.store_exists_async,
                       key=key, callback=callback)

    def get(self, key):
        '''Get the key/value pairs stored at `key`. If the key is not found, a
        `KeyError` exception will be thrown.
        '''
        return self.store_get(key)

    def async_get(self, callback, key):
        '''Asynchronous version of :meth:`get`.

        :Callback arguments:
            `store`: :class:`AbstractStore` instance
                Store instance
            `key`: string
                Name of the key to search for
            `result`: dict
                Result of the query, None if any error
        '''
        self._schedule(self.store_get_async, key=key, callback=callback)

    def put(self, key, **values):
        '''Put new key/value pairs (given in *values*) into the storage. Any
        existing key/value pairs will be removed.
        '''
        need_sync = self.store_put(key, values)
        if need_sync:
            self.store_sync()
        return need_sync

    def async_put(self, callback, key, **values):
        '''Asynchronous version of :meth:`put`.

        :Callback arguments:
            `store`: :class:`AbstractStore` instance
                Store instance
            `key`: string
                Name of the key to search for
            `result`: bool
                Indicate True if the storage has been updated, or False if
                nothing has been done (no changes). None if any error.
        '''
        self._schedule(self.store_put_async,
                       key=key, value=values, callback=callback)

    def delete(self, key):
        '''Delete a key from the storage. If the key is not found, a `KeyError`
        exception will be thrown.'''
        need_sync = self.store_delete(key)
        if need_sync:
            self.store_sync()
        return need_sync

    def async_delete(self, callback, key):
        '''Asynchronous version of :meth:`delete`.

        :Callback arguments:
            `store`: :class:`AbstractStore` instance
                Store instance
            `key`: string
                Name of the key to search for
            `result`: bool
                Indicate True if the storage has been updated, or False if
                nothing has been done (no changes). None if any error.
        '''
        self._schedule(self.store_delete_async, key=key,
                       callback=callback)

    def find(self, **filters):
        '''Return all the entries matching the filters. The entries are
        returned through a generator as a list of (key, entry) pairs
        where *entry* is a dict of key/value pairs ::

            for key, entry in store.find(name='Mathieu'):
                print('key:', key, ', entry:', entry)

        Because it's a generator, you cannot directly use it as a list. You can
        do::

            # get all the (key, entry) availables
            entries = list(store.find(name='Mathieu'))
            # get only the entry from (key, entry)
            entries = list((x[1] for x in store.find(name='Mathieu')))
        '''
        return self.store_find(filters)

    def async_find(self, callback, **filters):
        '''Asynchronous version of :meth:`find`.

        The callback will be called for each entry in the result.

        :Callback arguments:
            `store`: :class:`AbstractStore` instance
                Store instance
            `key`: string
                Name of the key to search for, or None if we reach the end of
                the results
            `result`: bool
                Indicate True if the storage has been updated, or False if
                nothing has been done (no changes). None if any error.
        '''
        self._schedule(self.store_find_async,
                       callback=callback, filters=filters)

    def keys(self):
        '''Return a list of all the keys in the storage.
        '''
        return self.store_keys()

    def async_keys(self, callback):
        '''Asynchronously return all the keys in the storage.
        '''
        self._schedule(self.store_keys_async, callback=callback)

    def count(self):
        '''Return the number of entries in the storage.
        '''
        return self.store_count()

    def async_count(self, callback):
        '''Asynchronously return the number of entries in the storage.
        '''
        self._schedule(self.store_count_async, callback=callback)

    def clear(self):
        '''Wipe the whole storage.
        '''
        return self.store_clear()

    def async_clear(self, callback):
        '''Asynchronous version of :meth:`clear`.
        '''
        self._schedule(self.store_clear_async, callback=callback)

    #
    # Operators
    #

    def __setitem__(self, key, values):
        if not isinstance(values, dict):
            raise Exception('Only dict are accepted for the store[key] = dict')
        self.put(key, **values)

    def __getitem__(self, key):
        return self.get(key)

    def __delitem__(self, key):
        return self.keys()

    def __contains__(self, key):
        return self.exists(key)

    def __len__(self):
        return self.count()

    def __iter__(self):
        for key in self.keys():
            yield key

    #
    # Used for implementation
    #

    def store_load(self):
        pass

    def store_sync(self):
        pass

    def store_get(self, key):
        raise NotImplemented()

    def store_put(self, key, value):
        raise NotImplemented()

    def store_exists(self, key):
        raise NotImplemented()

    def store_delete(self, key):
        raise NotImplemented()

    def store_find(self, filters):
        return []

    def store_keys(self):
        return []

    def store_count(self):
        return len(self.store_keys())

    def store_clear(self):
        for key in self.store_keys():
            self.store_delete(key)

    def store_get_async(self, key, callback):
        try:
            value = self.store_get(key)
            callback(self, key, value)
        except KeyError:
            callback(self, key, None)

    def store_put_async(self, key, value, callback):
        try:
            value = self.store_put(key, value)
            callback(self, key, value)
        except:
            callback(self, key, None)

    def store_exists_async(self, key, callback):
        try:
            value = self.store_exists(key)
            callback(self, key, value)
        except:
            callback(self, key, None)

    def store_delete_async(self, key, callback):
        try:
            value = self.store_delete(key)
            callback(self, key, value)
        except:
            callback(self, key, None)

    def store_find_async(self, filters, callback):
        for key, entry in self.store_find(filters):
            callback(self, filters, key, entry)
        callback(self, filters, None, None)

    def store_count_async(self, callback):
        try:
            value = self.store_count()
            callback(self, value)
        except:
            callback(self, 0)

    def store_keys_async(self, callback):
        try:
            keys = self.store_keys()
            callback(self, keys)
        except:
            callback(self, [])

    def store_clear_async(self, callback):
        self.store_clear()
        callback(self)

    #
    # Privates
    #

    def _schedule(self, cb, **kwargs):
        # XXX not entirely sure about the best value (0 or -1).
        Clock.schedule_once(partial(cb, **kwargs), 0)
<|endoftext|>