<commit_before>#
# Kiwi: a Framework and Enhanced Widgets for Python
#
# Copyright (C) 2003-2005 Async Open Source
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
# USA
#
# Author(s): Christian Reis <kiko@async.com.br>
#            Lorenzo Gil Sanchez <lgs@sicem.biz>
#            Johan Dahlin <jdahlin@async.com.br>
#

import gettext
import os
import sys

__all__ = ['Application', 'Library', 'environ', 'require_gazpacho',
           'is_gazpacho_required']

# copied from twisted/python/reflect.py
def namedAny(name):
    """Get a fully named package, module, module-global object, or attribute.
    """
    names = name.split('.')
    topLevelPackage = None
    moduleNames = names[:]
    while not topLevelPackage:
        try:
            trialname = '.'.join(moduleNames)
            topLevelPackage = __import__(trialname)
        except ImportError:
            # if the ImportError happened in the module being imported,
            # this is a failure that should be handed to our caller.
            # count stack frames to tell the difference.
            import traceback
            exc_info = sys.exc_info()
            if len(traceback.extract_tb(exc_info[2])) > 1:
                try:
                    # Clean up garbage left in sys.modules.
                    del sys.modules[trialname]
                except KeyError:
                    # Python 2.4 has fixed this.  Yay!
                    pass
                raise exc_info[0], exc_info[1], exc_info[2]
            moduleNames.pop()

    obj = topLevelPackage
    for n in names[1:]:
        obj = getattr(obj, n)

    return obj

class Environment:
    """Environment control

    When you want to access a resource on the filesystem, such as
    an image or a glade file you use this object.

    External libraries or applications are free to add extra directories"""

    def __init__(self, root='.'):
        self._resources = {}
        self._root = root

        self._add_resource_variable("glade", "KIWI_GLADE_PATH")
        self._add_resource_variable("image", "KIWI_IMAGE_PATH")

    def get_root(self):
        return self._root

    def _add_resource_variable(self, resource, variable):
        """Add resources from an environment variable"""
        env = os.environ.get(variable, '')
        for path in env.split(':'):
            if not path:
                continue
            self.add_resource(resource, env)

    def add_resource(self, resource, path):
        path = os.path.join(self._root, path)
        if not os.path.isdir(path):
            raise EnvironmentError("path %s must be a directory" % path)

        resources = self._resources
        if not resource in resources:
            resources[resource] = [path]
        else:
            resources[resource].append(path)

    def add_resources(self, **kwargs):
        for resource, path in kwargs.items():
            self.add_resource(resource, path)

    def find_resource(self, resource, name):
        """Locate a specific resource of called name of type resource"""

        if not resource in self._resources:
            raise EnvironmentError("No resource called: %s" % resource)

        for path in self._resources[resource]:
            filename = os.path.join(self._root, path, name)
            if os.path.exists(filename):
                return filename

        # Finally try to load the file from the current directory
        filename = os.path.join(self._root, name)
        if os.path.exists(filename):
            return filename

        raise EnvironmentError("Could not find %s resource: %s" % (
            resource, name))

class Library(Environment):
    """A Library is a local environment object, it's a subclass of the
    Environment class.
    It's used by libraries and applications (through the Application class)

    It provides a way to manage local resources, which should only be seen
    in the current context.
    """
    def __init__(self, name, root='..', dirname=None):
        self.name = name
        if dirname == None:
            # Figure out the absolute path to the caller
            caller = sys._getframe(1).f_locals['__file__']
            dirname = os.path.split(caller)[0]

        dirname = os.path.realpath(os.path.abspath(dirname))
        root = os.path.abspath(os.path.join(dirname, root))
        Environment.__init__(self, root=root)

        # Load installed
        try:
            module = __import__(name + '.__installed__',
                                globals(), locals(), name)
        except ImportError, e:
            uninstalled = True
        else:
            uninstalled = False
            if not hasattr(module, 'resources'):
                raise ValueError("module %s.__installed__ must define a "
                                 "resources attribute" % name)
            if not hasattr(module, 'global_resources'):
                raise ValueError("module %s.__installed__ must define a "
                                 "global_resources attribute" % name)
            self.add_resources(**module.resources)
            self.add_global_resources(**module.global_resources)

        self.uninstalled = uninstalled

    def enable_translation(self, domain=None, locale='locale',
                           charset='utf-8'):
        if not domain:
            domain = self.name

        # XXX: locale should not be a list
        localedir = self._resources.get(locale)
        if localedir:
            gettext.bindtextdomain(domain, localedir[0])
        gettext.bind_textdomain_codeset(domain, charset)

    def add_global_resource(self, resource, path):
        """Convenience method to add a global resource.
        This is the same as calling kiwi.environ.environ.add_resource
        """
        global environ
        environ.add_resource(resource, os.path.join(self._root, path))

    def add_global_resources(self, **kwargs):
        for resource, path in kwargs.items():
            self.add_global_resource(resource, path)

class Application(Library):
    """An Application extends a library. The additions are:
    - a run() method, used to run the application
    - path, a reference to the callable object to run, defaults to 'main'
    """
    def __init__(self, name, root='..', path='main'):
        dirname = os.path.abspath(os.path.dirname(sys.argv[0]))
        Library.__init__(self, name, root, dirname)
        self._path = path

    def _get_main(self):
        try:
            module = namedAny(self._path)
        except (ValueError, AttributeError, ImportError), e:
            raise SystemExit("ERROR: Could not find item '%s', %s" %
                             (self._path, e))

        main = getattr(module, 'main', None)
        if not main or not callable(main):
            raise SystemExit("ERROR: Could not find item '%s' in module %s" %
                             'main', path)
        return main

    def enable_translation(self, domain=None, locale='locale',
                           charset='utf-8'):
        if not domain:
            domain = self.name

        Library.enable_translation(self, domain, locale, charset)
        gettext.textdomain(domain)

    def run(self):
        main = self._get_main()

        try:
            sys.exit(main(sys.argv))
        except KeyboardInterrupt:
            raise SystemExit

_require_gazpacho_loader = False

def require_gazpacho():
    global _require_gazpacho_loader
    _require_gazpacho_loader = True

def is_gazpacho_required():
    global _require_gazpacho_loader
    return _require_gazpacho_loader

# Global instance, shared between apps and libraries
environ = Environment()
<commit_msg>pyflakes<commit_after>#
# Kiwi: a Framework and Enhanced Widgets for Python
#
# Copyright (C) 2003-2005 Async Open Source
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
# USA
#
# Author(s): Christian Reis <kiko@async.com.br>
#            Lorenzo Gil Sanchez <lgs@sicem.biz>
#            Johan Dahlin <jdahlin@async.com.br>
#

import gettext
import os
import sys

__all__ = ['Application', 'Library', 'environ', 'require_gazpacho',
           'is_gazpacho_required']

# copied from twisted/python/reflect.py
def namedAny(name):
    """Get a fully named package, module, module-global object, or attribute.
    """
    names = name.split('.')
    topLevelPackage = None
    moduleNames = names[:]
    while not topLevelPackage:
        try:
            trialname = '.'.join(moduleNames)
            topLevelPackage = __import__(trialname)
        except ImportError:
            # if the ImportError happened in the module being imported,
            # this is a failure that should be handed to our caller.
            # count stack frames to tell the difference.
            import traceback
            exc_info = sys.exc_info()
            if len(traceback.extract_tb(exc_info[2])) > 1:
                try:
                    # Clean up garbage left in sys.modules.
                    del sys.modules[trialname]
                except KeyError:
                    # Python 2.4 has fixed this.  Yay!
                    pass
                raise exc_info[0], exc_info[1], exc_info[2]
            moduleNames.pop()

    obj = topLevelPackage
    for n in names[1:]:
        obj = getattr(obj, n)

    return obj

class Environment:
    """Environment control

    When you want to access a resource on the filesystem, such as
    an image or a glade file you use this object.

    External libraries or applications are free to add extra directories"""

    def __init__(self, root='.'):
        self._resources = {}
        self._root = root

        self._add_resource_variable("glade", "KIWI_GLADE_PATH")
        self._add_resource_variable("image", "KIWI_IMAGE_PATH")

    def get_root(self):
        return self._root

    def _add_resource_variable(self, resource, variable):
        """Add resources from an environment variable"""
        env = os.environ.get(variable, '')
        for path in env.split(':'):
            if not path:
                continue
            self.add_resource(resource, env)

    def add_resource(self, resource, path):
        path = os.path.join(self._root, path)
        if not os.path.isdir(path):
            raise EnvironmentError("path %s must be a directory" % path)

        resources = self._resources
        if not resource in resources:
            resources[resource] = [path]
        else:
            resources[resource].append(path)

    def add_resources(self, **kwargs):
        for resource, path in kwargs.items():
            self.add_resource(resource, path)

    def find_resource(self, resource, name):
        """Locate a specific resource of called name of type resource"""

        if not resource in self._resources:
            raise EnvironmentError("No resource called: %s" % resource)

        for path in self._resources[resource]:
            filename = os.path.join(self._root, path, name)
            if os.path.exists(filename):
                return filename

        # Finally try to load the file from the current directory
        filename = os.path.join(self._root, name)
        if os.path.exists(filename):
            return filename

        raise EnvironmentError("Could not find %s resource: %s" % (
            resource, name))

class Library(Environment):
    """A Library is a local environment object, it's a subclass of the
    Environment class.
    It's used by libraries and applications (through the Application class)

    It provides a way to manage local resources, which should only be seen
    in the current context.
    """
    def __init__(self, name, root='..', dirname=None):
        self.name = name
        if dirname == None:
            # Figure out the absolute path to the caller
            caller = sys._getframe(1).f_locals['__file__']
            dirname = os.path.split(caller)[0]

        dirname = os.path.realpath(os.path.abspath(dirname))
        root = os.path.abspath(os.path.join(dirname, root))
        Environment.__init__(self, root=root)

        # Load installed
        try:
            module = __import__(name + '.__installed__',
                                globals(), locals(), name)
        except ImportError, e:
            uninstalled = True
        else:
            uninstalled = False
            if not hasattr(module, 'resources'):
                raise ValueError("module %s.__installed__ must define a "
                                 "resources attribute" % name)
            if not hasattr(module, 'global_resources'):
                raise ValueError("module %s.__installed__ must define a "
                                 "global_resources attribute" % name)
            self.add_resources(**module.resources)
            self.add_global_resources(**module.global_resources)

        self.uninstalled = uninstalled

    def enable_translation(self, domain=None, locale='locale',
                           charset='utf-8'):
        if not domain:
            domain = self.name

        # XXX: locale should not be a list
        localedir = self._resources.get(locale)
        if localedir:
            gettext.bindtextdomain(domain, localedir[0])
        gettext.bind_textdomain_codeset(domain, charset)

    def add_global_resource(self, resource, path):
        """Convenience method to add a global resource.
        This is the same as calling kiwi.environ.environ.add_resource
        """
        global environ
        environ.add_resource(resource, os.path.join(self._root, path))

    def add_global_resources(self, **kwargs):
        for resource, path in kwargs.items():
            self.add_global_resource(resource, path)

class Application(Library):
    """An Application extends a library. The additions are:
    - a run() method, used to run the application
    - path, a reference to the callable object to run, defaults to 'main'
    """
    def __init__(self, name, root='..', path='main'):
        dirname = os.path.abspath(os.path.dirname(sys.argv[0]))
        Library.__init__(self, name, root, dirname)
        self._path = path

    def _get_main(self):
        try:
            module = namedAny(self._path)
        except (ValueError, AttributeError, ImportError), e:
            raise SystemExit("ERROR: Could not find item '%s', %s" %
                             (self._path, e))

        main = getattr(module, 'main', None)
        if not main or not callable(main):
            raise SystemExit("ERROR: Could not find item '%s' in module %s" %
                             'main', self._path)
        return main

    def enable_translation(self, domain=None, locale='locale',
                           charset='utf-8'):
        if not domain:
            domain = self.name

        Library.enable_translation(self, domain, locale, charset)
        gettext.textdomain(domain)

    def run(self):
        main = self._get_main()

        try:
            sys.exit(main(sys.argv))
        except KeyboardInterrupt:
            raise SystemExit

_require_gazpacho_loader = False

def require_gazpacho():
    global _require_gazpacho_loader
    _require_gazpacho_loader = True

def is_gazpacho_required():
    global _require_gazpacho_loader
    return _require_gazpacho_loader

# Global instance, shared between apps and libraries
environ = Environment()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2013-2015 Marcos Organizador de Negocios SRL http://marcos.do
#    Write by Eneldo Serrata (eneldo@marcos.do)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

{
    'name': 'Dominican Republic - Payroll',
    'version': '1.0',
    'category': 'Localization',
    'description': """
This is the base module to manage the payroll for Dominican Republic.
==============================================================================
    * Base salary rules for Dominican Republic
    """,
    'author': 'Manuel Guilamo & Eneldo Serrata - Marcos Organizador de Negocios, SRL.',
    'website': 'http://marcos.do',

    # Categories can be used to filter modules in modules listing
    # Check https://github.com/odoo/odoo/blob/master/openerp/addons/base/module/module_data.xml
    # for the full list
    'version': '0.1',

    # any module necessary for this one to work correctly
    'depends': ['base', 'account', 'marcos_l10n_do', "hr", "hr_payroll", "hr_payslip_paid"],

    # always loaded
    'data': [
        'views/views.xml',
        'data/hr.salary.rule.category.csv',
        'data/hr.salary.rule.csv',
        # 'data/no_account/hr.salary.rule.csv',
        'data/hr.payroll.structure.csv',
        # 'security/ir.model.access.csv',
        'templates.xml',
    ],
    # only loaded in demonstration mode
    'demo': [
        'demo.xml',
    ],
}<commit_msg>just test<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2013-2015 Marcos Organizador de Negocios SRL http://marcos.do
#    Write by Eneldo Serrata (eneldo@marcos.do)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

{
    'name': 'Dominican Republic - Payroll',
    'version': '1.0',
    'category': 'Localization',
    'description': """
This is the base module to manage the payroll for Dominican Republic.
==============================================================================
    * Base salary rules for Dominican Republic
    """,
    'author': 'Manuel Guilamo & Eneldo Serrata - Marcos Organizador de Negocios, SRL.',
    'website': 'http://marcos.do',

    # Categories can be used to filter modules in modules listing
    # Check https://github.com/odoo/odoo/blob/master/openerp/addons/base/module/module_data.xml
    # for the full list
    'version': '0.1',

    # any module necessary for this one to work correctly
    'depends': ['base', 'account', 'marcos_l10n_do', "hr", "hr_payroll", "hr_payslip_paid"],

    # always loaded
    'data': [
        # 'views/views.xml',
        # 'data/hr.salary.rule.category.csv',
        # 'data/hr.salary.rule.csv',
        # 'data/no_account/hr.salary.rule.csv',
        # 'data/hr.payroll.structure.csv',
        # 'security/ir.model.access.csv',
        # 'templates.xml',
    ],
    # only loaded in demonstration mode
    'demo': [
        'demo.xml',
    ],
}<|endoftext|>


<commit_before>"""
kombu.pidbox
===============

Generic process mailbox.

:copyright: (c) 2009 - 2012 by Ask Solem.
:license: BSD, see LICENSE for more details.

"""
from __future__ import absolute_import

import socket

from copy import copy
from itertools import count

from .entity import Exchange, Queue
from .messaging import Consumer, Producer
from .utils import kwdict, uuid

__all__ = ["Node", "Mailbox"]

class Node(object):

    #: hostname of the node.
    hostname = None

    #: the :class:`Mailbox` this is a node for.
    mailbox = None

    #: map of method name/handlers.
    handlers = None

    #: current context (passed on to handlers)
    state = None

    #: current channel.
    channel = None

    def __init__(self, hostname, state=None, channel=None, handlers=None,
            mailbox=None):
        self.channel = channel
        self.mailbox = mailbox
        self.hostname = hostname
        self.state = state
        if handlers is None:
            handlers = {}
        self.handlers = handlers

    def Consumer(self, channel=None, **options):
        options.setdefault("no_ack", True)
        return Consumer(channel or self.channel,
                        [self.mailbox.get_queue(self.hostname)],
                        **options)

    def handler(self, fun):
        self.handlers[fun.__name__] = fun
        return fun

    def listen(self, channel=None, callback=None):
        callback = callback or self.handle_message
        consumer = self.Consumer(channel=channel,
                                 callbacks=[callback or self.handle_message])
        consumer.consume()
        return consumer

    def dispatch_from_message(self, message):
        message = dict(message)
        method = message["method"]
        destination = message.get("destination")
        reply_to = message.get("reply_to")
        arguments = message.get("arguments")
        if not destination or self.hostname in destination:
            return self.dispatch(method, arguments, reply_to)

    def dispatch(self, method, arguments=None, reply_to=None):
        arguments = arguments or {}
        handle = reply_to and self.handle_call or self.handle_cast
        try:
            reply = handle(method, kwdict(arguments))
        except SystemExit:
            raise
        except Exception, exc:
            reply = {"error": repr(exc)}

        if reply_to:
            self.reply({self.hostname: reply},
                       exchange=reply_to["exchange"],
                       routing_key=reply_to["routing_key"])
        return reply

    def handle(self, method, arguments={}):
        return self.handlers[method](self.state, **arguments)

    def handle_call(self, method, arguments):
        return self.handle(method, arguments)

    def handle_cast(self, method, arguments):
        return self.handle(method, arguments)

    def handle_message(self, body, message):
        return self.dispatch_from_message(body)

    def reply(self, data, exchange, routing_key, **kwargs):
        self.mailbox._publish_reply(data, exchange, routing_key,
                                    channel=self.channel)

class Mailbox(object):
    node_cls = Node
    exchange_fmt = "%s.pidbox"
    reply_exchange_fmt = "reply.%s.pidbox"

    #: Name of application.
    namespace = None

    #: Connection (if bound).
    connection = None

    #: Exchange type (usually direct, or fanout for broadcast).
    type = "direct"

    #: mailbox exchange (init by constructor).
    exchange = None

    #: exchange to send replies to.
    reply_exchange = None

    def __init__(self, namespace, type="direct", connection=None):
        self.namespace = namespace
        self.connection = connection
        self.type = type
        self.exchange = self._get_exchange(self.namespace, self.type)
        self.reply_exchange = self._get_reply_exchange(self.namespace)

    def __call__(self, connection):
        bound = copy(self)
        bound.connection = connection
        return bound

    def Node(self, hostname=None, state=None, channel=None, handlers=None):
        hostname = hostname or socket.gethostname()
        return self.node_cls(hostname, state, channel, handlers, mailbox=self)

    def call(self, destination, command, kwargs={}, timeout=None,
            callback=None, channel=None):
        return self._broadcast(command, kwargs, destination,
                               reply=True, timeout=timeout,
                               callback=callback,
                               channel=channel)

    def cast(self, destination, command, kwargs={}):
        return self._broadcast(command, kwargs, destination, reply=False)

    def abcast(self, command, kwargs={}):
        return self._broadcast(command, kwargs, reply=False)

    def multi_call(self, command, kwargs={}, timeout=1,
            limit=None, callback=None, channel=None):
        return self._broadcast(command, kwargs, reply=True,
                               timeout=timeout, limit=limit,
                               callback=callback,
                               channel=channel)

    def get_reply_queue(self, ticket):
        return Queue("%s.%s" % (ticket, self.reply_exchange.name),
                     exchange=self.reply_exchange,
                     routing_key=ticket,
                     durable=False,
                     auto_delete=True)

    def get_queue(self, hostname):
        return Queue("%s.%s.pidbox" % (hostname, self.namespace),
                     exchange=self.exchange,
                     durable=False,
                     auto_delete=True)

    def _publish_reply(self, reply, exchange, routing_key, channel=None):
        chan = channel or self.connection.channel()
        try:
            exchange = Exchange(exchange, exchange_type="direct",
                                          delivery_mode="transient",
                                          durable=False,
                                          auto_delete=True)
            producer = Producer(chan, exchange=exchange,
                                      auto_declare=True)
            producer.publish(reply, routing_key=routing_key)
        finally:
            channel or chan.close()

    def _publish(self, type, arguments, destination=None, reply_ticket=None,
            channel=None):
        message = {"method": type,
                   "arguments": arguments,
                   "destination": destination}
        if reply_ticket:
            message["reply_to"] = {"exchange": self.reply_exchange.name,
                                   "routing_key": reply_ticket}
        chan = channel or self.connection.channel()
        producer = Producer(chan, exchange=self.exchange)
        try:
            producer.publish(message)
        finally:
            channel or chan.close()

    def _broadcast(self, command, arguments=None, destination=None,
            reply=False, timeout=1, limit=None, callback=None, channel=None):
        if destination is not None and \
                not isinstance(destination, (list, tuple)):
            raise ValueError("destination must be a list/tuple not %s" % (
                    type(destination)))

        arguments = arguments or {}
        reply_ticket = reply and uuid() or None
        chan = channel or self.connection.default_channel

        # Set reply limit to number of destinations (if specified)
        if limit is None and destination:
            limit = destination and len(destination) or None

        if reply_ticket:
            self.get_reply_queue(reply_ticket)(chan).declare()

        self._publish(command, arguments, destination=destination,
                                          reply_ticket=reply_ticket,
                                          channel=chan)

        if reply_ticket:
            return self._collect(reply_ticket, limit=limit,
                                               timeout=timeout,
                                               callback=callback,
                                               channel=chan)

    def _collect(self, ticket, limit=None, timeout=1,
            callback=None, channel=None):
        chan = channel or self.connection.channel()
        queue = self.get_reply_queue(ticket)
        consumer = Consumer(channel, [queue], no_ack=True)
        responses = []

        def on_message(body, message):
            if callback:
                callback(body)
            responses.append(body)

        try:
            consumer.register_callback(on_message)
            consumer.consume()
            for i in limit and range(limit) or count():
                try:
                    self.connection.drain_events(timeout=timeout)
                except socket.timeout:
                    break
            chan.after_reply_message_received(queue.name)
            return responses
        finally:
            channel or chan.close()

    def _get_exchange(self, namespace, type):
        return Exchange(self.exchange_fmt % namespace,
                        type=type,
                        durable=False,
                        auto_delete=True,
                        delivery_mode="transient")

    def _get_reply_exchange(self, namespace):
        return Exchange(self.reply_exchange_fmt % namespace,
                        type="direct",
                        durable=False,
                        auto_delete=True,
                        delivery_mode="transient")
<commit_msg>Pidbox: Now uses connection.default_channel<commit_after>"""
kombu.pidbox
===============

Generic process mailbox.

:copyright: (c) 2009 - 2012 by Ask Solem.
:license: BSD, see LICENSE for more details.

"""
from __future__ import absolute_import

import socket

from copy import copy
from itertools import count

from .entity import Exchange, Queue
from .messaging import Consumer, Producer
from .utils import kwdict, uuid

__all__ = ["Node", "Mailbox"]

class Node(object):

    #: hostname of the node.
    hostname = None

    #: the :class:`Mailbox` this is a node for.
    mailbox = None

    #: map of method name/handlers.
    handlers = None

    #: current context (passed on to handlers)
    state = None

    #: current channel.
    channel = None

    def __init__(self, hostname, state=None, channel=None, handlers=None,
            mailbox=None):
        self.channel = channel
        self.mailbox = mailbox
        self.hostname = hostname
        self.state = state
        if handlers is None:
            handlers = {}
        self.handlers = handlers

    def Consumer(self, channel=None, **options):
        options.setdefault("no_ack", True)
        return Consumer(channel or self.channel,
                        [self.mailbox.get_queue(self.hostname)],
                        **options)

    def handler(self, fun):
        self.handlers[fun.__name__] = fun
        return fun

    def listen(self, channel=None, callback=None):
        callback = callback or self.handle_message
        consumer = self.Consumer(channel=channel,
                                 callbacks=[callback or self.handle_message])
        consumer.consume()
        return consumer

    def dispatch_from_message(self, message):
        message = dict(message)
        method = message["method"]
        destination = message.get("destination")
        reply_to = message.get("reply_to")
        arguments = message.get("arguments")
        if not destination or self.hostname in destination:
            return self.dispatch(method, arguments, reply_to)

    def dispatch(self, method, arguments=None, reply_to=None):
        arguments = arguments or {}
        handle = reply_to and self.handle_call or self.handle_cast
        try:
            reply = handle(method, kwdict(arguments))
        except SystemExit:
            raise
        except Exception, exc:
            reply = {"error": repr(exc)}

        if reply_to:
            self.reply({self.hostname: reply},
                       exchange=reply_to["exchange"],
                       routing_key=reply_to["routing_key"])
        return reply

    def handle(self, method, arguments={}):
        return self.handlers[method](self.state, **arguments)

    def handle_call(self, method, arguments):
        return self.handle(method, arguments)

    def handle_cast(self, method, arguments):
        return self.handle(method, arguments)

    def handle_message(self, body, message):
        return self.dispatch_from_message(body)

    def reply(self, data, exchange, routing_key, **kwargs):
        self.mailbox._publish_reply(data, exchange, routing_key,
                                    channel=self.channel)

class Mailbox(object):
    node_cls = Node
    exchange_fmt = "%s.pidbox"
    reply_exchange_fmt = "reply.%s.pidbox"

    #: Name of application.
    namespace = None

    #: Connection (if bound).
    connection = None

    #: Exchange type (usually direct, or fanout for broadcast).
    type = "direct"

    #: mailbox exchange (init by constructor).
    exchange = None

    #: exchange to send replies to.
    reply_exchange = None

    def __init__(self, namespace, type="direct", connection=None):
        self.namespace = namespace
        self.connection = connection
        self.type = type
        self.exchange = self._get_exchange(self.namespace, self.type)
        self.reply_exchange = self._get_reply_exchange(self.namespace)

    def __call__(self, connection):
        bound = copy(self)
        bound.connection = connection
        return bound

    def Node(self, hostname=None, state=None, channel=None, handlers=None):
        hostname = hostname or socket.gethostname()
        return self.node_cls(hostname, state, channel, handlers, mailbox=self)

    def call(self, destination, command, kwargs={}, timeout=None,
            callback=None, channel=None):
        return self._broadcast(command, kwargs, destination,
                               reply=True, timeout=timeout,
                               callback=callback,
                               channel=channel)

    def cast(self, destination, command, kwargs={}):
        return self._broadcast(command, kwargs, destination, reply=False)

    def abcast(self, command, kwargs={}):
        return self._broadcast(command, kwargs, reply=False)

    def multi_call(self, command, kwargs={}, timeout=1,
            limit=None, callback=None, channel=None):
        return self._broadcast(command, kwargs, reply=True,
                               timeout=timeout, limit=limit,
                               callback=callback,
                               channel=channel)

    def get_reply_queue(self, ticket):
        return Queue("%s.%s" % (ticket, self.reply_exchange.name),
                     exchange=self.reply_exchange,
                     routing_key=ticket,
                     durable=False,
                     auto_delete=True)

    def get_queue(self, hostname):
        return Queue("%s.%s.pidbox" % (hostname, self.namespace),
                     exchange=self.exchange,
                     durable=False,
                     auto_delete=True)

    def _publish_reply(self, reply, exchange, routing_key, channel=None):
        chan = channel or self.connection.default_channel
        exchange = Exchange(exchange, exchange_type="direct",
                                      delivery_mode="transient",
                                      durable=False,
                                      auto_delete=True)
        producer = Producer(chan, exchange=exchange,
                                  auto_declare=True)
        producer.publish(reply, routing_key=routing_key)

    def _publish(self, type, arguments, destination=None, reply_ticket=None,
            channel=None):
        message = {"method": type,
                   "arguments": arguments,
                   "destination": destination}
        if reply_ticket:
            message["reply_to"] = {"exchange": self.reply_exchange.name,
                                   "routing_key": reply_ticket}
        chan = channel or self.connection.default_channel
        producer = Producer(chan, exchange=self.exchange)
        producer.publish(message)

    def _broadcast(self, command, arguments=None, destination=None,
            reply=False, timeout=1, limit=None, callback=None, channel=None):
        if destination is not None and \
                not isinstance(destination, (list, tuple)):
            raise ValueError("destination must be a list/tuple not %s" % (
                    type(destination)))

        arguments = arguments or {}
        reply_ticket = reply and uuid() or None
        chan = channel or self.connection.default_channel

        # Set reply limit to number of destinations (if specified)
        if limit is None and destination:
            limit = destination and len(destination) or None

        if reply_ticket:
            self.get_reply_queue(reply_ticket)(chan).declare()

        self._publish(command, arguments, destination=destination,
                                          reply_ticket=reply_ticket,
                                          channel=chan)

        if reply_ticket:
            return self._collect(reply_ticket, limit=limit,
                                               timeout=timeout,
                                               callback=callback,
                                               channel=chan)

    def _collect(self, ticket, limit=None, timeout=1,
            callback=None, channel=None):
        chan = channel or self.connection.default_channel
        queue = self.get_reply_queue(ticket)
        consumer = Consumer(channel, [queue], no_ack=True)
        responses = []

        def on_message(body, message):
            if callback:
                callback(body)
            responses.append(body)

        consumer.register_callback(on_message)
        with consumer:
            for i in limit and range(limit) or count():
                try:
                    self.connection.drain_events(timeout=timeout)
                except socket.timeout:
                    break
            return responses
        chan.after_reply_message_received(queue.name)

    def _get_exchange(self, namespace, type):
        return Exchange(self.exchange_fmt % namespace,
                        type=type,
                        durable=False,
                        auto_delete=True,
                        delivery_mode="transient")

    def _get_reply_exchange(self, namespace):
        return Exchange(self.reply_exchange_fmt % namespace,
                        type="direct",
                        durable=False,
                        auto_delete=True,
                        delivery_mode="transient")
<|endoftext|>


<commit_before># Copyright (C) 2015 Curoverse, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
from __future__ import print_function
from future import standard_library
standard_library.install_aliases()
from builtins import str
import json
import os
import subprocess
import re
import shutil
import urllib.request, urllib.parse, urllib.error
import zipfile

# Python 3 compatibility imports
from six.moves import StringIO
from six import u as str

from toil.test import ToilTest, needs_cwl

@needs_cwl
class CWLTest(ToilTest):

    def _tester(self, cwlfile, jobfile, outDir, expect):
        from toil.cwl import cwltoil
        rootDir = self._projectRootPath()
        st = StringIO()
        cwltoil.main(['--outdir', outDir,
                            os.path.join(rootDir, cwlfile),
                            os.path.join(rootDir, jobfile)],
                     stdout=st)
        out = json.loads(st.getvalue())
        out["output"].pop("http://commonwl.org/cwltool#generation", None)
        out["output"].pop("nameext", None)
        out["output"].pop("nameroot", None)
        self.assertEquals(out, expect)

    def test_run_revsort(self):
        outDir = self._createTempDir()
        self._tester('src/toil/test/cwl/revsort.cwl',
                     'src/toil/test/cwl/revsort-job.json',
                     outDir, {
            # Having unicode string literals isn't necessary for the assertion but makes for a
            # less noisy diff in case the assertion fails.
            u'output': {
                u'location': "file://" + str(os.path.join(outDir, 'output.txt')),
                u'basename': str("output.txt"),
                u'size': 1111,
                u'class': u'File',
                u'checksum': u'sha1$b9214658cc453331b62c2282b772a5c063dbd284'}})

    def test_restart(self):
        """Enable restarts with CWLtoil -- run failing test, re-run correct test.
        """
        from toil.cwl import cwltoil
        from toil.jobStores.abstractJobStore import NoSuchJobStoreException
        from toil.leader import FailedJobsException
        outDir = self._createTempDir()
        cwlDir = os.path.join(self._projectRootPath(), "src", "toil", "test", "cwl")
        cmd = ['--outdir', outDir, '--jobStore', os.path.join(outDir, 'jobStore'), "--no-container",
               os.path.join(cwlDir, "revsort.cwl"), os.path.join(cwlDir, "revsort-job.json")]
        def path_without_rev():
            return ":".join([d for d in os.environ["PATH"].split(":")
                             if not os.path.exists(os.path.join(d, "rev"))])
        orig_path = os.environ["PATH"]
        # Force a failure and half finished job by removing `rev` from the PATH
        os.environ["PATH"] = path_without_rev()
        try:
            cwltoil.main(cmd)
            self.fail("Expected problem job with incorrect PATH did not fail")
        except FailedJobsException:
            pass
        # Finish the job with a correct PATH
        os.environ["PATH"] = orig_path
        cwltoil.main(["--restart"] + cmd)
        # Should fail because previous job completed successfully
        try:
            cwltoil.main(["--restart"] + cmd)
            self.fail("Restart with missing directory did not fail")
        except NoSuchJobStoreException:
            pass

    def test_run_conformance(self):
        rootDir = self._projectRootPath()
        cwlSpec = os.path.join(rootDir, 'src/toil/test/cwl/spec')
        testhash = "7f510ec768b424601beb8c86700343afe722ac76"
        url = "https://github.com/common-workflow-language/common-workflow-language/archive/%s.zip" % testhash
        if not os.path.exists(cwlSpec):
            urllib.request.urlretrieve(url, "spec.zip")
            with zipfile.ZipFile('spec.zip', "r") as z:
                z.extractall()
            shutil.move("common-workflow-language-%s" % testhash, cwlSpec)
            os.remove("spec.zip")
        try:
            subprocess.check_output(["bash", "run_test.sh", "RUNNER=toil-cwl-runner", "DRAFT=v1.0"], cwd=cwlSpec,
                                    stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            only_unsupported = False
            # check output -- if we failed but only have unsupported features, we're okay
            p = re.compile(r"(?P<failures>\d+) failures, (?P<unsupported>\d+) unsupported features")
            for line in e.output.split("\n"):
                m = p.search(line)
                if m:
                    if int(m.group("failures")) == 0 and int(m.group("unsupported")) > 0:
                        only_unsupported = True
                        break
            if not only_unsupported:
                print(e.output)
                raise e
<commit_msg>Trying alternative urllib modules<commit_after># Copyright (C) 2015 Curoverse, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
from __future__ import print_function
from future import standard_library
from future.standard_library import hooks

import json
import os
import subprocess
import re
import shutil
with hooks():
    import urllib.request, urllib.parse, urllib.error
import zipfile

# Python 3 compatibility imports
from six.moves import StringIO
from six import u as str

from toil.test import ToilTest, needs_cwl

@needs_cwl
class CWLTest(ToilTest):

    def _tester(self, cwlfile, jobfile, outDir, expect):
        from toil.cwl import cwltoil
        rootDir = self._projectRootPath()
        st = StringIO()
        cwltoil.main(['--outdir', outDir,
                            os.path.join(rootDir, cwlfile),
                            os.path.join(rootDir, jobfile)],
                     stdout=st)
        out = json.loads(st.getvalue())
        out["output"].pop("http://commonwl.org/cwltool#generation", None)
        out["output"].pop("nameext", None)
        out["output"].pop("nameroot", None)
        self.assertEquals(out, expect)

    def test_run_revsort(self):
        outDir = self._createTempDir()
        self._tester('src/toil/test/cwl/revsort.cwl',
                     'src/toil/test/cwl/revsort-job.json',
                     outDir, {
            # Having unicode string literals isn't necessary for the assertion but makes for a
            # less noisy diff in case the assertion fails.
            u'output': {
                u'location': "file://" + str(os.path.join(outDir, 'output.txt')),
                u'basename': str("output.txt"),
                u'size': 1111,
                u'class': u'File',
                u'checksum': u'sha1$b9214658cc453331b62c2282b772a5c063dbd284'}})

    def test_restart(self):
        """Enable restarts with CWLtoil -- run failing test, re-run correct test.
        """
        from toil.cwl import cwltoil
        from toil.jobStores.abstractJobStore import NoSuchJobStoreException
        from toil.leader import FailedJobsException
        outDir = self._createTempDir()
        cwlDir = os.path.join(self._projectRootPath(), "src", "toil", "test", "cwl")
        cmd = ['--outdir', outDir, '--jobStore', os.path.join(outDir, 'jobStore'), "--no-container",
               os.path.join(cwlDir, "revsort.cwl"), os.path.join(cwlDir, "revsort-job.json")]
        def path_without_rev():
            return ":".join([d for d in os.environ["PATH"].split(":")
                             if not os.path.exists(os.path.join(d, "rev"))])
        orig_path = os.environ["PATH"]
        # Force a failure and half finished job by removing `rev` from the PATH
        os.environ["PATH"] = path_without_rev()
        try:
            cwltoil.main(cmd)
            self.fail("Expected problem job with incorrect PATH did not fail")
        except FailedJobsException:
            pass
        # Finish the job with a correct PATH
        os.environ["PATH"] = orig_path
        cwltoil.main(["--restart"] + cmd)
        # Should fail because previous job completed successfully
        try:
            cwltoil.main(["--restart"] + cmd)
            self.fail("Restart with missing directory did not fail")
        except NoSuchJobStoreException:
            pass

    def test_run_conformance(self):
        rootDir = self._projectRootPath()
        cwlSpec = os.path.join(rootDir, 'src/toil/test/cwl/spec')
        testhash = "7f510ec768b424601beb8c86700343afe722ac76"
        url = "https://github.com/common-workflow-language/common-workflow-language/archive/%s.zip" % testhash
        if not os.path.exists(cwlSpec):
            urllib.request.urlretrieve(url, "spec.zip")
            with zipfile.ZipFile('spec.zip', "r") as z:
                z.extractall()
            shutil.move("common-workflow-language-%s" % testhash, cwlSpec)
            os.remove("spec.zip")
        try:
            subprocess.check_output(["bash", "run_test.sh", "RUNNER=toil-cwl-runner", "DRAFT=v1.0"], cwd=cwlSpec,
                                    stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            only_unsupported = False
            # check output -- if we failed but only have unsupported features, we're okay
            p = re.compile(r"(?P<failures>\d+) failures, (?P<unsupported>\d+) unsupported features")
            for line in e.output.split("\n"):
                m = p.search(line)
                if m:
                    if int(m.group("failures")) == 0 and int(m.group("unsupported")) > 0:
                        only_unsupported = True
                        break
            if not only_unsupported:
                print(e.output)
                raise e
<|endoftext|>


<commit_before># Copyright (C) 2015 Curoverse, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
from __future__ import print_function
import json
import os
from toil import subprocess
import unittest
import re
import shutil
import pytest
from future.moves.urllib.request import urlretrieve
import zipfile

# Python 3 compatibility imports
from six.moves import StringIO
from six import u as str

from toil.test import (ToilTest, needs_cwl, slow, needs_docker, needs_lsf,
                       needs_mesos, needs_parasol, needs_gridengine, needs_slurm,
                       needs_torque)

@needs_cwl
class CWLTest(ToilTest):

    def _tester(self, cwlfile, jobfile, outDir, expect, main_args=[], out_name="output"):
        from toil.cwl import cwltoil
        rootDir = self._projectRootPath()
        st = StringIO()
        main_args = main_args[:]
        main_args.extend(['--outdir', outDir, os.path.join(rootDir, cwlfile), os.path.join(rootDir, jobfile)])
        cwltoil.main(main_args, stdout=st)
        out = json.loads(st.getvalue())
        out[out_name].pop("http://commonwl.org/cwltool#generation", None)
        out[out_name].pop("nameext", None)
        out[out_name].pop("nameroot", None)
        self.assertEquals(out, expect)

    def _debug_worker_tester(self, cwlfile, jobfile, outDir, expect):
        from toil.cwl import cwltoil
        rootDir = self._projectRootPath()
        st = StringIO()
        cwltoil.main(['--debugWorker', '--outdir', outDir,
                     os.path.join(rootDir, cwlfile),
                     os.path.join(rootDir, jobfile)], stdout=st)
        out = json.loads(st.getvalue())
        out["output"].pop("http://commonwl.org/cwltool#generation", None)
        out["output"].pop("nameext", None)
        out["output"].pop("nameroot", None)
        self.assertEquals(out, expect)

    def test_run_revsort(self):
        outDir = self._createTempDir()
        self._tester('src/toil/test/cwl/revsort.cwl',
                     'src/toil/test/cwl/revsort-job.json',
                     outDir, {
            # Having unicode string literals isn't necessary for the assertion but makes for a
            # less noisy diff in case the assertion fails.
            u'output': {
                u'location': "file://" + str(os.path.join(outDir, 'output.txt')),
                u'basename': str("output.txt"),
                u'size': 1111,
                u'class': u'File',
                u'checksum': u'sha1$b9214658cc453331b62c2282b772a5c063dbd284'}})

    def test_run_revsort2(self):
        outDir = self._createTempDir()
        self._tester('src/toil/test/cwl/revsort2.cwl',
                     'src/toil/test/cwl/revsort-job.json',
                     outDir, {
            # Having unicode string literals isn't necessary for the assertion but makes for a
            # less noisy diff in case the assertion fails.
            u'output': {
                u'location': "file://" + str(os.path.join(outDir, 'output.txt')),
                u'basename': str("output.txt"),
                u'size': 1111,
                u'class': u'File',
                u'checksum': u'sha1$b9214658cc453331b62c2282b772a5c063dbd284'}})

    def test_run_revsort_debug_worker(self):
        outDir = self._createTempDir()
        # Having unicode string literals isn't necessary for the assertion
        # but makes for a less noisy diff in case the assertion fails.
        self._debug_worker_tester(
                'src/toil/test/cwl/revsort.cwl',
                'src/toil/test/cwl/revsort-job.json', outDir, {u'output': {
                    u'location': "file://" + str(os.path.join(
                        outDir, 'output.txt')),
                    u'basename': str("output.txt"),
                    u'size': 1111,
                    u'class': u'File',
                    u'checksum':
                        u'sha1$b9214658cc453331b62c2282b772a5c063dbd284'}})

    @slow
    def test_restart(self):
        """Enable restarts with CWLtoil -- run failing test, re-run correct test.
        """
        from toil.cwl import cwltoil
        from toil.jobStores.abstractJobStore import NoSuchJobStoreException
        from toil.leader import FailedJobsException
        outDir = self._createTempDir()
        cwlDir = os.path.join(self._projectRootPath(), "src", "toil", "test", "cwl")
        cmd = ['--outdir', outDir, '--jobStore', os.path.join(outDir, 'jobStore'), "--no-container",
               os.path.join(cwlDir, "revsort.cwl"), os.path.join(cwlDir, "revsort-job.json")]

        def path_without_rev():
            return ":".join([d for d in os.environ["PATH"].split(":")
                             if not os.path.exists(os.path.join(d, "rev"))])
        orig_path = os.environ["PATH"]
        # Force a failure and half finished job by removing `rev` from the PATH
        os.environ["PATH"] = path_without_rev()
        try:
            cwltoil.main(cmd)
            self.fail("Expected problem job with incorrect PATH did not fail")
        except FailedJobsException:
            pass
        # Finish the job with a correct PATH
        os.environ["PATH"] = orig_path
        cwltoil.main(["--restart"] + cmd)
        # Should fail because previous job completed successfully
        try:
            cwltoil.main(["--restart"] + cmd)
            self.fail("Restart with missing directory did not fail")
        except NoSuchJobStoreException:
            pass

    @slow
    @pytest.mark.timeout(1800)
    def test_run_conformance(self, batchSystem=None):
        rootDir = self._projectRootPath()
        cwlSpec = os.path.join(rootDir, 'src/toil/test/cwl/spec')
        # The latest cwl git hash. Update it to get the latest tests.
        testhash = "f96bca6911b6688ff614c02dbefe819bed260a13"
        url = "https://github.com/common-workflow-language/common-workflow-language/archive/%s.zip" % testhash
        if not os.path.exists(cwlSpec):
            urlretrieve(url, "spec.zip")
            with zipfile.ZipFile('spec.zip', "r") as z:
                z.extractall()
            shutil.move("common-workflow-language-%s" % testhash, cwlSpec)
            os.remove("spec.zip")
        try:
            cmd = ['bash', 'run_test.sh', 'RUNNER=toil-cwl-runner', 'DRAFT=v1.0',
                   'TEST_N="--timeout=1800"', '-j4']
            if batchSystem:
                cmd.extend(["--batchSystem", batchSystem])
            subprocess.check_output(cmd, cwd=cwlSpec, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            only_unsupported = False
            # check output -- if we failed but only have unsupported features, we're okay
            p = re.compile(r"(?P<failures>\d+) failures, (?P<unsupported>\d+) unsupported features")
            for line in e.output.split("\n"):
                m = p.search(line)
                if m:
                    if int(m.group("failures")) == 0 and int(m.group("unsupported")) > 0:
                        only_unsupported = True
                        break
            if not only_unsupported:
                print(e.output)
                raise e

    @slow
    def test_bioconda(self):
        outDir = self._createTempDir()
        self._tester('src/toil/test/cwl/seqtk_seq.cwl',
                     'src/toil/test/cwl/seqtk_seq_job.json',
                     outDir,
                     self._expected_seqtk_output(outDir),
                     main_args=["--beta-conda-dependencies"],
                     out_name="output1")

    @needs_docker
    def test_biocontainers(self):
        outDir = self._createTempDir()
        self._tester('src/toil/test/cwl/seqtk_seq.cwl',
                     'src/toil/test/cwl/seqtk_seq_job.json',
                     outDir,
                     self._expected_seqtk_output(outDir),
                     main_args=["--beta-use-biocontainers"],
                     out_name="output1")

    @slow
    @needs_lsf
    @unittest.skip
    def test_lsf_cwl_conformance(self):
        return self.test_run_conformance("LSF")

    @slow
    @needs_slurm
    @unittest.skip
    def test_slurm_cwl_conformance(self):
        return self.test_run_conformance("Slurm")

    @slow
    @needs_torque
    @unittest.skip
    def test_torque_cwl_conformance(self):
        return self.test_run_conformance("Torque")

    @slow
    @needs_gridengine
    @unittest.skip
    def test_gridengine_cwl_conformance(self):
        return self.test_run_conformance("gridEngine")

    @slow
    @needs_mesos
    @unittest.skip
    def test_mesos_cwl_conformance(self):
        return self.test_run_conformance("mesos")

    @slow
    @needs_parasol
    @unittest.skip
    def test_parasol_cwl_conformance(self):
        return self.test_run_conformance("parasol")

    def _expected_seqtk_output(self, outDir):
        return {
            u"output1":  {
                u"location": "file://" + str(os.path.join(outDir, 'out')),
                u"checksum": u"sha1$322e001e5a99f19abdce9f02ad0f02a17b5066c2",
                u"basename": str("out"),
                u"class": u"File",
                u"size": 150}
        }
<commit_msg>Call cwltest directly.<commit_after># Copyright (C) 2015 Curoverse, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
from __future__ import print_function
import json
import os
from toil import subprocess
import unittest
import re
import shutil
import pytest
from future.moves.urllib.request import urlretrieve
import zipfile

# Python 3 compatibility imports
from six.moves import StringIO
from six import u as str

from toil.test import (ToilTest, needs_cwl, slow, needs_docker, needs_lsf,
                       needs_mesos, needs_parasol, needs_gridengine, needs_slurm,
                       needs_torque)

@needs_cwl
class CWLTest(ToilTest):

    def _tester(self, cwlfile, jobfile, outDir, expect, main_args=[], out_name="output"):
        from toil.cwl import cwltoil
        rootDir = self._projectRootPath()
        st = StringIO()
        main_args = main_args[:]
        main_args.extend(['--outdir', outDir, os.path.join(rootDir, cwlfile), os.path.join(rootDir, jobfile)])
        cwltoil.main(main_args, stdout=st)
        out = json.loads(st.getvalue())
        out[out_name].pop("http://commonwl.org/cwltool#generation", None)
        out[out_name].pop("nameext", None)
        out[out_name].pop("nameroot", None)
        self.assertEquals(out, expect)

    def _debug_worker_tester(self, cwlfile, jobfile, outDir, expect):
        from toil.cwl import cwltoil
        rootDir = self._projectRootPath()
        st = StringIO()
        cwltoil.main(['--debugWorker', '--outdir', outDir,
                     os.path.join(rootDir, cwlfile),
                     os.path.join(rootDir, jobfile)], stdout=st)
        out = json.loads(st.getvalue())
        out["output"].pop("http://commonwl.org/cwltool#generation", None)
        out["output"].pop("nameext", None)
        out["output"].pop("nameroot", None)
        self.assertEquals(out, expect)

    def test_run_revsort(self):
        outDir = self._createTempDir()
        self._tester('src/toil/test/cwl/revsort.cwl',
                     'src/toil/test/cwl/revsort-job.json',
                     outDir, {
            # Having unicode string literals isn't necessary for the assertion but makes for a
            # less noisy diff in case the assertion fails.
            u'output': {
                u'location': "file://" + str(os.path.join(outDir, 'output.txt')),
                u'basename': str("output.txt"),
                u'size': 1111,
                u'class': u'File',
                u'checksum': u'sha1$b9214658cc453331b62c2282b772a5c063dbd284'}})

    def test_run_revsort2(self):
        outDir = self._createTempDir()
        self._tester('src/toil/test/cwl/revsort2.cwl',
                     'src/toil/test/cwl/revsort-job.json',
                     outDir, {
            # Having unicode string literals isn't necessary for the assertion but makes for a
            # less noisy diff in case the assertion fails.
            u'output': {
                u'location': "file://" + str(os.path.join(outDir, 'output.txt')),
                u'basename': str("output.txt"),
                u'size': 1111,
                u'class': u'File',
                u'checksum': u'sha1$b9214658cc453331b62c2282b772a5c063dbd284'}})

    def test_run_revsort_debug_worker(self):
        outDir = self._createTempDir()
        # Having unicode string literals isn't necessary for the assertion
        # but makes for a less noisy diff in case the assertion fails.
        self._debug_worker_tester(
                'src/toil/test/cwl/revsort.cwl',
                'src/toil/test/cwl/revsort-job.json', outDir, {u'output': {
                    u'location': "file://" + str(os.path.join(
                        outDir, 'output.txt')),
                    u'basename': str("output.txt"),
                    u'size': 1111,
                    u'class': u'File',
                    u'checksum':
                        u'sha1$b9214658cc453331b62c2282b772a5c063dbd284'}})

    @slow
    def test_restart(self):
        """Enable restarts with CWLtoil -- run failing test, re-run correct test.
        """
        from toil.cwl import cwltoil
        from toil.jobStores.abstractJobStore import NoSuchJobStoreException
        from toil.leader import FailedJobsException
        outDir = self._createTempDir()
        cwlDir = os.path.join(self._projectRootPath(), "src", "toil", "test", "cwl")
        cmd = ['--outdir', outDir, '--jobStore', os.path.join(outDir, 'jobStore'), "--no-container",
               os.path.join(cwlDir, "revsort.cwl"), os.path.join(cwlDir, "revsort-job.json")]

        def path_without_rev():
            return ":".join([d for d in os.environ["PATH"].split(":")
                             if not os.path.exists(os.path.join(d, "rev"))])
        orig_path = os.environ["PATH"]
        # Force a failure and half finished job by removing `rev` from the PATH
        os.environ["PATH"] = path_without_rev()
        try:
            cwltoil.main(cmd)
            self.fail("Expected problem job with incorrect PATH did not fail")
        except FailedJobsException:
            pass
        # Finish the job with a correct PATH
        os.environ["PATH"] = orig_path
        cwltoil.main(["--restart"] + cmd)
        # Should fail because previous job completed successfully
        try:
            cwltoil.main(["--restart"] + cmd)
            self.fail("Restart with missing directory did not fail")
        except NoSuchJobStoreException:
            pass

    @slow
    @pytest.mark.timeout(1800)
    def test_run_conformance(self, batchSystem=None):
        rootDir = self._projectRootPath()
        cwlSpec = os.path.join(rootDir, 'src/toil/test/cwl/spec')
        # The latest cwl git hash. Update it to get the latest tests.
        testhash = "f96bca6911b6688ff614c02dbefe819bed260a13"
        url = "https://github.com/common-workflow-language/common-workflow-language/archive/%s.zip" % testhash
        if not os.path.exists(cwlSpec):
            urlretrieve(url, "spec.zip")
            with zipfile.ZipFile('spec.zip', "r") as z:
                z.extractall()
            shutil.move("common-workflow-language-%s" % testhash, cwlSpec)
            os.remove("spec.zip")
        try:
            cmd = ['cwltest', '--tool', 'toil-cwl-runner', '--test=conformance_test_v1.0.yaml',
                   '--timeout=1800', '--basedir=' + cwlSpec]
            if batchSystem:
                cmd.extend(["--batchSystem", batchSystem])
            subprocess.check_output(cmd, cwd=cwlSpec, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            only_unsupported = False
            # check output -- if we failed but only have unsupported features, we're okay
            p = re.compile(r"(?P<failures>\d+) failures, (?P<unsupported>\d+) unsupported features")
            for line in e.output.split("\n"):
                m = p.search(line)
                if m:
                    if int(m.group("failures")) == 0 and int(m.group("unsupported")) > 0:
                        only_unsupported = True
                        break
            if not only_unsupported:
                print(e.output)
                raise e

    @slow
    def test_bioconda(self):
        outDir = self._createTempDir()
        self._tester('src/toil/test/cwl/seqtk_seq.cwl',
                     'src/toil/test/cwl/seqtk_seq_job.json',
                     outDir,
                     self._expected_seqtk_output(outDir),
                     main_args=["--beta-conda-dependencies"],
                     out_name="output1")

    @needs_docker
    def test_biocontainers(self):
        outDir = self._createTempDir()
        self._tester('src/toil/test/cwl/seqtk_seq.cwl',
                     'src/toil/test/cwl/seqtk_seq_job.json',
                     outDir,
                     self._expected_seqtk_output(outDir),
                     main_args=["--beta-use-biocontainers"],
                     out_name="output1")

    @slow
    @needs_lsf
    @unittest.skip
    def test_lsf_cwl_conformance(self):
        return self.test_run_conformance("LSF")

    @slow
    @needs_slurm
    @unittest.skip
    def test_slurm_cwl_conformance(self):
        return self.test_run_conformance("Slurm")

    @slow
    @needs_torque
    @unittest.skip
    def test_torque_cwl_conformance(self):
        return self.test_run_conformance("Torque")

    @slow
    @needs_gridengine
    @unittest.skip
    def test_gridengine_cwl_conformance(self):
        return self.test_run_conformance("gridEngine")

    @slow
    @needs_mesos
    @unittest.skip
    def test_mesos_cwl_conformance(self):
        return self.test_run_conformance("mesos")

    @slow
    @needs_parasol
    @unittest.skip
    def test_parasol_cwl_conformance(self):
        return self.test_run_conformance("parasol")

    def _expected_seqtk_output(self, outDir):
        return {
            u"output1":  {
                u"location": "file://" + str(os.path.join(outDir, 'out')),
                u"checksum": u"sha1$322e001e5a99f19abdce9f02ad0f02a17b5066c2",
                u"basename": str("out"),
                u"class": u"File",
                u"size": 150}
        }
<|endoftext|>


<commit_before>import logging

_log = logging.getLogger(__name__)

def download_from_s3_url(file_path, url):
    from urlparse import urlparse
    from boto.s3.connection import S3Connection
    s3 = S3Connection()
    try:
        parsed_url = urlparse(url)
        if not parsed_url.netloc or not parsed_url.path.startswith('/'):
            raise RuntimeError("An S3 URL must be of the form s3:/BUCKET/ or "
                               "s3://BUCKET/KEY. '%s' is not." % url)
        bucket = s3.get_bucket(parsed_url.netloc)
        key = bucket.get_key(parsed_url.path[1:])
        _log.info("From URL %s, parsed --> %s, bucket %s, key %s", url, parsed_url, bucket, key)
        key.get_contents_to_filename(file_path)
    finally:
        s3.close()
<commit_msg>Added TODO to remove a function<commit_after>import logging

_log = logging.getLogger(__name__)

# TODO: replace this with toil_scripts.lib.urls._download_s3_url

def download_from_s3_url(file_path, url):
    from urlparse import urlparse
    from boto.s3.connection import S3Connection
    s3 = S3Connection()
    try:
        parsed_url = urlparse(url)
        if not parsed_url.netloc or not parsed_url.path.startswith('/'):
            raise RuntimeError("An S3 URL must be of the form s3:/BUCKET/ or "
                               "s3://BUCKET/KEY. '%s' is not." % url)
        bucket = s3.get_bucket(parsed_url.netloc)
        key = bucket.get_key(parsed_url.path[1:])
        _log.info("From URL %s, parsed --> %s, bucket %s, key %s", url, parsed_url, bucket, key)
        key.get_contents_to_filename(file_path)
    finally:
        s3.close()
<|endoftext|>


<commit_before>from nose.tools import assert_raises

from mediawords.db.schema.version import *

def test_schema_version_from_lines():
    assert_raises(SchemaVersionFromLinesException, schema_version_from_lines, 'no version')

    # noinspection SqlDialectInspection,SqlNoDataSourceInspection
    assert schema_version_from_lines("""
CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$
DECLARE

    -- Database schema version number (same as a SVN revision number)
    -- Increase it by 1 if you make major database schema changes.
    MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4588;

BEGIN

    -- Update / set database schema version
    DELETE FROM database_variables WHERE name = 'database-schema-version';
    INSERT INTO database_variables (name, value) VALUES
        ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int);

    return true;

END;
$$
LANGUAGE 'plpgsql';
    """) == 4588
<commit_msg>Prepend “Mc” to expected exception name<commit_after>from nose.tools import assert_raises

from mediawords.db.schema.version import *

def test_schema_version_from_lines():
    assert_raises(McSchemaVersionFromLinesException, schema_version_from_lines, 'no version')

    # noinspection SqlDialectInspection,SqlNoDataSourceInspection
    assert schema_version_from_lines("""
CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$
DECLARE

    -- Database schema version number (same as a SVN revision number)
    -- Increase it by 1 if you make major database schema changes.
    MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4588;

BEGIN

    -- Update / set database schema version
    DELETE FROM database_variables WHERE name = 'database-schema-version';
    INSERT INTO database_variables (name, value) VALUES
        ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int);

    return true;

END;
$$
LANGUAGE 'plpgsql';
    """) == 4588
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2021 Teklia
# Copyright 2021 Benjamin Kiessling
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
align
~~~~~

A character alignment module using a network output lattice and ground truth to
accuractely determine grapheme locations in input data. Requires OpenFST and
either the official python bindings or pywrapfst.
"""
import logging
import numpy as np

from PIL import Image
from bidi.algorithm import get_display

from kraken import rpred
from kraken.lib.exceptions import KrakenInputException, KrakenEncodeException
from kraken.lib.segmentation import compute_polygon_section

logger = logging.getLogger('kraken')

try:
    import pywrapfst as fst

    _get_fst = lambda: fst.VectorFst() # NOQA
    _get_best_weight = lambda x: fst.Weight.one(fst.weight_type()) # NOQA
except ImportError:
    logger.info('pywrapfst not available. Falling back to openfst_python.')
    try:
        import openfst_python as fst
        _get_fst = lambda: fst.Fst() # NOQA
        _get_best_weight = lambda x: 0 # NOQA
    except ImportError:
        logger.error('Neither pywrapfst nor openfst_python bindings available.')
        raise

def _get_arc(input_label, output_label, weight, state):
    return fst.Arc(input_label, output_label, weight, state)

def _compose(first_graph, second_graph):
    return fst.compose(first_graph, second_graph)

def _shortest_path(graph):
    return fst.shortestpath(graph)

def fst_from_lattice(outputs: np.ndarray):
    """
    Generates an FST from the network output tensor.

    Arguments:
        outputs (np.ndarray): Output tensor of shape (Batch, Labels, Length).

    Returns:
        An OpenFST FST object.
    """
    f = _get_fst()
    state = f.add_state()
    f.set_start(state)
    for idx, frame in enumerate(np.transpose(outputs[0], [1, 0])):
        next_state = f.add_state()
        for label, conf in enumerate(frame):
            f.add_arc(state, _get_arc(idx, label+1, -np.log(conf), next_state))
        state = next_state

    f.set_final(state)

    if not f.verify():
        raise KrakenInputException('Invalid FST')

    return f

def fst_from_text(line, codec):
    """
    Creates an FST of a text line in label space.

    In order to match the Kraken output, we take into account the fact that
    Kraken will output a large number of frames and therefore each character
    of our line must be able to be detected between several non-characters "#".

    In the same way, Kraken can predict several times the same character on
    neighboring frames, if this is the case it doesn't mean that there is
    several times the same letter but only that it appears on several frames.

    To handle this particular case we use Ɛ:Ɛ transitions between each character
    of our line and we add a Ɛ:mychar loop to eliminate identical neighboring
    characters.

    To detect the word "loop", we'll have something like:

       Ɛ:Ɛ    l:l    Ɛ:Ɛ    o:o    Ɛ:Ɛ    o:o    Ɛ:Ɛ    p:p
    [0]───>(1)───>(2)───>(3)───>(4)───>(5)───>(6)───>(7)───>((8))
     ↺      ↺      ↺      ↺      ↺      ↺      ↺      ↺       ↺
    #:Ɛ    l:Ɛ    #:Ɛ    o:Ɛ    #:Ɛ    o:Ɛ    #:Ɛ    p:Ɛ     #:Ɛ
    """
    f = _get_fst()
    one = _get_best_weight(f)
    state = f.add_state()
    f.set_start(state)

    def _detect_char(graph, one, start_state, char, codec):
        # Empty char loop
        graph.add_arc(start_state, _get_arc(1, 0, one, start_state))
        epsilon_state = graph.add_state()
        # Epsilon transition
        graph.add_arc(start_state, _get_arc(0, 0, one, epsilon_state))
        # Adding one to expand neural network codec with epsilon at pos 0
        # We will end with the empty char # at pos 1 (see: kraken.lib.codec.py)
        try:
            encoded_char = codec.encode(char)[0] + 1
        except KrakenEncodeException:
            encoded_char = 0
            logger.warning(f'Error while encoding code point {char}. Skipping.')
        # Espilon loop
        graph.add_arc(epsilon_state, _get_arc(encoded_char, 0, one, epsilon_state))
        end_state = graph.add_state()
        # Char transition
        graph.add_arc(epsilon_state, _get_arc(encoded_char, encoded_char, one, end_state))
        return end_state

    for char in line:
        state = _detect_char(f, one, state, char, codec)

    f.add_arc(state, _get_arc(1, 0, one, state))
    f.set_final(state)

    if not f.verify():
        raise KrakenInputException('Invalid FST')

    return f

def _generate_line_record(graph, final_step):
    next_state = graph.start()
    labels = []
    activations = [0]

    while next_state:
        state = next_state
        next_state = None
        for arc in graph.arcs(state):
            next_state = arc.nextstate
            olabel = arc.olabel - 1
            if olabel == -1:
                continue
            labels.append(olabel)
            activations.append(arc.ilabel)
    activations.append(final_step)
    # spread activations to halfway point between activations
    bounds = []
    for prev_ac, cur_ac, next_ac in zip(activations[:-2], activations[1:-1], activations[2:]):
        bounds.append((prev_ac + (cur_ac-prev_ac)//2, cur_ac + (next_ac-cur_ac)//2))
    return zip(bounds, labels)

def forced_align(doc, model):
    """
    Performs a forced character alignment of text with recognition model
    output activations.

    Argument:
        doc (dict): Parsed document.
        model (kraken.lib.model.TorchSeqRecognizer): Recognition model to use for alignment.

    Returns:
        A list of kraken.rpred.ocr_record.
    """
    im = Image.open(doc['image'])
    predictor = rpred.rpred(model, im, doc)
    records = []
    for line in doc['lines']:
        bidi_text = get_display(line['text'])
        gt_fst = fst_from_text(bidi_text, model.codec)
        next(predictor)
        lat_fst = fst_from_lattice(model.outputs)
        composed_graph = _compose(lat_fst, gt_fst)
        short_graph = _shortest_path(composed_graph)
        pred = []
        pos = []
        conf = []
        for act, label in _generate_line_record(short_graph, model.outputs.shape[2]-1):
            pos.append(compute_polygon_section(line['baseline'],
                                               line['boundary'],
                                               predictor._scale_val(act[0], 0, predictor.box.size[0]),
                                               predictor._scale_val(act[1], 0, predictor.box.size[0])))
            conf.append(1.0)
            pred.append(model.codec.decode([(label, 0, 0, 0)])[0][0])
        records.append(rpred.bidi_record(rpred.ocr_record(pred, pos, conf, line)))
    return records
<commit_msg>fix regression in alignment code with non-strict codecs<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2021 Teklia
# Copyright 2021 Benjamin Kiessling
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
align
~~~~~

A character alignment module using a network output lattice and ground truth to
accuractely determine grapheme locations in input data. Requires OpenFST and
either the official python bindings or pywrapfst.
"""
import logging
import numpy as np

from PIL import Image
from bidi.algorithm import get_display

from kraken import rpred
from kraken.lib.exceptions import KrakenInputException, KrakenEncodeException
from kraken.lib.segmentation import compute_polygon_section

logger = logging.getLogger('kraken')

try:
    import pywrapfst as fst

    _get_fst = lambda: fst.VectorFst() # NOQA
    _get_best_weight = lambda x: fst.Weight.one(fst.weight_type()) # NOQA
except ImportError:
    logger.info('pywrapfst not available. Falling back to openfst_python.')
    try:
        import openfst_python as fst
        _get_fst = lambda: fst.Fst() # NOQA
        _get_best_weight = lambda x: 0 # NOQA
    except ImportError:
        logger.error('Neither pywrapfst nor openfst_python bindings available.')
        raise

def _get_arc(input_label, output_label, weight, state):
    return fst.Arc(input_label, output_label, weight, state)

def _compose(first_graph, second_graph):
    return fst.compose(first_graph, second_graph)

def _shortest_path(graph):
    return fst.shortestpath(graph)

def fst_from_lattice(outputs: np.ndarray):
    """
    Generates an FST from the network output tensor.

    Arguments:
        outputs (np.ndarray): Output tensor of shape (Batch, Labels, Length).

    Returns:
        An OpenFST FST object.
    """
    f = _get_fst()
    state = f.add_state()
    f.set_start(state)
    for idx, frame in enumerate(np.transpose(outputs[0], [1, 0])):
        next_state = f.add_state()
        for label, conf in enumerate(frame):
            f.add_arc(state, _get_arc(idx, label+1, -np.log(conf), next_state))
        state = next_state

    f.set_final(state)

    if not f.verify():
        raise KrakenInputException('Invalid FST')

    return f

def fst_from_text(line, codec):
    """
    Creates an FST of a text line in label space.

    In order to match the Kraken output, we take into account the fact that
    Kraken will output a large number of frames and therefore each character
    of our line must be able to be detected between several non-characters "#".

    In the same way, Kraken can predict several times the same character on
    neighboring frames, if this is the case it doesn't mean that there is
    several times the same letter but only that it appears on several frames.

    To handle this particular case we use Ɛ:Ɛ transitions between each character
    of our line and we add a Ɛ:mychar loop to eliminate identical neighboring
    characters.

    To detect the word "loop", we'll have something like:

       Ɛ:Ɛ    l:l    Ɛ:Ɛ    o:o    Ɛ:Ɛ    o:o    Ɛ:Ɛ    p:p
    [0]───>(1)───>(2)───>(3)───>(4)───>(5)───>(6)───>(7)───>((8))
     ↺      ↺      ↺      ↺      ↺      ↺      ↺      ↺       ↺
    #:Ɛ    l:Ɛ    #:Ɛ    o:Ɛ    #:Ɛ    o:Ɛ    #:Ɛ    p:Ɛ     #:Ɛ
    """
    f = _get_fst()
    one = _get_best_weight(f)
    state = f.add_state()
    f.set_start(state)

    codec.strict = True

    def _detect_char(graph, one, start_state, char, codec):
        # Empty char loop
        graph.add_arc(start_state, _get_arc(1, 0, one, start_state))
        epsilon_state = graph.add_state()
        # Epsilon transition
        graph.add_arc(start_state, _get_arc(0, 0, one, epsilon_state))
        # Adding one to expand neural network codec with epsilon at pos 0
        # We will end with the empty char # at pos 1 (see: kraken.lib.codec.py)
        try:
            encoded_char = codec.encode(char)[0] + 1
        except KrakenEncodeException:
            encoded_char = 0
            logger.warning(f'Error while encoding code point {char}. Skipping.')
        # Espilon loop
        graph.add_arc(epsilon_state, _get_arc(encoded_char, 0, one, epsilon_state))
        end_state = graph.add_state()
        # Char transition
        graph.add_arc(epsilon_state, _get_arc(encoded_char, encoded_char, one, end_state))
        return end_state

    for char in line:
        state = _detect_char(f, one, state, char, codec)

    f.add_arc(state, _get_arc(1, 0, one, state))
    f.set_final(state)

    if not f.verify():
        raise KrakenInputException('Invalid FST')

    return f

def _generate_line_record(graph, final_step):
    next_state = graph.start()
    labels = []
    activations = [0]

    while next_state:
        state = next_state
        next_state = None
        for arc in graph.arcs(state):
            next_state = arc.nextstate
            olabel = arc.olabel - 1
            if olabel == -1:
                continue
            labels.append(olabel)
            activations.append(arc.ilabel)
    activations.append(final_step)
    # spread activations to halfway point between activations
    bounds = []
    for prev_ac, cur_ac, next_ac in zip(activations[:-2], activations[1:-1], activations[2:]):
        bounds.append((prev_ac + (cur_ac-prev_ac)//2, cur_ac + (next_ac-cur_ac)//2))
    return zip(bounds, labels)

def forced_align(doc, model):
    """
    Performs a forced character alignment of text with recognition model
    output activations.

    Argument:
        doc (dict): Parsed document.
        model (kraken.lib.model.TorchSeqRecognizer): Recognition model to use for alignment.

    Returns:
        A list of kraken.rpred.ocr_record.
    """
    im = Image.open(doc['image'])
    predictor = rpred.rpred(model, im, doc)
    records = []
    for line in doc['lines']:
        bidi_text = get_display(line['text'])
        gt_fst = fst_from_text(bidi_text, model.codec)
        next(predictor)
        lat_fst = fst_from_lattice(model.outputs)
        composed_graph = _compose(lat_fst, gt_fst)
        short_graph = _shortest_path(composed_graph)
        pred = []
        pos = []
        conf = []
        for act, label in _generate_line_record(short_graph, model.outputs.shape[2]-1):
            pos.append(compute_polygon_section(line['baseline'],
                                               line['boundary'],
                                               predictor._scale_val(act[0], 0, predictor.box.size[0]),
                                               predictor._scale_val(act[1], 0, predictor.box.size[0])))
            conf.append(1.0)
            pred.append(model.codec.decode([(label, 0, 0, 0)])[0][0])
        records.append(rpred.bidi_record(rpred.ocr_record(pred, pos, conf, line)))
    return records
<|endoftext|>