<commit_before># Copyright (c) 2019 Daniel Campos <danielcampos@avanzosc.es> - Avanzosc S.L.
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).

from odoo.tests import common
from odoo import fields
from odoo.exceptions import ValidationError

class TestSupplierinfoUpdate(common.SavepointCase):

    @classmethod
    def setUpClass(cls):
        super(TestSupplierinfoUpdate, cls).setUpClass()
        cls.product_model = cls.env['product.product']
        cls.partner_model = cls.env['res.partner']
        cls.supplierinfo_model = cls.env['product.supplierinfo']
        cls.uom_unit = cls.env.ref('uom.product_uom_unit')
        cls.purchase_model = cls.env['purchase.order']
        cls.supplier = cls.partner_model.create({
            'name': 'Supplier1',
            'supplier': 'True',
            'date_start': fields.Date.today(),
        })
        cls.seller = cls.supplierinfo_model.create({
            'name': cls.supplier.id,
            'price': 12.0,
        })
        cls.seller_2 = cls.supplierinfo_model.create({
            'name': cls.supplier.id,
            'price': 10.0,
            'min_qty': 50,
        })
        cls.product = cls.product_model.create({
            'name': 'Product',
            'type': 'product',
            'seller_ids': [(6, 0, [cls.seller.id])],
            'default_code': 'P1',
            'uom_id': cls.uom_unit.id,
            'uom_po_id': cls.uom_unit.id
        })
        cls.product_2 = cls.product_model.create({
            'name': 'Product_2',
            'type': 'product',
            'default_code': 'P2',
            'seller_ids': [(6, 0, [cls.seller_2.id])],
            'uom_id': cls.uom_unit.id,
            'uom_po_id': cls.uom_unit.id
        })

    def test_supplierinfo_price_update(self):
        purchase = self.purchase_model.create({
            'partner_id': self.supplier.id,
            'order_line': [
                (0, 0, {
                    'name': self.product.name,
                    'product_id': self.product.id,
                    'product_qty': 10.0,
                    'product_uom': self.product.uom_po_id.id,
                    'price_unit': self.product.standard_price,
                    'date_planned': fields.Date.today(),
                })],
        })
        order_line = purchase.order_line[:1]
        self.assertNotEquals(
            self.product.standard_price, self.seller.price,
            "Product's cost price must be different from seller's price")
        self.assertNotEquals(
            order_line.price_unit, self.seller.price,
            "Purchase line unit price must be different from seller's price")
        order_line._onchange_quantity()
        self.assertEquals(
            order_line.price_unit, self.seller.price,
            "The product's price on the purchase line is not correct.")
        order_line.price_unit = 10.0
        sellers_price = self.seller.price
        self.assertNotEquals(
            order_line.price_unit, sellers_price,
            "Purchase line unit price must be different from seller's price")
        purchase.button_confirm()
        self.assertNotEquals(
            self.seller.price, sellers_price,
            "Seller's price hasn't been updated")
        self.assertEqual(order_line.price_unit, self.seller.price,
                         'The price on the purchase order line and seller '
                         'price is not the same.')

    def test_seller_price_error(self):
        purchase = self.purchase_model.create({
            'partner_id': self.supplier.id,
            'order_line': [
                (0, 0, {
                    'name': self.product_2.name,
                    'product_id': self.product_2.id,
                    'product_qty': 10.0,
                    'product_uom': self.product_2.uom_po_id.id,
                    'price_unit': self.product_2.standard_price,
                    'date_planned': fields.Date.today(),
                })],
        })
        with self.assertRaises(ValidationError):
            purchase.button_confirm()
<commit_msg>[FIX] product_supplierinfo_purchase_price: remove warnings from tests (travis) (#1081)<commit_after># Copyright (c) 2019 Daniel Campos <danielcampos@avanzosc.es> - Avanzosc S.L.
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).

from odoo.tests import common
from odoo import fields
from odoo.exceptions import ValidationError

@common.at_install(False)
@common.post_install(True)
class TestSupplierinfoUpdate(common.SavepointCase):

    @classmethod
    def setUpClass(cls):
        super(TestSupplierinfoUpdate, cls).setUpClass()
        cls.product_model = cls.env['product.product']
        cls.partner_model = cls.env['res.partner']
        cls.supplierinfo_model = cls.env['product.supplierinfo']
        cls.uom_unit = cls.env.ref('uom.product_uom_unit')
        cls.purchase_model = cls.env['purchase.order']
        cls.supplier = cls.partner_model.create({
            'name': 'Supplier1',
            'supplier': 'True',
        })
        cls.seller = cls.supplierinfo_model.create({
            'name': cls.supplier.id,
            'price': 12.0,
        })
        cls.seller_2 = cls.supplierinfo_model.create({
            'name': cls.supplier.id,
            'price': 10.0,
            'min_qty': 50,
        })
        cls.product = cls.product_model.create({
            'name': 'Product',
            'type': 'product',
            'seller_ids': [(6, 0, [cls.seller.id])],
            'default_code': 'P1',
            'uom_id': cls.uom_unit.id,
            'uom_po_id': cls.uom_unit.id
        })
        cls.product_2 = cls.product_model.create({
            'name': 'Product_2',
            'type': 'product',
            'default_code': 'P2',
            'seller_ids': [(6, 0, [cls.seller_2.id])],
            'uom_id': cls.uom_unit.id,
            'uom_po_id': cls.uom_unit.id
        })

    def test_supplierinfo_price_update(self):
        purchase = self.purchase_model.create({
            'partner_id': self.supplier.id,
            'order_line': [
                (0, 0, {
                    'name': self.product.name,
                    'product_id': self.product.id,
                    'product_qty': 10.0,
                    'product_uom': self.product.uom_po_id.id,
                    'price_unit': self.product.standard_price,
                    'date_planned': fields.Date.today(),
                })],
        })
        order_line = purchase.order_line[:1]
        self.assertNotEquals(
            self.product.standard_price, self.seller.price,
            "Product's cost price must be different from seller's price")
        self.assertNotEquals(
            order_line.price_unit, self.seller.price,
            "Purchase line unit price must be different from seller's price")
        order_line._onchange_quantity()
        self.assertEquals(
            order_line.price_unit, self.seller.price,
            "The product's price on the purchase line is not correct.")
        order_line.price_unit = 10.0
        sellers_price = self.seller.price
        self.assertNotEquals(
            order_line.price_unit, sellers_price,
            "Purchase line unit price must be different from seller's price")
        purchase.button_confirm()
        self.assertNotEquals(
            self.seller.price, sellers_price,
            "Seller's price hasn't been updated")
        self.assertEqual(order_line.price_unit, self.seller.price,
                         'The price on the purchase order line and seller '
                         'price is not the same.')

    def test_seller_price_error(self):
        purchase = self.purchase_model.create({
            'partner_id': self.supplier.id,
            'order_line': [
                (0, 0, {
                    'name': self.product_2.name,
                    'product_id': self.product_2.id,
                    'product_qty': 10.0,
                    'product_uom': self.product_2.uom_po_id.id,
                    'price_unit': self.product_2.standard_price,
                    'date_planned': fields.Date.today(),
                })],
        })
        with self.assertRaises(ValidationError):
            purchase.button_confirm()
<|endoftext|>


<commit_before>import os
import re
import zmq

context = zmq.Context()

# from integer socket_type to string
SOCKTYPE = {zmq.REQ: 'REQ', zmq.REP: 'REP',
            zmq.PUSH: 'PUSH', zmq.PULL: 'PULL',
            zmq.ROUTER: 'ROUTER', zmq.DEALER: 'DEALER'}

def bind(end_point, socket_type):
    """
    Bind to a zmq URL; raise a proper error if the URL is invalid; return
    a zmq socket.
    """
    sock = context.socket(socket_type)
    try:
        sock.bind(end_point)
    except zmq.error.ZMQError as exc:
        sock.close()
        raise exc.__class__('%s: %s' % (exc, end_point))
    return sock

def connect(end_point, socket_type):
    """
    Connect to a zmq URL; raise a proper error if the URL is invalid; return
    a zmq socket.
    """
    sock = context.socket(socket_type)
    try:
        sock.connect(end_point)
    except zmq.error.ZMQError as exc:
        sock.close()
        raise exc.__class__('%s: %s' % (exc, end_point))
    return sock

class Socket(object):
    """
    A Socket class to be used with code like the following::

     # server
     sock = Socket('tcp://127.0.0.1:9000', zmq.REP)
     for cmd, *args in sock:
         sock.send(cmd(*args))

     # client
     Socket('tcp://127.0.0.1:9000', zmq.REQ).req(cmd, *args)

    It also support zmq.PULL/zmq.PUSH sockets, which are asynchronous.

    :param end_point: zmq end point string
    :param socket_type: zmq socket type (integer)
    :param mode: default 'bind', accepts also 'connect'
    :param timeout: default 1000 ms, used when polling the underlying socket
    """
    def __init__(self, end_point, socket_type, mode='bind', timeout=1000):
        assert socket_type in (zmq.REP, zmq.REQ, zmq.PULL, zmq.PUSH)
        assert mode in ('bind', 'connect'), mode
        self.end_point = end_point
        self.socket_type = socket_type
        self.mode = mode
        self.timeout = timeout
        self.running = False

    def __enter__(self):
        """Instantiate the undelying zmq socket"""
        # first check if the end_point ends in :<min_port>-<max_port>
        port_range = re.search(r':(\d+)-(\d+)$', self.end_point)
        if port_range:
            assert self.mode == 'bind', self.mode
            p1, p2 = map(int, port_range.groups())
            end_point = self.end_point.rsplit(':', 1)[0]  # strip port range
            self.zsocket = context.socket(self.socket_type)
            port = self.zsocket.bind_to_random_port(end_point, p1, p2)
            self.true_end_point = '%s:%d' % (end_point, port)
        elif self.mode == 'bind':
            self.zsocket = bind(self.end_point, self.socket_type)
        else:  # connect
            self.zsocket = connect(self.end_point, self.socket_type)
        self.zsocket.__enter__()
        return self

    def __exit__(self, *args):
        self.zsocket.__exit__(*args)
        del self.zsocket

    def __iter__(self):
        """
        Iterate on the socket and yield the received arguments. Exits if

        1. the flag .running is set to False
        2. the message 'stop' is sent
        3. SIGINT is sent
        4. SIGTERM is sent
        """
        # works zmq.REP and zmq.PULL sockets
        with self:
            self.running = True
            while self.running:
                try:
                    if self.zsocket.poll(self.timeout):
                        args = self.zsocket.recv_pyobj()
                    else:
                        continue
                except (KeyboardInterrupt, zmq.error.ZMQError):
                    # sending SIGTERM raises ZMQError
                    break
                if args[0] == 'stop':
                    if self.socket_type == zmq.REP:
                        self.send((None, None, None))
                    break
                else:
                    yield args

    def send(self, obj, noblock=0):
        """
        Send an object to the remote server; block and return the reply
        if the socket type is REQ.

        :param obj:
            the Python object to send
        :param noblock:
            if the socket is of type REQ and the reply is not ready, raise a
            ZMQError or wait, depending on this flag
        """
        self.zsocket.send_pyobj(obj)
        if self.socket_type == zmq.REQ:
            return self.zsocket.recv_pyobj(noblock)

if __name__ == '__main__':
    print('started echo server, pid=%d' % os.getpid())
    sock = Socket('tcp://127.0.0.1:9000', zmq.REP)
    for args in sock:  # server for testing purposes
        sock.send(args)
<commit_msg>Added an useful Socket.__repr__<commit_after>import os
import re
import zmq

context = zmq.Context()

# from integer socket_type to string
SOCKTYPE = {zmq.REQ: 'REQ', zmq.REP: 'REP',
            zmq.PUSH: 'PUSH', zmq.PULL: 'PULL',
            zmq.ROUTER: 'ROUTER', zmq.DEALER: 'DEALER'}

def bind(end_point, socket_type):
    """
    Bind to a zmq URL; raise a proper error if the URL is invalid; return
    a zmq socket.
    """
    sock = context.socket(socket_type)
    try:
        sock.bind(end_point)
    except zmq.error.ZMQError as exc:
        sock.close()
        raise exc.__class__('%s: %s' % (exc, end_point))
    return sock

def connect(end_point, socket_type):
    """
    Connect to a zmq URL; raise a proper error if the URL is invalid; return
    a zmq socket.
    """
    sock = context.socket(socket_type)
    try:
        sock.connect(end_point)
    except zmq.error.ZMQError as exc:
        sock.close()
        raise exc.__class__('%s: %s' % (exc, end_point))
    return sock

class Socket(object):
    """
    A Socket class to be used with code like the following::

     # server
     sock = Socket('tcp://127.0.0.1:9000', zmq.REP)
     for cmd, *args in sock:
         sock.send(cmd(*args))

     # client
     Socket('tcp://127.0.0.1:9000', zmq.REQ).req(cmd, *args)

    It also support zmq.PULL/zmq.PUSH sockets, which are asynchronous.

    :param end_point: zmq end point string
    :param socket_type: zmq socket type (integer)
    :param mode: default 'bind', accepts also 'connect'
    :param timeout: default 1000 ms, used when polling the underlying socket
    """
    def __init__(self, end_point, socket_type, mode='bind', timeout=1000):
        assert socket_type in (zmq.REP, zmq.REQ, zmq.PULL, zmq.PUSH)
        assert mode in ('bind', 'connect'), mode
        self.end_point = end_point
        self.socket_type = socket_type
        self.mode = mode
        self.timeout = timeout
        self.running = False

    def __enter__(self):
        """Instantiate the undelying zmq socket"""
        # first check if the end_point ends in :<min_port>-<max_port>
        port_range = re.search(r':(\d+)-(\d+)$', self.end_point)
        if port_range:
            assert self.mode == 'bind', self.mode
            p1, p2 = map(int, port_range.groups())
            end_point = self.end_point.rsplit(':', 1)[0]  # strip port range
            self.zsocket = context.socket(self.socket_type)
            port = self.zsocket.bind_to_random_port(end_point, p1, p2)
            self.true_end_point = '%s:%d' % (end_point, port)
        elif self.mode == 'bind':
            self.zsocket = bind(self.end_point, self.socket_type)
        else:  # connect
            self.zsocket = connect(self.end_point, self.socket_type)
        self.zsocket.__enter__()
        return self

    def __exit__(self, *args):
        self.zsocket.__exit__(*args)
        del self.zsocket

    def __iter__(self):
        """
        Iterate on the socket and yield the received arguments. Exits if

        1. the flag .running is set to False
        2. the message 'stop' is sent
        3. SIGINT is sent
        4. SIGTERM is sent
        """
        # works zmq.REP and zmq.PULL sockets
        with self:
            self.running = True
            while self.running:
                try:
                    if self.zsocket.poll(self.timeout):
                        args = self.zsocket.recv_pyobj()
                    else:
                        continue
                except (KeyboardInterrupt, zmq.error.ZMQError):
                    # sending SIGTERM raises ZMQError
                    break
                if args[0] == 'stop':
                    if self.socket_type == zmq.REP:
                        self.send((None, None, None))
                    break
                else:
                    yield args

    def send(self, obj, noblock=0):
        """
        Send an object to the remote server; block and return the reply
        if the socket type is REQ.

        :param obj:
            the Python object to send
        :param noblock:
            if the socket is of type REQ and the reply is not ready, raise a
            ZMQError or wait, depending on this flag
        """
        self.zsocket.send_pyobj(obj)
        if self.socket_type == zmq.REQ:
            return self.zsocket.recv_pyobj(noblock)

    def __repr__(self):
        st = SOCKTYPE[self.socket_type]
        return '<%s %s %s>' % (self.__class__.__name__, st, self.mode)

if __name__ == '__main__':
    print('started echo server, pid=%d' % os.getpid())
    sock = Socket('tcp://127.0.0.1:9000', zmq.REP)
    for args in sock:  # server for testing purposes
        sock.send(args)
<|endoftext|>


<commit_before>from django.conf import settings
from hashlib import sha1
from mediagenerator.generators.bundles.base import Filter
from mediagenerator.utils import get_media_dirs, find_file
from subprocess import Popen, PIPE
import os
import re
import sys

# Emits extra debug info that can be used by the FireSass Firebug plugin
SASS_DEBUG_INFO = getattr(settings, 'SASS_DEBUG_INFO', False)

_RE_FLAGS = re.MULTILINE | re.UNICODE
multi_line_comment_re = re.compile(r'/\*.*?\*/', _RE_FLAGS | re.DOTALL)
one_line_comment_re = re.compile(r'//.*', _RE_FLAGS)
import_re = re.compile(r'^@import\s+["\']?(.+?)["\']?\s*;?\s*$', _RE_FLAGS)

class Sass(Filter):
    takes_input = False

    def __init__(self, **kwargs):
        self.config(kwargs, path=(), main_module=None)
        if isinstance(self.path, basestring):
            self.path = (self.path,)
        super(Sass, self).__init__(**kwargs)
        assert self.filetype == 'css', (
            'Sass only supports compilation to css. '
            'The parent filter expects "%s".' % self.filetype)
        assert self.main_module, \
            'You must provide a main module'

        self.path += tuple(get_media_dirs())
        self.path_args = []
        for path in self.path:
            self.path_args.extend(('-I', path))

        self._compiled = None
        self._compiled_hash = None
        self._dependencies = {}

    @classmethod
    def from_default(cls, name):
        return {'main_module': name}

    def get_output(self, variation):
        self._regenerate(debug=False)
        yield self._compiled

    def get_dev_output(self, name, variation):
        assert name == self.main_module
        self._regenerate(debug=True)
        return self._compiled

    def get_dev_output_names(self, variation):
        self._regenerate(debug=True)
        yield self.main_module, self._compiled_hash

    def _compile(self, debug=False):
        run = ['sass', '-C', '-t', 'expanded']
        if debug:
            run.append('--line-numbers')
            if SASS_DEBUG_INFO:
                run.append('--debug-info')
        run.extend(self.path_args)
        shell = sys.platform == 'win32'
        cmd = Popen(run, shell=shell, universal_newlines=True,
                    stdin=PIPE, stdout=PIPE, stderr=PIPE)
        output, error = cmd.communicate('@import %s' % self.main_module)
        assert cmd.wait() == 0, 'Command returned bad result:\n%s' % error
        return output

    def _regenerate(self, debug=False):
        if self._dependencies:
            for name, mtime in self._dependencies.items():
                path = self._find_file(name)
                if not path or os.path.getmtime(path) != mtime:
                    # Just recompile everything
                    self._dependencies = {}
                    break
            else:
                # No changes
                return

        modules = [self.main_module]
        while True:
            if not modules:
                break

            module_name = modules.pop()
            path = self._find_file(module_name)
            assert path, 'Could not find the Sass module %s' % module_name
            mtime = os.path.getmtime(path)
            self._dependencies[module_name] = mtime

            fp = open(path, 'r')
            source = fp.read()
            fp.close()

            dependencies = self._get_dependencies(source)

            for name in dependencies:
                path = self._find_file(name)
                assert path, ('The Sass module %s could not find the '
                              'dependency %s' % (module_name, name))
                if name not in self._dependencies:
                    modules.append(name)

        self._compiled = self._compile(debug=debug)
        self._compiled_hash = sha1(self._compiled).hexdigest()

    def _get_dependencies(self, source):
        clean_source = multi_line_comment_re.sub('\n', source)
        clean_source = one_line_comment_re.sub('', clean_source)
        return [name for name in import_re.findall(clean_source)
                if not name.endswith('.css')]

    def _find_file(self, name):
        parts = name.rsplit('/', 1)
        parts[-1] = '_' + parts[-1]
        partial = '/'.join(parts)
        if not name.endswith(('.sass', '.scss')):
            names = (name + '.sass', name + '.scss', partial + '.sass',
                     partial + '.scss')
        else:
            names = (name, partial)
        for name in names:
            path = find_file(name, media_dirs=self.path)
            if path:
                return path
<commit_msg>error message slightly improved<commit_after>from django.conf import settings
from hashlib import sha1
from mediagenerator.generators.bundles.base import Filter
from mediagenerator.utils import get_media_dirs, find_file
from subprocess import Popen, PIPE
import os
import re
import sys

# Emits extra debug info that can be used by the FireSass Firebug plugin
SASS_DEBUG_INFO = getattr(settings, 'SASS_DEBUG_INFO', False)

_RE_FLAGS = re.MULTILINE | re.UNICODE
multi_line_comment_re = re.compile(r'/\*.*?\*/', _RE_FLAGS | re.DOTALL)
one_line_comment_re = re.compile(r'//.*', _RE_FLAGS)
import_re = re.compile(r'^@import\s+["\']?(.+?)["\']?\s*;?\s*$', _RE_FLAGS)

class Sass(Filter):
    takes_input = False

    def __init__(self, **kwargs):
        self.config(kwargs, path=(), main_module=None)
        if isinstance(self.path, basestring):
            self.path = (self.path,)
        super(Sass, self).__init__(**kwargs)
        assert self.filetype == 'css', (
            'Sass only supports compilation to css. '
            'The parent filter expects "%s".' % self.filetype)
        assert self.main_module, \
            'You must provide a main module'

        self.path += tuple(get_media_dirs())
        self.path_args = []
        for path in self.path:
            self.path_args.extend(('-I', path))

        self._compiled = None
        self._compiled_hash = None
        self._dependencies = {}

    @classmethod
    def from_default(cls, name):
        return {'main_module': name}

    def get_output(self, variation):
        self._regenerate(debug=False)
        yield self._compiled

    def get_dev_output(self, name, variation):
        assert name == self.main_module
        self._regenerate(debug=True)
        return self._compiled

    def get_dev_output_names(self, variation):
        self._regenerate(debug=True)
        yield self.main_module, self._compiled_hash

    def _compile(self, debug=False):
        run = ['sass', '-C', '-t', 'expanded']
        if debug:
            run.append('--line-numbers')
            if SASS_DEBUG_INFO:
                run.append('--debug-info')
        run.extend(self.path_args)
        shell = sys.platform == 'win32'
        cmd = Popen(run, shell=shell, universal_newlines=True,
                    stdin=PIPE, stdout=PIPE, stderr=PIPE)
        output, error = cmd.communicate('@import %s' % self.main_module)
        assert cmd.wait() == 0, 'Sass command returned bad result:\n%s' % error
        return output

    def _regenerate(self, debug=False):
        if self._dependencies:
            for name, mtime in self._dependencies.items():
                path = self._find_file(name)
                if not path or os.path.getmtime(path) != mtime:
                    # Just recompile everything
                    self._dependencies = {}
                    break
            else:
                # No changes
                return

        modules = [self.main_module]
        while True:
            if not modules:
                break

            module_name = modules.pop()
            path = self._find_file(module_name)
            assert path, 'Could not find the Sass module %s' % module_name
            mtime = os.path.getmtime(path)
            self._dependencies[module_name] = mtime

            fp = open(path, 'r')
            source = fp.read()
            fp.close()

            dependencies = self._get_dependencies(source)

            for name in dependencies:
                path = self._find_file(name)
                assert path, ('The Sass module %s could not find the '
                              'dependency %s' % (module_name, name))
                if name not in self._dependencies:
                    modules.append(name)

        self._compiled = self._compile(debug=debug)
        self._compiled_hash = sha1(self._compiled).hexdigest()

    def _get_dependencies(self, source):
        clean_source = multi_line_comment_re.sub('\n', source)
        clean_source = one_line_comment_re.sub('', clean_source)
        return [name for name in import_re.findall(clean_source)
                if not name.endswith('.css')]

    def _find_file(self, name):
        parts = name.rsplit('/', 1)
        parts[-1] = '_' + parts[-1]
        partial = '/'.join(parts)
        if not name.endswith(('.sass', '.scss')):
            names = (name + '.sass', name + '.scss', partial + '.sass',
                     partial + '.scss')
        else:
            names = (name, partial)
        for name in names:
            path = find_file(name, media_dirs=self.path)
            if path:
                return path
<|endoftext|>


<commit_before>"""Run importlib's test suite.

Specifying the ``--builtin`` flag will run tests, where applicable, with
builtins.__import__ instead of importlib.__import__.

"""
import argparse
from importlib.test.import_ import util
import os.path
from test.support import run_unittest
import unittest

def test_main():
    parser = argparse.ArgumentParser(description='Execute the importlib test '
                                                  'suite')
    parser.add_argument('-b', '--builtin', action='store_true', default=False,
                        help='use builtins.__import__() instead of importlib')
    args = parser.parse_args()
    if args.builtin:
        util.using___import__ = True
    start_dir = os.path.dirname(__file__)
    top_dir = os.path.dirname(os.path.dirname(start_dir))
    test_loader = unittest.TestLoader()
    run_unittest(test_loader.discover(start_dir, top_level_dir=top_dir))

if __name__ == '__main__':
    test_main()
<commit_msg>Fix importlib.test.__main__ to only worry about command-line flags when directly executed.<commit_after>"""Run importlib's test suite.

Specifying the ``--builtin`` flag will run tests, where applicable, with
builtins.__import__ instead of importlib.__import__.

"""
from importlib.test.import_ import util
import os.path
from test.support import run_unittest
import unittest

def test_main():
    start_dir = os.path.dirname(__file__)
    top_dir = os.path.dirname(os.path.dirname(start_dir))
    test_loader = unittest.TestLoader()
    run_unittest(test_loader.discover(start_dir, top_level_dir=top_dir))

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser(description='Execute the importlib test '
                                                  'suite')
    parser.add_argument('-b', '--builtin', action='store_true', default=False,
                        help='use builtins.__import__() instead of importlib')
    args = parser.parse_args()
    if args.builtin:
        util.using___import__ = True
    test_main()
<|endoftext|>


<commit_before>import bencode
import hashlib
import json
from datetime import datetime
from itertools import chain

from django.core.cache import cache
from django.conf import settings
from django.core.paginator import Paginator
from django.db.models import Q

from util import convert_to_utc, convert_to_dict
from ..models import Statement, Agent
from ..exceptions import NotFound, IDNotFoundError

MORE_ENDPOINT = '/xapi/statements/more/'

def complex_get(param_dict, limit, language, format, attachments):
    # Tests if value is True or "true"
    voidQ = Q(voided=False)
    # keep track if a filter other than time or sequence is used
    reffilter = False

    sinceQ = Q()
    if 'since' in param_dict:
        sinceQ = Q(stored__gt=convert_to_utc(param_dict['since']))

    untilQ = Q()
    if 'until' in param_dict:
        untilQ = Q(stored__lte=convert_to_utc(param_dict['until']))

    # For statements/read/mine oauth scope
    authQ = Q()
    if 'auth' in param_dict and (param_dict['auth'] and 'statements_mine_only' in param_dict['auth']):
        q_auth = param_dict['auth']['authority']

        # If oauth - set authority to look for as the user
        if q_auth.oauth_identifier:
            authQ = Q(authority=q_auth) | Q(authority=q_auth.get_user_from_oauth_group())
        # Chain all of user's oauth clients as well
        else:
            oauth_clients = Agent.objects.filter(member__in=[q_auth])
            authQ = Q(authority=q_auth)
            for client in oauth_clients:
                authQ = authQ | Q(authority=client.get_user_from_oauth_group())

    agentQ = Q()
    if 'agent' in param_dict:
        reffilter = True
        agent = None
        data = param_dict['agent']
        related = 'related_agents' in param_dict and param_dict['related_agents']

        if not type(data) is dict:
            data = convert_to_dict(data)

        try:
            agent = Agent.objects.retrieve_or_create(**data)[0]
            # If agent is already a group, it can't be part of another group
            if agent.objectType == "Group":
                groups = []
            # Since single agent, return all groups it is in
            else:
                groups = agent.member.all()
            agentQ = Q(actor=agent)
            for g in groups:
                agentQ = agentQ | Q(actor=g)
            if related:
                me = chain([agent], groups)
                for a in me:
                    agentQ = agentQ | Q(object_agent=a) | Q(authority=a) \
                          | Q(context_instructor=a) | Q(context_team=a) \
                          | Q(object_substatement__actor=a) \
                          | Q(object_substatement__object_agent=a) \
                          | Q(object_substatement__context_instructor=a) \
                          | Q(object_substatement__context_team=a)
        except IDNotFoundError:
            return[]

    verbQ = Q()
    if 'verb' in param_dict:
        reffilter = True
        verbQ = Q(verb__verb_id=param_dict['verb'])

    # activity
    activityQ = Q()
    if 'activity' in param_dict:
        reffilter = True
        activityQ = Q(object_activity__activity_id=param_dict['activity'])
        if 'related_activities' in param_dict and param_dict['related_activities']:
            activityQ = activityQ | Q(statementcontextactivity__context_activity__activity_id=param_dict['activity']) \
                    | Q(object_substatement__object_activity__activity_id=param_dict['activity']) \
                    | Q(object_substatement__substatementcontextactivity__context_activity__activity_id=param_dict['activity'])

    registrationQ = Q()
    if 'registration' in param_dict:
        reffilter = True
        registrationQ = Q(context_registration=param_dict['registration'])

    # If want ordered by ascending
    stored_param = '-stored'
    if 'ascending' in param_dict and param_dict['ascending']:
            stored_param = 'stored'

    stmtset = Statement.objects.prefetch_related('object_agent','object_activity','object_substatement','object_statementref','actor','verb','context_team','context_instructor','authority').filter(voidQ & untilQ & sinceQ & authQ & agentQ & verbQ & activityQ & registrationQ).distinct()

    # only find references when a filter other than
    # since, until, or limit was used
    if reffilter:
        stmtset = findstmtrefs(stmtset, sinceQ, untilQ)

    # Calculate limit of stmts to return
    return_limit = set_limit(limit)

    # If there are more stmts than the limit, need to break it up and return more id
    if stmtset.count() > return_limit:
        return initial_cache_return(stmtset, stored_param, return_limit, language, format, attachments)
    else:
        return create_stmt_result(stmtset, stored_param, language, format)

def create_stmt_result(stmt_set, stored, language, format):
    stmt_result = {}

    if stmt_set.count() > 0:
        if format == 'exact':
            stmt_result = '{"statements": [%s], "more": ""}' % ",".join([json.dumps(stmt.full_statement) for stmt in stmt_set.order_by(stored)])
        else:
            stmt_result['statements'] = [stmt.to_dict(language, format) for stmt in \
                stmt_set.order_by(stored)]
            stmt_result['more'] = ""
    else:
        stmt_result['statements'] = []
        stmt_result['more'] = ""
    return stmt_result

def findstmtrefs(stmtset, sinceQ, untilQ):
    if stmtset.exists():
        return stmtset
    q = Q()
    for s in stmtset:
        q = q | Q(object_statementref__ref_id=s.statement_id)

    if sinceQ and untilQ:
        q = q & Q(sinceQ, untilQ)
    elif sinceQ:
        q = q & sinceQ
    elif untilQ:
        q = q & untilQ
    # finally weed out voided statements in this lookup
    q = q & Q(voided=False)
    return findstmtrefs(Statement.objects.filter(q).distinct(), sinceQ, untilQ) | stmtset

def create_cache_key(stmt_list):
    # Create unique hash data to use for the cache key
    hash_data = []
    hash_data.append(str(datetime.now()))
    hash_data.append(str(stmt_list))

    # Create cache key from hashed data (always 32 digits)
    key = hashlib.md5(bencode.bencode(hash_data)).hexdigest()
    return key

def initial_cache_return(stmt_list, stored, limit, language, format, attachments):
    # First time someone queries POST/GET
    result = {}
    cache_list = []

    cache_list.append([s for s in stmt_list.order_by(stored).values_list('id', flat=True)])
    stmt_pager = Paginator(cache_list[0], limit)

    # Always start on first page
    current_page = 1
    total_pages = stmt_pager.num_pages

    # Create cache key from hashed data (always 32 digits)
    cache_key = create_cache_key(cache_list[0])

    # Add data to cache
    cache_list.append(current_page)
    cache_list.append(total_pages)
    cache_list.append(limit)
    cache_list.append(attachments)
    cache_list.append(language)
    cache_list.append(format)
    cache_list.append(stored)

    # Encode data
    encoded_info = json.dumps(cache_list)

    # Save encoded_dict in cache
    cache.set(cache_key,encoded_info)

    # Return first page of results
    if format == 'exact':
        result = '{"statements": [%s], "more": "%s"}' % (",".join([json.dumps(stmt.full_statement) for stmt in \
                Statement.objects.filter(id__in=stmt_pager.page(1).object_list).order_by(stored)]), MORE_ENDPOINT + cache_key)
    else:
        result['statements'] = [stmt.to_dict(language, format) for stmt in \
                        Statement.objects.filter(id__in=stmt_pager.page(1).object_list).order_by(stored)]
        result['more'] = MORE_ENDPOINT + cache_key

    return result

def set_limit(req_limit):
    if not req_limit or req_limit > settings.SERVER_STMT_LIMIT:
        req_limit = settings.SERVER_STMT_LIMIT
    return req_limit

def get_more_statement_request(req_id):
    # Retrieve encoded info for statements
    encoded_info = cache.get(req_id)

    # Could have expired or never existed
    if not encoded_info:
        raise NotFound("List does not exist - may have expired after 24 hours")

    # Decode info
    decoded_info = json.loads(encoded_info)

    # Info is always cached as [stmt_list, start_page, total_pages, limit, attachments, language, format]
    stmt_list = decoded_info[0]
    start_page = decoded_info[1]
    total_pages = decoded_info[2]
    limit = decoded_info[3]
    attachments = decoded_info[4]
    language = decoded_info[5]
    format = decoded_info[6]
    stored = decoded_info[7]

    # Build statementResult
    stmt_result = build_statement_result(stmt_list, start_page, total_pages, limit, attachments, language, format, stored, req_id)
    return stmt_result, attachments

# Gets called from req_process after complex_get with list of django objects and also gets called from get_more_statement_request when
# more_id is used so list will be serialized
def build_statement_result(stmt_list, start_page, total_pages, limit, attachments, language, format, stored, more_id):
    result = {}
    current_page = start_page + 1
    # If that was the last page to display then just return the remaining stmts
    if current_page == total_pages:
        stmt_pager = Paginator(stmt_list, limit)
        # Return first page of results
        if format == 'exact':
            result = '{"statements": [%s], "more": ""}' % ",".join([json.dumps(stmt.to_dict(language, format)) for stmt in \
                Statement.objects.filter(id__in=stmt_pager.page(current_page).object_list).order_by(stored)])
        else:
            result['statements'] = [stmt.to_dict(language, format) for stmt in \
                    Statement.objects.filter(id__in=stmt_pager.page(current_page).object_list).order_by(stored)]
            result['more'] = ""
        # Set current page back for when someone hits the URL again
        current_page -= 1
        # Retrieve list stored in cache
        encoded_list = cache.get(more_id)
        # Decode info to set the current page back then encode again
        decoded_list = json.loads(encoded_list)
        decoded_list[1] = current_page
        encoded_list = json.dumps(decoded_list)
        cache.set(more_id, encoded_list)
    # There are more pages to display
    else:
        stmt_pager = Paginator(stmt_list, limit)
        # Create cache key from hashed data (always 32 digits)
        cache_key = create_cache_key(stmt_list)
        # Return first page of results
        if format == 'exact':
            result = '{"statements": [%s], "more": "%s"}' % (",".join([json.dumps(stmt.to_dict(language, format)) for stmt in \
                Statement.objects.filter(id__in=stmt_pager.page(current_page).object_list).order_by(stored)]), MORE_ENDPOINT + cache_key)
        else:
            # Set result to have selected page of stmts and more endpoint
            result['statements'] = [stmt.to_dict(language, format) for stmt in \
                    Statement.objects.filter(id__in=stmt_pager.page(current_page).object_list).order_by(stored)]
            result['more'] = MORE_ENDPOINT + cache_key
        more_cache_list = []
        # Increment next page
        start_page = current_page
        more_cache_list.append(stmt_list)
        more_cache_list.append(start_page)
        more_cache_list.append(total_pages)
        more_cache_list.append(limit)
        more_cache_list.append(attachments)
        more_cache_list.append(language)
        more_cache_list.append(format)
        more_cache_list.append(stored)
        # Encode info
        encoded_list = json.dumps(more_cache_list)
        cache.set(cache_key, encoded_list)
    return result<commit_msg>reverted retreive_statement-slower but all tests passing<commit_after>import bencode
import hashlib
import json
from datetime import datetime
from itertools import chain

from django.core.cache import cache
from django.conf import settings
from django.core.paginator import Paginator
from django.db.models import Q

from util import convert_to_utc, convert_to_dict
from ..models import Statement, Agent
from ..exceptions import NotFound, IDNotFoundError

MORE_ENDPOINT = '/xapi/statements/more/'

def complex_get(param_dict, limit, language, format, attachments):
    # Tests if value is True or "true"
    voidQ = Q(voided=False)
    # keep track if a filter other than time or sequence is used
    reffilter = False

    sinceQ = Q()
    if 'since' in param_dict:
        sinceQ = Q(stored__gt=convert_to_utc(param_dict['since']))

    untilQ = Q()
    if 'until' in param_dict:
        untilQ = Q(stored__lte=convert_to_utc(param_dict['until']))

    # For statements/read/mine oauth scope
    authQ = Q()
    if 'auth' in param_dict and (param_dict['auth'] and 'statements_mine_only' in param_dict['auth']):
        q_auth = param_dict['auth']['authority']

        # If oauth - set authority to look for as the user
        if q_auth.oauth_identifier:
            authQ = Q(authority=q_auth) | Q(authority=q_auth.get_user_from_oauth_group())
        # Chain all of user's oauth clients as well
        else:
            oauth_clients = Agent.objects.filter(member__in=[q_auth])
            authQ = Q(authority=q_auth)
            for client in oauth_clients:
                authQ = authQ | Q(authority=client.get_user_from_oauth_group())

    agentQ = Q()
    if 'agent' in param_dict:
        reffilter = True
        agent = None
        data = param_dict['agent']
        related = 'related_agents' in param_dict and param_dict['related_agents']

        if not type(data) is dict:
            data = convert_to_dict(data)

        try:
            agent = Agent.objects.retrieve_or_create(**data)[0]
            # If agent is already a group, it can't be part of another group
            if agent.objectType == "Group":
                groups = []
            # Since single agent, return all groups it is in
            else:
                groups = agent.member.all()
            agentQ = Q(actor=agent)
            for g in groups:
                agentQ = agentQ | Q(actor=g)
            if related:
                me = chain([agent], groups)
                for a in me:
                    agentQ = agentQ | Q(object_agent=a) | Q(authority=a) \
                          | Q(context_instructor=a) | Q(context_team=a) \
                          | Q(object_substatement__actor=a) \
                          | Q(object_substatement__object_agent=a) \
                          | Q(object_substatement__context_instructor=a) \
                          | Q(object_substatement__context_team=a)
        except IDNotFoundError:
            return[]

    verbQ = Q()
    if 'verb' in param_dict:
        reffilter = True
        verbQ = Q(verb__verb_id=param_dict['verb'])

    # activity
    activityQ = Q()
    if 'activity' in param_dict:
        reffilter = True
        activityQ = Q(object_activity__activity_id=param_dict['activity'])
        if 'related_activities' in param_dict and param_dict['related_activities']:
            activityQ = activityQ | Q(statementcontextactivity__context_activity__activity_id=param_dict['activity']) \
                    | Q(object_substatement__object_activity__activity_id=param_dict['activity']) \
                    | Q(object_substatement__substatementcontextactivity__context_activity__activity_id=param_dict['activity'])

    registrationQ = Q()
    if 'registration' in param_dict:
        reffilter = True
        registrationQ = Q(context_registration=param_dict['registration'])

    # If want ordered by ascending
    stored_param = '-stored'
    if 'ascending' in param_dict and param_dict['ascending']:
            stored_param = 'stored'

    stmtset = Statement.objects.filter(voidQ & untilQ & sinceQ & authQ & agentQ & verbQ & activityQ & registrationQ).distinct()

    # only find references when a filter other than
    # since, until, or limit was used
    if reffilter:
        stmtset = findstmtrefs(stmtset, sinceQ, untilQ)

    # Calculate limit of stmts to return
    return_limit = set_limit(limit)

    # If there are more stmts than the limit, need to break it up and return more id
    if stmtset.count() > return_limit:
        return initial_cache_return(stmtset, stored_param, return_limit, language, format, attachments)
    else:
        return create_stmt_result(stmtset, stored_param, language, format)

def create_stmt_result(stmt_set, stored, language, format):
    stmt_result = {}

    if stmt_set.count() > 0:
        if format == 'exact':
            stmt_result = '{"statements": [%s], "more": ""}' % ",".join([json.dumps(stmt.full_statement) for stmt in stmt_set.order_by(stored)])
        else:
            stmt_result['statements'] = [stmt.to_dict(language, format) for stmt in \
                stmt_set.order_by(stored)]
            stmt_result['more'] = ""
    else:
        stmt_result['statements'] = []
        stmt_result['more'] = ""
    return stmt_result

def findstmtrefs(stmtset, sinceQ, untilQ):
    if stmtset.count() == 0:
        return stmtset
    q = Q()
    for s in stmtset:
        q = q | Q(object_statementref__ref_id=s.statement_id)

    if sinceQ and untilQ:
        q = q & Q(sinceQ, untilQ)
    elif sinceQ:
        q = q & sinceQ
    elif untilQ:
        q = q & untilQ
    # finally weed out voided statements in this lookup
    q = q & Q(voided=False)
    return findstmtrefs(Statement.objects.filter(q).distinct(), sinceQ, untilQ) | stmtset

def create_cache_key(stmt_list):
    # Create unique hash data to use for the cache key
    hash_data = []
    hash_data.append(str(datetime.now()))
    hash_data.append(str(stmt_list))

    # Create cache key from hashed data (always 32 digits)
    key = hashlib.md5(bencode.bencode(hash_data)).hexdigest()
    return key

def initial_cache_return(stmt_list, stored, limit, language, format, attachments):
    # First time someone queries POST/GET
    result = {}
    cache_list = []

    cache_list.append([s for s in stmt_list.order_by(stored).values_list('id', flat=True)])
    stmt_pager = Paginator(cache_list[0], limit)

    # Always start on first page
    current_page = 1
    total_pages = stmt_pager.num_pages

    # Create cache key from hashed data (always 32 digits)
    cache_key = create_cache_key(cache_list[0])

    # Add data to cache
    cache_list.append(current_page)
    cache_list.append(total_pages)
    cache_list.append(limit)
    cache_list.append(attachments)
    cache_list.append(language)
    cache_list.append(format)
    cache_list.append(stored)

    # Encode data
    encoded_info = json.dumps(cache_list)

    # Save encoded_dict in cache
    cache.set(cache_key,encoded_info)

    # Return first page of results
    if format == 'exact':
        result = '{"statements": [%s], "more": "%s"}' % (",".join([json.dumps(stmt.full_statement) for stmt in \
                Statement.objects.filter(id__in=stmt_pager.page(1).object_list).order_by(stored)]), MORE_ENDPOINT + cache_key)
    else:
        result['statements'] = [stmt.to_dict(language, format) for stmt in \
                        Statement.objects.filter(id__in=stmt_pager.page(1).object_list).order_by(stored)]
        result['more'] = MORE_ENDPOINT + cache_key

    return result

def set_limit(req_limit):
    if not req_limit or req_limit > settings.SERVER_STMT_LIMIT:
        req_limit = settings.SERVER_STMT_LIMIT
    return req_limit

def get_more_statement_request(req_id):
    # Retrieve encoded info for statements
    encoded_info = cache.get(req_id)

    # Could have expired or never existed
    if not encoded_info:
        raise NotFound("List does not exist - may have expired after 24 hours")

    # Decode info
    decoded_info = json.loads(encoded_info)

    # Info is always cached as [stmt_list, start_page, total_pages, limit, attachments, language, format]
    stmt_list = decoded_info[0]
    start_page = decoded_info[1]
    total_pages = decoded_info[2]
    limit = decoded_info[3]
    attachments = decoded_info[4]
    language = decoded_info[5]
    format = decoded_info[6]
    stored = decoded_info[7]

    # Build statementResult
    stmt_result = build_statement_result(stmt_list, start_page, total_pages, limit, attachments, language, format, stored, req_id)
    return stmt_result, attachments

# Gets called from req_process after complex_get with list of django objects and also gets called from get_more_statement_request when
# more_id is used so list will be serialized
def build_statement_result(stmt_list, start_page, total_pages, limit, attachments, language, format, stored, more_id):
    result = {}
    current_page = start_page + 1
    # If that was the last page to display then just return the remaining stmts
    if current_page == total_pages:
        stmt_pager = Paginator(stmt_list, limit)
        # Return first page of results
        if format == 'exact':
            result = '{"statements": [%s], "more": ""}' % ",".join([json.dumps(stmt.to_dict(language, format)) for stmt in \
                Statement.objects.filter(id__in=stmt_pager.page(current_page).object_list).order_by(stored)])
        else:
            result['statements'] = [stmt.to_dict(language, format) for stmt in \
                    Statement.objects.filter(id__in=stmt_pager.page(current_page).object_list).order_by(stored)]
            result['more'] = ""
        # Set current page back for when someone hits the URL again
        current_page -= 1
        # Retrieve list stored in cache
        encoded_list = cache.get(more_id)
        # Decode info to set the current page back then encode again
        decoded_list = json.loads(encoded_list)
        decoded_list[1] = current_page
        encoded_list = json.dumps(decoded_list)
        cache.set(more_id, encoded_list)
    # There are more pages to display
    else:
        stmt_pager = Paginator(stmt_list, limit)
        # Create cache key from hashed data (always 32 digits)
        cache_key = create_cache_key(stmt_list)
        # Return first page of results
        if format == 'exact':
            result = '{"statements": [%s], "more": "%s"}' % (",".join([json.dumps(stmt.to_dict(language, format)) for stmt in \
                Statement.objects.filter(id__in=stmt_pager.page(current_page).object_list).order_by(stored)]), MORE_ENDPOINT + cache_key)
        else:
            # Set result to have selected page of stmts and more endpoint
            result['statements'] = [stmt.to_dict(language, format) for stmt in \
                    Statement.objects.filter(id__in=stmt_pager.page(current_page).object_list).order_by(stored)]
            result['more'] = MORE_ENDPOINT + cache_key
        more_cache_list = []
        # Increment next page
        start_page = current_page
        more_cache_list.append(stmt_list)
        more_cache_list.append(start_page)
        more_cache_list.append(total_pages)
        more_cache_list.append(limit)
        more_cache_list.append(attachments)
        more_cache_list.append(language)
        more_cache_list.append(format)
        more_cache_list.append(stored)
        # Encode info
        encoded_list = json.dumps(more_cache_list)
        cache.set(cache_key, encoded_list)
    return result<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
"""
Set up some system-wide loggers
"""

import os.path
import logging
import multiprocessing
from datetime import datetime
from contextlib import contextmanager
from multiprocessing.connection import Client
import numpy

from openquake.commonlib import config

LEVELS = {'debug': logging.DEBUG,
          'info': logging.INFO,
          'warn': logging.WARNING,
          'error': logging.ERROR,
          'critical': logging.CRITICAL}

LOG_FORMAT = ('[%(asctime)s job #%(job_id)s %(hostname)s '
              '%(levelname)s %(processName)s/%(process)s] %(message)s')

LOG = logging.getLogger()

def dbcmd(action, *args):
    """
    A dispatcher to the database server.

    :param action: database action to perform
    :param args: arguments
    """
    try:
        client = Client(config.DBS_ADDRESS, authkey=config.DBS_AUTHKEY)
    except:
        raise RuntimeError('Cannot connect on %s:%s' % config.DBS_ADDRESS)
    try:
        client.send((action,) + args)
        res, etype = client.recv()
    finally:
        client.close()
    if etype:
        raise etype(res)
    return res

def touch_log_file(log_file):
    """
    If a log file destination is specified, attempt to open the file in
    'append' mode ('a'). If the specified file is not writable, an
    :exc:`IOError` will be raised.
    """
    open(os.path.abspath(log_file), 'a').close()

def set_level(level):
    """
    Initialize logs to write records with level `level` or above.
    """
    logging.root.setLevel(LEVELS.get(level, logging.WARNING))

def _update_log_record(self, record):
    """
    Massage a log record before emitting it. Intended to be used by the
    custom log handlers defined in this module.
    """
    if not hasattr(record, 'hostname'):
        record.hostname = '-'
    if not hasattr(record, 'job_id'):
        record.job_id = self.job_id

class LogStreamHandler(logging.StreamHandler):
    """
    Log stream handler
    """
    def __init__(self, job_id):
        super(LogStreamHandler, self).__init__()
        self.setFormatter(logging.Formatter(LOG_FORMAT))
        self.job_id = job_id

    def emit(self, record):  # pylint: disable=E0202
        _update_log_record(self, record)
        super(LogStreamHandler, self).emit(record)

class LogFileHandler(logging.FileHandler):
    """
    Log file handler
    """
    def __init__(self, job_id, log_file):
        super(LogFileHandler, self).__init__(log_file)
        self.setFormatter(logging.Formatter(LOG_FORMAT))
        self.job_id = job_id
        self.log_file = log_file

    def emit(self, record):  # pylint: disable=E0202
        _update_log_record(self, record)
        super(LogFileHandler, self).emit(record)

class LogDatabaseHandler(logging.Handler):
    """
    Log stream handler
    """
    def __init__(self, job_id):
        super(LogDatabaseHandler, self).__init__()
        self.job_id = job_id

    def emit(self, record):  # pylint: disable=E0202
        if record.levelno >= logging.INFO:
            dbcmd('log', self.job_id, datetime.utcnow(), record.levelname,
                  '%s/%s' % (record.processName, record.process),
                  record.getMessage())

@contextmanager
def handle(job_id, log_level='info', log_file=None):
    """
    Context manager adding and removing log handlers.

    :param job_id:
         ID of the current job
    :param log_level:
         one of debug, info, warn, error, critical
    :param log_file:
         log file path (if None, logs on stdout only)
    """
    handlers = [LogDatabaseHandler(job_id)]  # log on db always
    if log_file is None:
        # add a StreamHandler if not already there
        if not any(h for h in logging.root.handlers
                   if isinstance(h, logging.StreamHandler)):
            handlers.append(LogStreamHandler(job_id))
    else:
        handlers.append(LogFileHandler(job_id, log_file))
    for handler in handlers:
        logging.root.addHandler(handler)
    set_level(log_level)
    try:
        yield
    finally:
        # sanity check to make sure that the logging on file is working
        if log_file and os.path.getsize(log_file) == 0:
            logging.root.warn('The log file %s is empty!?' % log_file)
        for handler in handlers:
            logging.root.removeHandler(handler)

# #################### getting sequential indices ###################### #

shared_index = multiprocessing.Value('L', 0)

def _get_seq_ids(num_ids):
    with shared_index.get_lock():
        start = shared_index.value
        stop = start + num_ids
        shared_index.value = stop
    return numpy.arange(start, stop, dtype=numpy.uint32)

def get_seq_ids(num_ids):
    """
    :param num_ids: the number of sequential IDs to return
    :returns: an array of length `num_ids` and dtype `numpy.uint32`
    """
    if os.environ.get('OQ_DISTRIBUTE') == 'celery':
        return dbcmd(_get_seq_ids, num_ids)
    else:
        return _get_seq_ids(num_ids)
<commit_msg>Small fix<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
"""
Set up some system-wide loggers
"""

import os.path
import logging
import multiprocessing
from datetime import datetime
from contextlib import contextmanager
from multiprocessing.connection import Client
import numpy

from openquake.commonlib import config

LEVELS = {'debug': logging.DEBUG,
          'info': logging.INFO,
          'warn': logging.WARNING,
          'error': logging.ERROR,
          'critical': logging.CRITICAL}

LOG_FORMAT = ('[%(asctime)s job #%(job_id)s %(hostname)s '
              '%(levelname)s %(processName)s/%(process)s] %(message)s')

LOG = logging.getLogger()

def dbcmd(action, *args):
    """
    A dispatcher to the database server.

    :param action: database action to perform
    :param args: arguments
    """
    try:
        client = Client(config.DBS_ADDRESS, authkey=config.DBS_AUTHKEY)
    except:
        raise RuntimeError('Cannot connect on %s:%s' % config.DBS_ADDRESS)
    try:
        client.send((action,) + args)
        res, etype = client.recv()
    finally:
        client.close()
    if etype:
        raise etype(res)
    return res

def touch_log_file(log_file):
    """
    If a log file destination is specified, attempt to open the file in
    'append' mode ('a'). If the specified file is not writable, an
    :exc:`IOError` will be raised.
    """
    open(os.path.abspath(log_file), 'a').close()

def set_level(level):
    """
    Initialize logs to write records with level `level` or above.
    """
    logging.root.setLevel(LEVELS.get(level, logging.WARNING))

def _update_log_record(self, record):
    """
    Massage a log record before emitting it. Intended to be used by the
    custom log handlers defined in this module.
    """
    if not hasattr(record, 'hostname'):
        record.hostname = '-'
    if not hasattr(record, 'job_id'):
        record.job_id = self.job_id

class LogStreamHandler(logging.StreamHandler):
    """
    Log stream handler
    """
    def __init__(self, job_id):
        super(LogStreamHandler, self).__init__()
        self.setFormatter(logging.Formatter(LOG_FORMAT))
        self.job_id = job_id

    def emit(self, record):  # pylint: disable=E0202
        _update_log_record(self, record)
        super(LogStreamHandler, self).emit(record)

class LogFileHandler(logging.FileHandler):
    """
    Log file handler
    """
    def __init__(self, job_id, log_file):
        super(LogFileHandler, self).__init__(log_file)
        self.setFormatter(logging.Formatter(LOG_FORMAT))
        self.job_id = job_id
        self.log_file = log_file

    def emit(self, record):  # pylint: disable=E0202
        _update_log_record(self, record)
        super(LogFileHandler, self).emit(record)

class LogDatabaseHandler(logging.Handler):
    """
    Log stream handler
    """
    def __init__(self, job_id):
        super(LogDatabaseHandler, self).__init__()
        self.job_id = job_id

    def emit(self, record):  # pylint: disable=E0202
        if record.levelno >= logging.INFO:
            dbcmd('log', self.job_id, datetime.utcnow(), record.levelname,
                  '%s/%s' % (record.processName, record.process),
                  record.getMessage())

@contextmanager
def handle(job_id, log_level='info', log_file=None):
    """
    Context manager adding and removing log handlers.

    :param job_id:
         ID of the current job
    :param log_level:
         one of debug, info, warn, error, critical
    :param log_file:
         log file path (if None, logs on stdout only)
    """
    handlers = [LogDatabaseHandler(job_id)]  # log on db always
    if log_file is None:
        # add a StreamHandler if not already there
        if not any(h for h in logging.root.handlers
                   if isinstance(h, logging.StreamHandler)):
            handlers.append(LogStreamHandler(job_id))
    else:
        handlers.append(LogFileHandler(job_id, log_file))
    for handler in handlers:
        logging.root.addHandler(handler)
    set_level(log_level)
    try:
        yield
    finally:
        # sanity check to make sure that the logging on file is working
        if log_file and os.path.getsize(log_file) == 0:
            logging.root.warn('The log file %s is empty!?' % log_file)
        for handler in handlers:
            logging.root.removeHandler(handler)

# #################### getting sequential indices ###################### #

shared_index = multiprocessing.Value('L', 0)

def _get_seq_ids(num_ids):
    with shared_index.get_lock():
        start = shared_index.value
        stop = start + num_ids
        shared_index.value = stop
    return numpy.arange(start, stop, dtype=numpy.uint32)

def get_seq_ids(num_ids):
    """
    :param num_ids: the number of sequential IDs to return
    :returns: an array of length `num_ids` and dtype `numpy.uint32`
    """
    if os.environ.get('OQ_DISTRIBUTE') == 'celery':
        return dbcmd('get_seq_ids', num_ids)
    else:
        return _get_seq_ids(num_ids)
<|endoftext|>


<commit_before>import argparse
import logging
import os
import shutil
import sys

import numpy as np

from ludwig import __file__ as ludwig_path
from ludwig.api import LudwigModel
from ludwig.constants import CATEGORY, NUMERICAL, BINARY, SEQUENCE, TEXT, SET, \
    VECTOR
from ludwig.globals import MODEL_HYPERPARAMETERS_FILE_NAME, \
    TRAIN_SET_METADATA_FILE_NAME, MODEL_WEIGHTS_FILE_NAME, LUDWIG_VERSION, \
    MODEL_WEIGHTS_PROGRESS_FILE_NAME
from ludwig.utils.data_utils import load_json
from ludwig.utils.print_utils import logging_level_registry, print_ludwig

logger = logging.getLogger(__name__)

class LudwigNeuropodModelWrapper:
    def __init__(self, data_root):
        self.ludwig_model = LudwigModel.load(data_root)

    def __call__(self, **kwargs):
        print('__call__', file=sys.stderr)
        data_dict = kwargs
        for key in data_dict:
            data_dict[key] = np.squeeze(data_dict[key], axis=1)
        predicted = self.ludwig_model.predict(
            data_dict=data_dict, return_type=dict
        )
        # print(predicted, file=sys.stderr)
        return postprocess_for_neuropod(
            predicted, self.ludwig_model.model_definition
        )

def get_model(data_root):
    print('get_model()', data_root, file=sys.stderr)
    return LudwigNeuropodModelWrapper(data_root)

def postprocess_for_neuropod(predicted, model_definition):
    postprocessed = {}
    for output_feature in model_definition['output_features']:
        feature_name = output_feature['name']
        feature_type = output_feature['type']
        if feature_type == BINARY:
            postprocessed[feature_name + "_predictions"] = \
                np.expand_dims(
                    predicted[feature_name]['predictions'].astype('str'), 1)
            postprocessed[feature_name + "_probabilities"] = \
                np.expand_dims(predicted[feature_name]['probabilities'], 1)
        elif feature_type == NUMERICAL:
            postprocessed[feature_name + "_predictions"] = \
                np.expand_dims(predicted[feature_name]['predictions'], 1)
        elif feature_type == CATEGORY:
            postprocessed[feature_name + "_predictions"] = np.expand_dims(
                np.array(predicted[feature_name]['predictions'], dtype='str'), 1
            )
            postprocessed[feature_name + "_probability"] = \
                np.expand_dims(predicted[feature_name]['probability'], 1)
            postprocessed[feature_name + "_probabilities"] = \
                predicted[feature_name]['probabilities']
        elif feature_type == SEQUENCE:
            predictions = list(map(
                lambda x: ' '.join(x),
                predicted[feature_name]['predictions']
            ))
            postprocessed[feature_name + "_predictions"] = np.expand_dims(
                np.array(predictions, dtype='str'), 1
            )
        elif feature_type == TEXT:
            predictions = list(map(
                lambda x: ' '.join(x),
                predicted[feature_name]['predictions']
            ))
            postprocessed[feature_name + "_predictions"] = np.expand_dims(
                np.array(predictions, dtype='str'), 1
            )
        elif feature_type == SET:
            predictions = list(map(
                lambda x: ' '.join(x),
                predicted[feature_name]['predictions']
            ))
            postprocessed[feature_name + "_predictions"] = np.expand_dims(
                np.array(predictions, dtype='str'), 1
            )
            probability = list(map(
                lambda x: ' '.join([str(e) for e in x]),
                predicted[feature_name]['probability']
            ))
            postprocessed[feature_name + "_probability"] = np.expand_dims(
                np.array(probability, dtype='str'), 1
            )
            postprocessed[feature_name + "_probabilities"] = \
                predicted[feature_name]['probabilities']
        elif feature_type == VECTOR:
            postprocessed[feature_name + "_predictions"] = \
                predicted[feature_name]['predictions']
        else:
            postprocessed[feature_name + "_predictions"] = np.expand_dims(
                np.array(predicted[feature_name]['predictions'], dtype='str'), 1
            )

    return postprocessed

def export_neuropod(
        ludwig_model_path,
        neuropod_path,
        neuropod_model_name="ludwig_model"
):
    try:
        from neuropod.backends.python.packager import create_python_neuropod
    except ImportError:
        logger.error(
            'The "neuropod" package is not installed in your environment.'
        )
        sys.exit(-1)

    data_paths = [
        {
            "path": os.path.join(
                ludwig_model_path, MODEL_HYPERPARAMETERS_FILE_NAME
            ),
            "packaged_name": MODEL_HYPERPARAMETERS_FILE_NAME
        },
        {
            "path": os.path.join(
                ludwig_model_path, TRAIN_SET_METADATA_FILE_NAME
            ),
            "packaged_name": TRAIN_SET_METADATA_FILE_NAME
        },
        {
            "path": os.path.join(
                ludwig_model_path, 'checkpoint'
            ),
            "packaged_name": 'checkpoint'
        },
    ]
    for filename in os.listdir(ludwig_model_path):
        if (MODEL_WEIGHTS_FILE_NAME in filename
                and MODEL_WEIGHTS_PROGRESS_FILE_NAME not in filename):
            data_paths.append(
                {
                    "path": os.path.join(
                        ludwig_model_path, filename
                    ),
                    "packaged_name": filename
                }
            )

    logger.debug('data_paths: {}'.format(data_paths))

    ludwig_model_definition = load_json(
        os.path.join(
            ludwig_model_path,
            MODEL_HYPERPARAMETERS_FILE_NAME
        )
    )
    training_set_metadata = load_json(
        os.path.join(
            ludwig_model_path,
            TRAIN_SET_METADATA_FILE_NAME
        )
    )

    input_spec = []
    for feature in ludwig_model_definition['input_features']:
        input_spec.append({
            "name": feature['name'],
            "dtype": "str",
            "shape": (None, 1)
        })
    logger.debug('input_spec: {}'.format(input_spec))

    output_spec = []
    for feature in ludwig_model_definition['output_features']:
        feature_type = feature['type']
        feature_name = feature['name']
        if feature_type == BINARY:
            output_spec.append({
                "name": feature['name'] + '_predictions',
                "dtype": "str",
                "shape": (None, 1)
            })
            output_spec.append({
                "name": feature['name'] + '_probabilities',
                "dtype": "float32",
                "shape": (None, 1)
            })
        elif feature_type == NUMERICAL:
            output_spec.append({
                "name": feature['name'] + '_predictions',
                "dtype": "float32",
                "shape": (None, 1)
            })
        elif feature_type == CATEGORY:
            output_spec.append({
                "name": feature['name'] + '_predictions',
                "dtype": "str",
                "shape": (None, 1)
            })
            output_spec.append({
                "name": feature['name'] + '_probability',
                "dtype": "float32",
                "shape": (None, 1)
            })
            output_spec.append({
                "name": feature['name'] + '_probabilities',
                "dtype": "float32",
                "shape": (
                    None, training_set_metadata[feature_name]['vocab_size']
                )
            })
        elif feature_type == SEQUENCE:
            output_spec.append({
                "name": feature['name'] + '_predictions',
                "dtype": "str",
                "shape": (None, 1)
            })
        elif feature_type == TEXT:
            output_spec.append({
                "name": feature['name'] + '_predictions',
                "dtype": "str",
                "shape": (None, 1)
            })
        elif feature_type == SET:
            output_spec.append({
                "name": feature['name'] + '_predictions',
                "dtype": "str",
                "shape": (None, 1)
            })
            output_spec.append({
                "name": feature['name'] + '_probability',
                "dtype": "str",
                "shape": (None, 1)
            })
            output_spec.append({
                "name": feature['name'] + '_probabilities',
                "dtype": "float32",
                "shape": (
                    None, training_set_metadata[feature_name]['vocab_size']
                )
            })
        elif feature_type == VECTOR:
            output_spec.append({
                "name": feature['name'] + '_predictions',
                "dtype": "float32",
                "shape": (
                    None, training_set_metadata[feature_name]['vector_size']
                )
            })
        else:
            output_spec.append({
                "name": feature['name'] + '_predictions',
                "dtype": "str",
                "shape": (None, 1)
            })
    logger.debug('output_spec: {}'.format(output_spec))

    if os.path.exists(neuropod_path):
        if os.path.isfile(neuropod_path):
            logger.warning('Removing file: {}'.format(neuropod_path))
            os.remove(neuropod_path)
        else:
            logger.warning('Removing directory: {}'.format(neuropod_path))
            shutil.rmtree(neuropod_path, ignore_errors=True)

    from pathlib import Path
    path = Path(ludwig_path)
    logger.debug('python_root: {}'.format(path.parent.parent))

    create_python_neuropod(
        neuropod_path=neuropod_path,
        model_name=neuropod_model_name,
        data_paths=data_paths,
        code_path_spec=[{
            "python_root": path.parent.parent,
            "dirs_to_package": [
                "ludwig"  # Package everything in the python_root
            ],
        }],
        entrypoint_package="ludwig.utils.neuropod_utils",
        entrypoint="get_model",
        # test_deps=['torch', 'numpy'],
        skip_virtualenv=True,
        input_spec=input_spec,
        output_spec=output_spec
    )
    logger.info('Neuropod saved to: {}'.format(neuropod_path))

def cli():
    parser = argparse.ArgumentParser(
        description='This script exports a Ludwig model in the Neuropod format'
    )

    # ----------------
    # Model parameters
    # ----------------
    parser.add_argument(
        '-m',
        '--ludwig_model_path',
        help='path to the Ludwig model to export',
        required=True
    )

    parser.add_argument(
        '-l',
        '--logging_level',
        default='info',
        help='the level of logging to use',
        choices=['critical', 'error', 'warning', 'info', 'debug', 'notset']
    )

    # -------------------
    # Neuropod parameters
    # -------------------
    parser.add_argument(
        '-n',
        '--neuropod_path',
        help='path of the output Neuropod package file',
        required=True
    )
    parser.add_argument(
        '-nm',
        '--neuropod_model_name',
        help='path of the output Neuropod package file',
        default='ludwig_model'
    )

    args = parser.parse_args()

    logging.getLogger('ludwig').setLevel(
        logging_level_registry[args.logging_level]
    )
    global logger
    logger = logging.getLogger('ludwig.serve')

    print_ludwig('Export Neuropod', LUDWIG_VERSION)

    export_neuropod(
        args.ludwig_model_path, args.neuropod_path, args.neuropod_model_name
    )

if __name__ == '__main__':
    # contrib_command("neuropod", *sys.argv)
    cli()
<commit_msg>Added package_as_dir to neuropod_utils<commit_after>import argparse
import logging
import os
import shutil
import sys

import numpy as np

from ludwig import __file__ as ludwig_path
from ludwig.api import LudwigModel
from ludwig.constants import CATEGORY, NUMERICAL, BINARY, SEQUENCE, TEXT, SET, \
    VECTOR
from ludwig.globals import MODEL_HYPERPARAMETERS_FILE_NAME, \
    TRAIN_SET_METADATA_FILE_NAME, MODEL_WEIGHTS_FILE_NAME, LUDWIG_VERSION, \
    MODEL_WEIGHTS_PROGRESS_FILE_NAME
from ludwig.utils.data_utils import load_json
from ludwig.utils.print_utils import logging_level_registry, print_ludwig

logger = logging.getLogger(__name__)

class LudwigNeuropodModelWrapper:
    def __init__(self, data_root):
        self.ludwig_model = LudwigModel.load(data_root)

    def __call__(self, **kwargs):
        print('__call__', file=sys.stderr)
        data_dict = kwargs
        for key in data_dict:
            data_dict[key] = np.squeeze(data_dict[key], axis=1)
        predicted = self.ludwig_model.predict(
            data_dict=data_dict, return_type=dict
        )
        # print(predicted, file=sys.stderr)
        return postprocess_for_neuropod(
            predicted, self.ludwig_model.model_definition
        )

def get_model(data_root):
    print('get_model()', data_root, file=sys.stderr)
    return LudwigNeuropodModelWrapper(data_root)

def postprocess_for_neuropod(predicted, model_definition):
    postprocessed = {}
    for output_feature in model_definition['output_features']:
        feature_name = output_feature['name']
        feature_type = output_feature['type']
        if feature_type == BINARY:
            postprocessed[feature_name + "_predictions"] = \
                np.expand_dims(
                    predicted[feature_name]['predictions'].astype('str'), 1)
            postprocessed[feature_name + "_probabilities"] = \
                np.expand_dims(predicted[feature_name]['probabilities'], 1)
        elif feature_type == NUMERICAL:
            postprocessed[feature_name + "_predictions"] = \
                np.expand_dims(predicted[feature_name]['predictions'], 1)
        elif feature_type == CATEGORY:
            postprocessed[feature_name + "_predictions"] = np.expand_dims(
                np.array(predicted[feature_name]['predictions'], dtype='str'), 1
            )
            postprocessed[feature_name + "_probability"] = \
                np.expand_dims(predicted[feature_name]['probability'], 1)
            postprocessed[feature_name + "_probabilities"] = \
                predicted[feature_name]['probabilities']
        elif feature_type == SEQUENCE:
            predictions = list(map(
                lambda x: ' '.join(x),
                predicted[feature_name]['predictions']
            ))
            postprocessed[feature_name + "_predictions"] = np.expand_dims(
                np.array(predictions, dtype='str'), 1
            )
        elif feature_type == TEXT:
            predictions = list(map(
                lambda x: ' '.join(x),
                predicted[feature_name]['predictions']
            ))
            postprocessed[feature_name + "_predictions"] = np.expand_dims(
                np.array(predictions, dtype='str'), 1
            )
        elif feature_type == SET:
            predictions = list(map(
                lambda x: ' '.join(x),
                predicted[feature_name]['predictions']
            ))
            postprocessed[feature_name + "_predictions"] = np.expand_dims(
                np.array(predictions, dtype='str'), 1
            )
            probability = list(map(
                lambda x: ' '.join([str(e) for e in x]),
                predicted[feature_name]['probability']
            ))
            postprocessed[feature_name + "_probability"] = np.expand_dims(
                np.array(probability, dtype='str'), 1
            )
            postprocessed[feature_name + "_probabilities"] = \
                predicted[feature_name]['probabilities']
        elif feature_type == VECTOR:
            postprocessed[feature_name + "_predictions"] = \
                predicted[feature_name]['predictions']
        else:
            postprocessed[feature_name + "_predictions"] = np.expand_dims(
                np.array(predicted[feature_name]['predictions'], dtype='str'), 1
            )

    return postprocessed

def export_neuropod(
        ludwig_model_path,
        neuropod_path,
        neuropod_model_name="ludwig_model",
        package_as_dir=False
):
    try:
        from neuropod.backends.python.packager import create_python_neuropod
    except ImportError:
        logger.error(
            'The "neuropod" package is not installed in your environment.'
        )
        sys.exit(-1)

    data_paths = [
        {
            "path": os.path.join(
                ludwig_model_path, MODEL_HYPERPARAMETERS_FILE_NAME
            ),
            "packaged_name": MODEL_HYPERPARAMETERS_FILE_NAME
        },
        {
            "path": os.path.join(
                ludwig_model_path, TRAIN_SET_METADATA_FILE_NAME
            ),
            "packaged_name": TRAIN_SET_METADATA_FILE_NAME
        },
        {
            "path": os.path.join(
                ludwig_model_path, 'checkpoint'
            ),
            "packaged_name": 'checkpoint'
        },
    ]
    for filename in os.listdir(ludwig_model_path):
        if (MODEL_WEIGHTS_FILE_NAME in filename
                and MODEL_WEIGHTS_PROGRESS_FILE_NAME not in filename):
            data_paths.append(
                {
                    "path": os.path.join(
                        ludwig_model_path, filename
                    ),
                    "packaged_name": filename
                }
            )

    logger.debug('data_paths: {}'.format(data_paths))

    ludwig_model_definition = load_json(
        os.path.join(
            ludwig_model_path,
            MODEL_HYPERPARAMETERS_FILE_NAME
        )
    )
    training_set_metadata = load_json(
        os.path.join(
            ludwig_model_path,
            TRAIN_SET_METADATA_FILE_NAME
        )
    )

    input_spec = []
    for feature in ludwig_model_definition['input_features']:
        input_spec.append({
            "name": feature['name'],
            "dtype": "str",
            "shape": (None, 1)
        })
    logger.debug('input_spec: {}'.format(input_spec))

    output_spec = []
    for feature in ludwig_model_definition['output_features']:
        feature_type = feature['type']
        feature_name = feature['name']
        if feature_type == BINARY:
            output_spec.append({
                "name": feature['name'] + '_predictions',
                "dtype": "str",
                "shape": (None, 1)
            })
            output_spec.append({
                "name": feature['name'] + '_probabilities',
                "dtype": "float32",
                "shape": (None, 1)
            })
        elif feature_type == NUMERICAL:
            output_spec.append({
                "name": feature['name'] + '_predictions',
                "dtype": "float32",
                "shape": (None, 1)
            })
        elif feature_type == CATEGORY:
            output_spec.append({
                "name": feature['name'] + '_predictions',
                "dtype": "str",
                "shape": (None, 1)
            })
            output_spec.append({
                "name": feature['name'] + '_probability',
                "dtype": "float32",
                "shape": (None, 1)
            })
            output_spec.append({
                "name": feature['name'] + '_probabilities',
                "dtype": "float32",
                "shape": (
                    None, training_set_metadata[feature_name]['vocab_size']
                )
            })
        elif feature_type == SEQUENCE:
            output_spec.append({
                "name": feature['name'] + '_predictions',
                "dtype": "str",
                "shape": (None, 1)
            })
        elif feature_type == TEXT:
            output_spec.append({
                "name": feature['name'] + '_predictions',
                "dtype": "str",
                "shape": (None, 1)
            })
        elif feature_type == SET:
            output_spec.append({
                "name": feature['name'] + '_predictions',
                "dtype": "str",
                "shape": (None, 1)
            })
            output_spec.append({
                "name": feature['name'] + '_probability',
                "dtype": "str",
                "shape": (None, 1)
            })
            output_spec.append({
                "name": feature['name'] + '_probabilities',
                "dtype": "float32",
                "shape": (
                    None, training_set_metadata[feature_name]['vocab_size']
                )
            })
        elif feature_type == VECTOR:
            output_spec.append({
                "name": feature['name'] + '_predictions',
                "dtype": "float32",
                "shape": (
                    None, training_set_metadata[feature_name]['vector_size']
                )
            })
        else:
            output_spec.append({
                "name": feature['name'] + '_predictions',
                "dtype": "str",
                "shape": (None, 1)
            })
    logger.debug('output_spec: {}'.format(output_spec))

    if os.path.exists(neuropod_path):
        if os.path.isfile(neuropod_path):
            logger.warning('Removing file: {}'.format(neuropod_path))
            os.remove(neuropod_path)
        else:
            logger.warning('Removing directory: {}'.format(neuropod_path))
            shutil.rmtree(neuropod_path, ignore_errors=True)

    from pathlib import Path
    path = Path(ludwig_path)
    logger.debug('python_root: {}'.format(path.parent.parent))

    create_python_neuropod(
        neuropod_path=neuropod_path,
        model_name=neuropod_model_name,
        data_paths=data_paths,
        code_path_spec=[{
            "python_root": path.parent.parent,
            "dirs_to_package": [
                "ludwig"  # Package everything in the python_root
            ],
        }],
        entrypoint_package="ludwig.utils.neuropod_utils",
        entrypoint="get_model",
        # test_deps=['torch', 'numpy'],
        package_as_zip=not package_as_dir,
        skip_virtualenv=True,
        input_spec=input_spec,
        output_spec=output_spec
    )
    logger.info('Neuropod saved to: {}'.format(neuropod_path))

def cli():
    parser = argparse.ArgumentParser(
        description='This script exports a Ludwig model in the Neuropod format'
    )

    # ----------------
    # Model parameters
    # ----------------
    parser.add_argument(
        '-m',
        '--ludwig_model_path',
        help='path to the Ludwig model to export',
        required=True
    )

    parser.add_argument(
        '-l',
        '--logging_level',
        default='info',
        help='the level of logging to use',
        choices=['critical', 'error', 'warning', 'info', 'debug', 'notset']
    )

    # -------------------
    # Neuropod parameters
    # -------------------
    parser.add_argument(
        '-n',
        '--neuropod_path',
        help='path of the output Neuropod package file',
        required=True
    )
    parser.add_argument(
        '-nm',
        '--neuropod_model_name',
        help='path of the output Neuropod package file',
        default='ludwig_model'
    )
    parser.add_argument(
        '-d',
        '--package_as_dir',
        help='output the Neuropod package a directory '
             '(by default it is a zip file)',
        action='store_true',
        default=False
    )

    args = parser.parse_args()

    logging.getLogger('ludwig').setLevel(
        logging_level_registry[args.logging_level]
    )
    global logger
    logger = logging.getLogger('ludwig.serve')

    print_ludwig('Export Neuropod', LUDWIG_VERSION)

    export_neuropod(
        args.ludwig_model_path,
        args.neuropod_path,
        args.neuropod_model_name,
        args.package_as_dir
    )

if __name__ == '__main__':
    # contrib_command("neuropod", *sys.argv)
    cli()
<|endoftext|>


<commit_before>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (c) 2015, GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import numpy

def max_rel_diff(curve_ref, curve, min_value=0.01):
    """
    Compute the maximum relative difference between two curves. Only values
    greather or equal than the min_value are considered.

    >>> curve_ref = [0.01, 0.02, 0.03, 0.05, 1.0]
    >>> curve = [0.011, 0.021, 0.031, 0.051, 1.0]
    >>> round(max_rel_diff(curve_ref, curve), 2)
    0.1
    """
    assert len(curve_ref) == len(curve), (len(curve_ref), len(curve))
    assert len(curve), 'The curves are empty!'
    max_diff = 0
    for c1, c2 in zip(curve_ref, curve):
        if c1 >= min_value:
            max_diff = max(max_diff, abs(c1 - c2) / c1)
    return max_diff

def max_rel_diff_index(curve_ref, curve, min_value=0.01):
    """
    Compute the maximum relative difference between two sets of curves.
    Only values greather or equal than the min_value are considered.
    Return both the maximum difference and its location (array index).

    >>> curve_refs = [[0.01, 0.02, 0.03, 0.05], [0.01, 0.02, 0.04, 0.06]]
    >>> curves = [[0.011, 0.021, 0.031, 0.051], [0.012, 0.022, 0.032, 0.051]]
    >>> max_rel_diff_index(curve_refs, curves)
    (0.2, 1)
    """
    assert len(curve_ref) == len(curve), (len(curve_ref), len(curve))
    assert len(curve), 'The curves are empty!'
    diffs = [max_rel_diff(c1, c2, min_value)
             for c1, c2 in zip(curve_ref, curve)]
    maxdiff = max(diffs)
    maxindex = diffs.index(maxdiff)
    return maxdiff, maxindex

def rmsep(array_ref, array, min_value=0.01):
    """
    Root Mean Square Error Percentage for two arrays.

    :param array_ref: reference array
    :param array: another array
    :param min_value: compare only the elements larger than min_value
    :returns: the relative distance between the arrays

    >>> curve_ref = numpy.array([[0.01, 0.02, 0.03, 0.05],
    ... [0.01, 0.02, 0.04, 0.06]])
    >>> curve = numpy.array([[0.011, 0.021, 0.031, 0.051],
    ... [0.012, 0.022, 0.032, 0.051]])
    >>> round(rmsep(curve_ref, curve), 5)
    0.11292
    """
    bigvalues = array_ref > min_value
    reldiffsquare = (1. - array[bigvalues] / array_ref[bigvalues]) ** 2
    return numpy.sqrt(reldiffsquare.mean())
<commit_msg>Fixed a bug with integer division<commit_after>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (c) 2015, GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import division
import numpy

def max_rel_diff(curve_ref, curve, min_value=0.01):
    """
    Compute the maximum relative difference between two curves. Only values
    greather or equal than the min_value are considered.

    >>> curve_ref = [0.01, 0.02, 0.03, 0.05, 1.0]
    >>> curve = [0.011, 0.021, 0.031, 0.051, 1.0]
    >>> round(max_rel_diff(curve_ref, curve), 2)
    0.1
    """
    assert len(curve_ref) == len(curve), (len(curve_ref), len(curve))
    assert len(curve), 'The curves are empty!'
    max_diff = 0
    for c1, c2 in zip(curve_ref, curve):
        if c1 >= min_value:
            max_diff = max(max_diff, abs(c1 - c2) / c1)
    return max_diff

def max_rel_diff_index(curve_ref, curve, min_value=0.01):
    """
    Compute the maximum relative difference between two sets of curves.
    Only values greather or equal than the min_value are considered.
    Return both the maximum difference and its location (array index).

    >>> curve_refs = [[0.01, 0.02, 0.03, 0.05], [0.01, 0.02, 0.04, 0.06]]
    >>> curves = [[0.011, 0.021, 0.031, 0.051], [0.012, 0.022, 0.032, 0.051]]
    >>> max_rel_diff_index(curve_refs, curves)
    (0.2, 1)
    """
    assert len(curve_ref) == len(curve), (len(curve_ref), len(curve))
    assert len(curve), 'The curves are empty!'
    diffs = [max_rel_diff(c1, c2, min_value)
             for c1, c2 in zip(curve_ref, curve)]
    maxdiff = max(diffs)
    maxindex = diffs.index(maxdiff)
    return maxdiff, maxindex

def rmsep(array_ref, array, min_value=0.01):
    """
    Root Mean Square Error Percentage for two arrays.

    :param array_ref: reference array
    :param array: another array
    :param min_value: compare only the elements larger than min_value
    :returns: the relative distance between the arrays

    >>> curve_ref = numpy.array([[0.01, 0.02, 0.03, 0.05],
    ... [0.01, 0.02, 0.04, 0.06]])
    >>> curve = numpy.array([[0.011, 0.021, 0.031, 0.051],
    ... [0.012, 0.022, 0.032, 0.051]])
    >>> round(rmsep(curve_ref, curve), 5)
    0.11292
    """
    bigvalues = array_ref > min_value
    reldiffsquare = (1. - array[bigvalues] / array_ref[bigvalues]) ** 2
    return numpy.sqrt(reldiffsquare.mean())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

__author__ = """Adrian Bulat"""
__email__ = 'adrian.bulat@nottingham.ac.uk'
__version__ = '1.0.1'

from .api import FaceAlignment, LandmarksType, NetworkSize
<commit_msg>forgot yo bump version<commit_after># -*- coding: utf-8 -*-

__author__ = """Adrian Bulat"""
__email__ = 'adrian.bulat@nottingham.ac.uk'
__version__ = '1.1.0'

from .api import FaceAlignment, LandmarksType, NetworkSize
<|endoftext|>