<commit_before>"""
@brief      test log(time=0s)
"""

import sys
import os
import unittest
import warnings

try:
    import pyquickhelper as skip_
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..",
                "..",
                "pyquickhelper",
                "src",)))
    if path not in sys.path:
        sys.path.append(path)
    import pyquickhelper as skip_

from pyquickhelper.loghelper import fLOG
from pyquickhelper.pycode import check_pep8
from pyquickhelper.pycode.utils_tests_helper import _extended_refactoring

class TestFlake8(unittest.TestCase):

    def test_flake8_src(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if sys.version_info[0] == 2 or "Anaconda" in sys.executable \
                or "condavir" in sys.executable:
            warnings.warn(
                "skipping test_flake8 because of Python 2 or " + sys.executable)
            return

        thi = os.path.abspath(os.path.dirname(__file__))
        src_ = os.path.normpath(os.path.join(thi, "..", "..", "src"))
        check_pep8(src_, fLOG=fLOG, extended=[("fLOG", _extended_refectoring)],
                   neg_filter=".*faq_python.py$")

    def test_flake8_test(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if sys.version_info[0] == 2 or "Anaconda" in sys.executable \
                or "condavir" in sys.executable:
            warnings.warn(
                "skipping test_flake8 because of Python 2 or " + sys.executable)
            return

        thi = os.path.abspath(os.path.dirname(__file__))
        test = os.path.normpath(os.path.join(thi, "..", ))
        check_pep8(test, fLOG=fLOG, neg_filter="temp_.*",
                   skip=["'src' imported but unused",
                         "'skip_' imported but unused",
                         "'skip__' imported but unused",
                         "'skip___' imported but unused",
                         ],
                   extended=[("fLOG", _extended_refectoring)],
                   max_line_length=320)

if __name__ == "__main__":
    unittest.main()
<commit_msg>fix test_flake8<commit_after>"""
@brief      test log(time=0s)
"""

import sys
import os
import unittest
import warnings

try:
    import pyquickhelper as skip_
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..",
                "..",
                "pyquickhelper",
                "src",)))
    if path not in sys.path:
        sys.path.append(path)
    import pyquickhelper as skip_

from pyquickhelper.loghelper import fLOG
from pyquickhelper.pycode import check_pep8
from pyquickhelper.pycode.utils_tests_helper import _extended_refactoring

class TestFlake8(unittest.TestCase):

    def test_flake8_src(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if sys.version_info[0] == 2 or "Anaconda" in sys.executable \
                or "condavir" in sys.executable:
            warnings.warn(
                "skipping test_flake8 because of Python 2 or " + sys.executable)
            return

        thi = os.path.abspath(os.path.dirname(__file__))
        src_ = os.path.normpath(os.path.join(thi, "..", "..", "src"))
        check_pep8(src_, fLOG=fLOG, extended=[("fLOG", _extended_refactoring)],
                   neg_filter=".*faq_python.py$")

    def test_flake8_test(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if sys.version_info[0] == 2 or "Anaconda" in sys.executable \
                or "condavir" in sys.executable:
            warnings.warn(
                "skipping test_flake8 because of Python 2 or " + sys.executable)
            return

        thi = os.path.abspath(os.path.dirname(__file__))
        test = os.path.normpath(os.path.join(thi, "..", ))
        check_pep8(test, fLOG=fLOG, neg_filter="temp_.*",
                   skip=["'src' imported but unused",
                         "'skip_' imported but unused",
                         "'skip__' imported but unused",
                         "'skip___' imported but unused",
                         ],
                   extended=[("fLOG", _extended_refactoring)],
                   max_line_length=320)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before># coding=utf-8
""""""
from __future__ import absolute_import, division, print_function, \
    unicode_literals

import json
from datetime import timedelta

import sqlalchemy as sa
from six import text_type

from abilian.core.extensions import db

__all__ = ['Setting']

class TransformerRegistry(object):

    def __init__(self):
        self.encoders = {}
        self.decoders = {}

    def encode(self, type_, value):
        # bytes is str for python2
        return self.encoders.get(type_, bytes)(value)

    def decode(self, type_, value):
        decoder = self.decoders.get(type_)
        if decoder is not None:
            value = decoder(value)
        return value

    def register(self, type_, encoder=None, decoder=None):
        assert type_
        assert any((encoder, decoder))
        if encoder:
            self.encoders[type_] = encoder
        if decoder:
            self.decoders[type_] = decoder

_transformers = TransformerRegistry()

class _EmptyValue(object):

    def __bool__(self):
        return False

    # Py2 compat
    __nonzero__ = __bool__

    def __repr__(self):
        return '<Empty Value>'

#: marker for emptyness, to distinguish from None
EmptyValue = _EmptyValue()

class Setting(db.Model):
    """A Setting is a very simple key/value object, key being a string
    identifier and the primary key.

    value must be stored as Unicode.
    """
    transformers = _transformers
    key = sa.Column('key', sa.String(length=1000), primary_key=True)

    #: Can be a string (Unicode), int, bool, json... or even a long dotted name
    #: if that's what you need. Type must be set before setting `value`
    _type = sa.Column('type', sa.String(length=1000), nullable=False)

    @property
    def type(self):
        return self._type

    @type.setter
    def type(self, type_):
        if not (
            type_ in self.transformers.encoders and
            type_ in self.transformers.decoders
        ):
            raise ValueError(
                'Invalid type "{}": not encoder and/or decoder registered'.
                format(type_),
            )
        self._type = type_

    _value = sa.Column('value', sa.Text())

    @property
    def value(self):
        if self._value is None:
            return EmptyValue
        return self.transformers.decode(self.type, self._value)

    @value.setter
    def value(self, value):
        assert self.type
        self._value = self.transformers.encode(self.type, value)

register = _transformers.register

def from_int(i):
    return "{}".format(i).encode()

register('int', from_int, int)

def from_bool(b):
    return 'true' if b else 'false'

def to_bool(s):
    return s == 'true'

register('bool', from_bool, to_bool)

def from_unicode(s):
    return text_type(s).encode('utf-8')

def to_unicode(s):
    return s.decode('utf-8')

register('string', from_unicode, to_unicode)
register('json', json.dumps, json.loads)  # FIXME: checks for dump/load?

def from_timedelta(s):
    return json.dumps(dict(days=s.days, seconds=s.seconds))

def to_timedelta(s):
    return timedelta(**json.loads(s))

register('timedelta', from_timedelta, to_timedelta)
<commit_msg>fix: typo in message.<commit_after># coding=utf-8
""""""
from __future__ import absolute_import, division, print_function, \
    unicode_literals

import json
from datetime import timedelta

import sqlalchemy as sa
from six import text_type

from abilian.core.extensions import db

__all__ = ['Setting']

class TransformerRegistry(object):

    def __init__(self):
        self.encoders = {}
        self.decoders = {}

    def encode(self, type_, value):
        # bytes is str for python2
        return self.encoders.get(type_, bytes)(value)

    def decode(self, type_, value):
        decoder = self.decoders.get(type_)
        if decoder is not None:
            value = decoder(value)
        return value

    def register(self, type_, encoder=None, decoder=None):
        assert type_
        assert any((encoder, decoder))
        if encoder:
            self.encoders[type_] = encoder
        if decoder:
            self.decoders[type_] = decoder

_transformers = TransformerRegistry()

class _EmptyValue(object):

    def __bool__(self):
        return False

    # Py2 compat
    __nonzero__ = __bool__

    def __repr__(self):
        return '<Empty Value>'

#: marker for emptyness, to distinguish from None
EmptyValue = _EmptyValue()

class Setting(db.Model):
    """A Setting is a very simple key/value object, key being a string
    identifier and the primary key.

    value must be stored as Unicode.
    """
    transformers = _transformers
    key = sa.Column('key', sa.String(length=1000), primary_key=True)

    #: Can be a string (Unicode), int, bool, json... or even a long dotted name
    #: if that's what you need. Type must be set before setting `value`
    _type = sa.Column('type', sa.String(length=1000), nullable=False)

    @property
    def type(self):
        return self._type

    @type.setter
    def type(self, type_):
        if not (
            type_ in self.transformers.encoders and
            type_ in self.transformers.decoders
        ):
            raise ValueError(
                'Invalid type "{}": no encoder and/or decoder registered'.
                format(type_),
            )
        self._type = type_

    _value = sa.Column('value', sa.Text())

    @property
    def value(self):
        if self._value is None:
            return EmptyValue
        return self.transformers.decode(self.type, self._value)

    @value.setter
    def value(self, value):
        assert self.type
        self._value = self.transformers.encode(self.type, value)

register = _transformers.register

def from_int(i):
    return "{}".format(i).encode()

register('int', from_int, int)

def from_bool(b):
    return 'true' if b else 'false'

def to_bool(s):
    return s == 'true'

register('bool', from_bool, to_bool)

def from_unicode(s):
    return text_type(s).encode('utf-8')

def to_unicode(s):
    return s.decode('utf-8')

register('string', from_unicode, to_unicode)
register('json', json.dumps, json.loads)  # FIXME: checks for dump/load?

def from_timedelta(s):
    return json.dumps(dict(days=s.days, seconds=s.seconds))

def to_timedelta(s):
    return timedelta(**json.loads(s))

register('timedelta', from_timedelta, to_timedelta)
<|endoftext|>


<commit_before>{
    "name": "Avalara Avatax Connector for Sales Orders",
    "version": "14.0.1.0.0",
    "author": "Open Source Integrators, Fabrice Henrion,"
    "Sodexis, Odoo Community Association (OCA)",
    "summary": "Sales Orders with automatic Tax application using Avatax",
    "license": "AGPL-3",
    "category": "Accounting",
    "website": "https://github.com/OCA/account-fiscal-rule",
    "depends": ["account_avatax", "sale"],
    "data": [
        "views/sale_order_view.xml",
        "views/partner_view.xml",
        "views/avalara_salestax_view.xml",
    ],
    "auto_install": True,
    "development_status": "Beta",
    "maintainers": ["dreispt"],
}
<commit_msg>account_avatax_sale 14.0.1.1.0<commit_after>{
    "name": "Avalara Avatax Connector for Sales Orders",
    "version": "14.0.1.1.0",
    "author": "Open Source Integrators, Fabrice Henrion,"
    "Sodexis, Odoo Community Association (OCA)",
    "summary": "Sales Orders with automatic Tax application using Avatax",
    "license": "AGPL-3",
    "category": "Accounting",
    "website": "https://github.com/OCA/account-fiscal-rule",
    "depends": ["account_avatax", "sale"],
    "data": [
        "views/sale_order_view.xml",
        "views/partner_view.xml",
        "views/avalara_salestax_view.xml",
    ],
    "auto_install": True,
    "development_status": "Beta",
    "maintainers": ["dreispt"],
}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import sys
import netsvc
import logging
import csv
from openerp.osv import osv, orm, fields
from datetime import datetime, timedelta
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT, DATETIME_FORMATS_MAP, float_compare)
import openerp.addons.decimal_precision as dp
from openerp.tools.translate import _
import pickle

_logger = logging.getLogger(__name__)

class EdiHistoryConfiguration(osv.osv):
    ''' Manage path for all company EDI that has active routes
    '''
    _name = 'edi.history.configuration'
    _description = 'EDI history configuration'

    _columns = {
        'name': fields.many2one('edi.company.importation',
            'EDI company', required=True),
        'code': fields.related('name', 'code',
            type='char', size=10, string='Code', store=True),
        'active': fields.boolean('Active'),
        'header': fields.integer('Header lines'),
        'delimiter': fields.char('Delimiter'),
        'verbose': fields.integer('Verbose every',
            help='Log an import event every X lines (0 = nothing)'),
        'history_path': fields.char('History path', size=100,
            help='Path of history files, use also like: "~/company/history"'),

        # TODO: maybe use invoice path if more than one file, else the file:
        'invoice_path': fields.char('Invoice path', size=100,
            help='Path of history files, use also like: "~/company/history"'),
        'invoice_file': fields.char('Invoice path', size=100,
            help='Path of invoice file, use: "~/company/account/invoice.csv"'),
        }

    # TODO set unique key for name
    _defaults = {
        'active': lambda *x: True,
        'delimiter': lambda *x: ';',
        'header': lambda *x: 0,
        'verbose': lambda *x: 100,
        }

class EdiHistoryCheck(osv.osv):
    ''' Manage path for all company EDI that has active routes
    '''

    _name = 'edi.history.check'
    _description = 'EDI history check'
    _order ='name,line_in,line_out'

    # -------------------------------------------------------------------------
    #                             Scheduled action
    # -------------------------------------------------------------------------
    def import_invoce_check(self, cr, uid, code, context=None):
        ''' Import procedure for check if order are correctly imported and
            match with invoice / delivery document in account
            self: instance object
            cr: cursor for database
            uid: user ID
            code: edi.company.importation for force
            delimiter
            context: parameter arguments passed
        '''
        # TODO code will be list if used in other company

        def load_order_from_history(order, history_list, order_files):
            ''' Function that load all files and create a dictionary with row
                key
            '''
            if order not in order_files:
                order_files[order] = {}
            for filename in history_list.get(order, []):
                # Read all files and update dict with row record:
                f = open(filename)
                for row in f:
                    file_type = row[10:16] # ORDERS or ORDCHG
                    update_type = row[16:19] # 003 ann, 001 mod. q.
                    line = row[2337:2342] # 5
                    article = row[2357:2368] # 11

                    if file_type == 'ORDERS':
                        line_type = 'original'
                    else: # ORDCHG
                        if update_type == '001':
                            line_type = 'update'
                        elif update_type == '003':
                            line_type = 'cancel'
                        else: # operation non in list
                            _logger.error(
                                'Update code not found: %s' % update_type)
                            line_type = 'error'
                    order_files[order][line] = (article, line_type)
            return
        # -----------------------------
        # Get configuration parameters:
        # -----------------------------
        config_pool = self.pool.get('edi.history.configuration')
        config_ids = config_pool.search(cr, uid, [
            ('code', '=', code)], context=context)
        if not config_ids:
            return False
        config_proxy = config_pool.browse(
            cr, uid, config_ids, context=context)[0]
        input_folder = config_proxy.history_path
        input_invoice = config_proxy.invoice_file

        # -------------------
        # Read files history:
        # -------------------
        # Save in dict for future access
        history_files = {} # list of file (key=order, value=filename)
        order_files = {} # record in file (key=order, value {row: (art, state)}
        for root, directories, files in os.walk(input_folder):
            for filename in files:
                filepath = os.path.join(root, filename)
                f = open(filepath, 'rb')
                line = f.read()
                f.close()
                order = line[19:29]
                if order not in history_files:
                    history_files[order] = []
                #os.path.getmtime(filepath)
                history_files[order].append(filepath)

        # ---------------------
        # Start import invoice:
        # ---------------------
        # Read all lines and save only duplicated
        invoice_row = {}
        i = -config_proxy.header
        old_order = False
        for invoice in csv.reader(
                open(input_invoice, 'rb'),
                delimiter=str(config_proxy.delimiter)):
            i += 1
            if i <= 0:
                _logger.info('Jump header lines')
                continue

            # Mapping fields:
            doc_type = invoice[0].strip()
            number = invoice[1].strip()
            order = invoice[2].strip() # header
            article = invoice[3].strip()
            order_detail = invoice[4].strip()
            line = invoice[5].strip()

            # Load order if not present in database:
            if order not in order_files:
                load_order_from_history(
                    order, history_files, order_files)

            date = {
                'name': order, # to search in history
                'name_detail': order_detail,
                'line_in': line, # TODO load from history
                'line_out': line,
                'product_code_in': article, # TODO load from history
                'product_code_out': article,
                'document_out': number,
                'document_type': doc_type,
                'state': state,
                }

            if not order:
                state = 'no_order' # and no history search
                date['state'] = 'duplicated'
                self.create(cr, uid, date, context=context)
                continue
            elif order != order_detail:
                state = 'order' # difference between line and header
            else:
                state = 'ok'

            if order not in invoice_row:
                invoice_row[order] = {}

            if line in invoice_row[order]:
                date['state'] = 'duplicated'
                _logger.error('Line duplicated: %s-%s' % (order, line))
                # ID non saved (raise only diplication)
                continue # Jump line

            # Save article:
            invoice_row[order][line] = (
                self.create(cr, uid, date, context=context),
                article,
                ) # ID, Article

        return True

    _columns = {
        'name': fields.char('Order name', size=25,
            help='Order ID of company'),
        'name_detail': fields.char('Order detail', size=25,
            help='Order ID in accounting detail'),
        'line_in': fields.char('Line in', size=5, help='Order line'),
        'line_out': fields.char('Line out', size=5,
            help='Delivery of invoice line'),
        'product_code_in': fields.char('Product in', size=18,
            help='Order product in'),
        'product_code_out': fields.char('Product out', size=18,
            help='Invoice or delivery product out'),
        'document_out': fields.char('Document out', size=20,
            help='Number of document, delivery or invoice, out'),
        'document_type': fields.selection([
            ('BC', 'Document of transport'),
            ('FT', 'Invoice'),
            ], 'Document type', help='Document out type'),
        'state': fields.selection([
            ('no_order', 'Order not present'),
            ('order', 'Order dont\'t match'),
            ('sequence', 'Sequence error'),
            ('duplicated', 'Row duplicated'),
            ('only_in', 'Not imported'),
            ('only_out', 'Extra line'),
            ('article', 'Article error'),
            ('ok', 'Correct'),
            ], 'State', help='Error state'),
        }
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>Case: row duplicated OK<commit_after># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import sys
import netsvc
import logging
import csv
from openerp.osv import osv, orm, fields
from datetime import datetime, timedelta
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT, DATETIME_FORMATS_MAP, float_compare)
import openerp.addons.decimal_precision as dp
from openerp.tools.translate import _
import pickle

_logger = logging.getLogger(__name__)

class EdiHistoryConfiguration(osv.osv):
    ''' Manage path for all company EDI that has active routes
    '''
    _name = 'edi.history.configuration'
    _description = 'EDI history configuration'

    _columns = {
        'name': fields.many2one('edi.company.importation',
            'EDI company', required=True),
        'code': fields.related('name', 'code',
            type='char', size=10, string='Code', store=True),
        'active': fields.boolean('Active'),
        'header': fields.integer('Header lines'),
        'delimiter': fields.char('Delimiter'),
        'verbose': fields.integer('Verbose every',
            help='Log an import event every X lines (0 = nothing)'),
        'history_path': fields.char('History path', size=100,
            help='Path of history files, use also like: "~/company/history"'),

        # TODO: maybe use invoice path if more than one file, else the file:
        'invoice_path': fields.char('Invoice path', size=100,
            help='Path of history files, use also like: "~/company/history"'),
        'invoice_file': fields.char('Invoice path', size=100,
            help='Path of invoice file, use: "~/company/account/invoice.csv"'),
        }

    # TODO set unique key for name
    _defaults = {
        'active': lambda *x: True,
        'delimiter': lambda *x: ';',
        'header': lambda *x: 0,
        'verbose': lambda *x: 100,
        }

class EdiHistoryCheck(osv.osv):
    ''' Manage path for all company EDI that has active routes
    '''

    _name = 'edi.history.check'
    _description = 'EDI history check'
    _order ='name,line_in,line_out'

    # -------------------------------------------------------------------------
    #                             Scheduled action
    # -------------------------------------------------------------------------
    def import_invoce_check(self, cr, uid, code, context=None):
        ''' Import procedure for check if order are correctly imported and
            match with invoice / delivery document in account
            self: instance object
            cr: cursor for database
            uid: user ID
            code: edi.company.importation for force
            delimiter
            context: parameter arguments passed
        '''
        # TODO code will be list if used in other company

        def load_order_from_history(order, history_list, order_files):
            ''' Function that load all files and create a dictionary with row
                key
            '''
            if order not in order_files:
                order_files[order] = {}
            for filename in history_list.get(order, []):
                # Read all files and update dict with row record:
                f = open(filename)
                for row in f:
                    file_type = row[10:16] # ORDERS or ORDCHG
                    update_type = row[16:19] # 003 ann, 001 mod. q.
                    line = row[2337:2342] # 5
                    article = row[2357:2368] # 11

                    if file_type == 'ORDERS':
                        line_type = 'original'
                    else: # ORDCHG
                        if update_type == '001':
                            line_type = 'update'
                        elif update_type == '003':
                            line_type = 'cancel'
                        else: # operation non in list
                            _logger.error(
                                'Update code not found: %s' % update_type)
                            line_type = 'error'
                    order_files[order][line] = (article, line_type)
            return

        # -----------------------------
        # Get configuration parameters:
        # -----------------------------
        config_pool = self.pool.get('edi.history.configuration')
        config_ids = config_pool.search(cr, uid, [
            ('code', '=', code)], context=context)
        if not config_ids:
            return False
        config_proxy = config_pool.browse(
            cr, uid, config_ids, context=context)[0]
        input_folder = config_proxy.history_path
        input_invoice = config_proxy.invoice_file

        # ---------------
        # Clean database:
        # ---------------
        remove_ids = self.search(cr, uid, [], context=context)
        self.unlink(cr, uid, remove_ids, context=context)

        # -------------------
        # Read files history:
        # -------------------
        # Save in dict for future access
        history_files = {} # list of file (key=order, value=filename)
        order_files = {} # record in file (key=order, value {row: (art, state)}
        for root, directories, files in os.walk(input_folder):
            for filename in files:
                filepath = os.path.join(root, filename)
                f = open(filepath, 'rb')
                line = f.read()
                f.close()
                order = line[19:29]
                if order not in history_files:
                    history_files[order] = []
                #os.path.getmtime(filepath)
                history_files[order].append(filepath)

        # ---------------------
        # Start import invoice:
        # ---------------------
        # Read all lines and save only duplicated
        invoice_row = {}
        i = -config_proxy.header
        old_order = False
        for invoice in csv.reader(
                open(input_invoice, 'rb'),
                delimiter=str(config_proxy.delimiter)):
            i += 1
            if i <= 0:
                _logger.info('Jump header lines')
                continue

            # Mapping fields:
            doc_type = invoice[0].strip()
            number = invoice[1].strip()
            order = invoice[2].strip() # header
            article = invoice[3].strip()
            order_detail = invoice[4].strip()
            line = invoice[5].strip()

            # Load order if not present in database:
            if order not in order_files:
                load_order_from_history(
                    order, history_files, order_files)

            date = {
                'name': order, # to search in history
                'name_detail': order_detail,
                'line_in': line, # TODO load from history
                'line_out': line,
                'product_code_in': article, # TODO load from history
                'product_code_out': article,
                'document_out': number,
                'document_type': doc_type,
                }

            # -------------
            # State manage:
            # -------------
            if not order:
                date['state'] = 'no_order'
                self.create(cr, uid, date, context=context)
                continue
            elif order != order_detail:
                date['state'] = 'order'
            else:
                date['state'] = 'ok'

            if order not in invoice_row:
                invoice_row[order] = {}

            if line in invoice_row[order]:
                date['state'] = 'duplicated'
                self.create(cr, uid, date, context=context)
                continue # Jump line

            # Save article:
            invoice_row[order][line] = (
                self.create(cr, uid, date, context=context),
                article,
                ) # ID, Article

        return True

    _columns = {
        'name': fields.char('Order name', size=25,
            help='Order ID of company'),
        'name_detail': fields.char('Order detail', size=25,
            help='Order ID in accounting detail'),
        'line_in': fields.char('Line in', size=5, help='Order line'),
        'line_out': fields.char('Line out', size=5,
            help='Delivery of invoice line'),
        'product_code_in': fields.char('Product in', size=18,
            help='Order product in'),
        'product_code_out': fields.char('Product out', size=18,
            help='Invoice or delivery product out'),
        'document_out': fields.char('Document out', size=20,
            help='Number of document, delivery or invoice, out'),
        'document_type': fields.selection([
            ('BC', 'Document of transport'),
            ('FT', 'Invoice'),
            ], 'Document type', help='Document out type'),
        'state': fields.selection([
            ('no_order', 'Order not present'),
            ('order', 'Order dont\'t match'),
            ('sequence', 'Sequence error'),
            ('duplicated', 'Row duplicated'),
            ('only_in', 'Not imported'),
            ('only_out', 'Extra line'),
            ('article', 'Article error'),
            ('ok', 'Correct'),
            ], 'State', help='Error state'),
        }
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
###############################################################################
#
# ODOO (ex OpenERP)
# Open Source Management Solution
# Copyright (C) 2001-2015 Micronaet S.r.l. (<https://micronaet.com>)
# Developer: Nicola Riolini @thebrush (<https://it.linkedin.com/in/thebrush>)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

import os
import sys
import logging
import openerp
import time
import openerp.netsvc as netsvc
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, osv, expression, orm
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from openerp import SUPERUSER_ID, api
from openerp import tools
from openerp.tools.translate import _
from openerp.tools.float_utils import float_round as round
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)

_logger = logging.getLogger(__name__)

class EtlLogImportation(orm.Model):
    ''' Importation log element
    '''
    _name = 'etl.log.importation'
    _description = 'ETL Importation log'
    _order = 'datetime desc'

    def etl_log_event(self, cr, uid, name, filename, note, error,
            context=None):
        """ Log ETL data
        """
        date_file = 'Data file: %s' % time.ctime(
            os.path.getctime(filename))
        _logger.warning('Log %s with %s [%s]' % (name, date_file, filename))
        log_ids = self.search(cr, uid, [
            ('name', '=', name),
            ], context=context)
        data = {
            'name': name,
            'note': note,
            'error': error,
            'date_file': date_file,
        }
        if log_ids:
            self.write(cr, uid, log_ids, data, context=context)
        else:
            self.create(cr, uid, data, context=context)

    _columns = {
        'name': fields.char('Sorgente ETL', size=90),
        'datetime': fields.datetime('Data creazione'),
        'date_file': fields.char('Data file', size=90),
        'note': fields.char('Note'),
        'error': fields.char('Error'),

        # File information:
        'filename': fields.char('Filename', size=80),
        }

    _defaults = {
        'datetime': lambda *x: datetime.now().strftime(
            DEFAULT_SERVER_DATETIME_FORMAT),
        }
<commit_msg>generic bugfix (fast message)<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
###############################################################################
#
# ODOO (ex OpenERP)
# Open Source Management Solution
# Copyright (C) 2001-2015 Micronaet S.r.l. (<https://micronaet.com>)
# Developer: Nicola Riolini @thebrush (<https://it.linkedin.com/in/thebrush>)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

import os
import sys
import logging
import openerp
import time
import openerp.netsvc as netsvc
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, osv, expression, orm
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from openerp import SUPERUSER_ID, api
from openerp import tools
from openerp.tools.translate import _
from openerp.tools.float_utils import float_round as round
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)

_logger = logging.getLogger(__name__)

class EtlLogImportation(orm.Model):
    ''' Importation log element
    '''
    _name = 'etl.log.importation'
    _description = 'ETL Importation log'
    _order = 'datetime desc'

    def etl_log_event(self, cr, uid, name, filename, note, error,
            context=None):
        """ Log ETL data
        """
        date_file = 'Data file: %s' % time.ctime(
            os.path.getctime(filename))
        _logger.warning('Log %s with %s [%s]' % (name, date_file, filename))
        log_ids = self.search(cr, uid, [
            ('name', '=', name),
            ], context=context)
        data = {
            'name': name,
            'filename': filename,
            'note': note,
            'error': error,
            'date_file': date_file,
        }
        if log_ids:
            self.write(cr, uid, log_ids, data, context=context)
        else:
            self.create(cr, uid, data, context=context)

    _columns = {
        'name': fields.char('Sorgente ETL', size=90),
        'datetime': fields.datetime('Data creazione'),
        'date_file': fields.char('Data file', size=90),
        'note': fields.char('Note'),
        'error': fields.char('Error'),

        # File information:
        'filename': fields.char('Filename', size=80),
        }

    _defaults = {
        'datetime': lambda *x: datetime.now().strftime(
            DEFAULT_SERVER_DATETIME_FORMAT),
        }
<|endoftext|>


<commit_before># Copyright 2015,  Tobit Raff,  A10 Networks.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
from __future__ import absolute_import
from __future__ import unicode_literals
import six

from acos_client import errors as acos_errors
from acos_client.v30 import base

class BaseSSL(base.BaseV30):

    def get(self, name, **kwargs):
        return self._get(self.url_prefix + name, **kwargs)

    def exists(self, name):
        try:
            self.get(name)
            return True
        except acos_errors.NotFound:
            return False

    def _set(self, name, cert="", key="", passphrase="", update=False,
             cipher_template=None, **kwargs):
        # Unimplemented options:
        # encrypted, session_ticket_enable, version, forward_proxy_enable,
        # close_notify, session_cache_size, session_cache_timeout,
        # server_certificate_error, cipher_without_prio_list,

        params = {}

        if cipher_template:
            params = {
                "cipher": {
                    "name": name,
                    "cipher-cfg": cipher_template
                }
            }

        else:
            obj_params = {
                "name": name,
                "cert-str": cert,
                "key-str": key,
                self.passphrase: passphrase,
                # Unimplemented options:
                # "encrypted": encrypted,
                # "session-ticket-enable": session_ticket_enable,
                # "version": version,
                # "forward-proxy-enable": forward_proxy_enable,
                # "close-notify": close_notify,
                # "session-cache-size": session_cache_size,
                # "session-cache-timeout": session_cache_timeout,
                # "server-certificate-error": server_certificate_error,
                # "cipher-without-prio-list": cipher_without_prio_list,
                # "ca-certs": ca_certs,
            }

            params = {'%s-ssl' % self.prefix: {}}
            for key, val in six.iteritems(obj_params):
                # Filter out invalid, or unset keys
                if val != "":
                    params['%s-ssl' % self.prefix][key] = val

            if not update:
                name = ''

        self._post(self.url_prefix + name, params, **kwargs)

    def create(self, name, cert="", key="", passphrase="", cipher_template=None, **kwargs):
        if self.exists(name):
            raise acos_errors.Exists

        self._set(name, cert, key, passphrase, cipher_template=cipher_template, **kwargs)

    def update(self, name, cert="", key="", passphrase="", **kwargs):
        self._set(name, cert, key, passphrase, update=True, **kwargs)

    def delete(self, name, **kwargs):
        self._delete(self.url_prefix + name, **kwargs)

class ClientSSL(BaseSSL):

    url_prefix = '/slb/template/client-ssl/'
    prefix = 'client'
    passphrase = 'key-passphrase'

class ServerSSL(BaseSSL):

    url_prefix = '/slb/template/server-ssl/'
    prefix = 'server'
    passphrase = 'passphrase'

class SSLCipher(BaseSSL):

    url_prefix = '/slb/template/cipher/'
    passphrase = None
    prefix = None
<commit_msg>Updated ssl template payload to reflect changes in AXAPI<commit_after># Copyright 2015,  Tobit Raff,  A10 Networks.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
from __future__ import absolute_import
from __future__ import unicode_literals
import six

from acos_client import errors as acos_errors
from acos_client.v30 import base

class BaseSSL(base.BaseV30):

    def get(self, name, **kwargs):
        return self._get(self.url_prefix + name, **kwargs)

    def exists(self, name):
        try:
            self.get(name)
            return True
        except acos_errors.NotFound:
            return False

    def _set(self, name, cert="", key="", passphrase="", update=False,
             cipher_template=None, **kwargs):
        # Unimplemented options:
        # encrypted, session_ticket_enable, version, forward_proxy_enable,
        # close_notify, session_cache_size, session_cache_timeout,
        # server_certificate_error, cipher_without_prio_list,

        params = {}

        if cipher_template:
            params = {
                "cipher": {
                    "name": name,
                    "cipher-cfg": cipher_template
                }
            }

        else:
            obj_params = {
                "name": name,
                "cert": cert,
                "key": key,
                self.passphrase: passphrase,
                # Unimplemented options:
                # "encrypted": encrypted,
                # "session-ticket-enable": session_ticket_enable,
                # "version": version,
                # "forward-proxy-enable": forward_proxy_enable,
                # "close-notify": close_notify,
                # "session-cache-size": session_cache_size,
                # "session-cache-timeout": session_cache_timeout,
                # "server-certificate-error": server_certificate_error,
                # "cipher-without-prio-list": cipher_without_prio_list,
                # "ca-certs": ca_certs,
            }

            params = {'%s-ssl' % self.prefix: {}}
            for key, val in six.iteritems(obj_params):
                # Filter out invalid, or unset keys
                if val != "":
                    params['%s-ssl' % self.prefix][key] = val

            if not update:
                name = ''

        self._post(self.url_prefix + name, params, **kwargs)

    def create(self, name, cert="", key="", passphrase="", cipher_template=None, **kwargs):
        if self.exists(name):
            raise acos_errors.Exists

        self._set(name, cert, key, passphrase, cipher_template=cipher_template, **kwargs)

    def update(self, name, cert="", key="", passphrase="", **kwargs):
        self._set(name, cert, key, passphrase, update=True, **kwargs)

    def delete(self, name, **kwargs):
        self._delete(self.url_prefix + name, **kwargs)

class ClientSSL(BaseSSL):

    url_prefix = '/slb/template/client-ssl/'
    prefix = 'client'
    passphrase = 'key-passphrase'

class ServerSSL(BaseSSL):

    url_prefix = '/slb/template/server-ssl/'
    prefix = 'server'
    passphrase = 'passphrase'

class SSLCipher(BaseSSL):

    url_prefix = '/slb/template/cipher/'
    passphrase = None
    prefix = None
<|endoftext|>


<commit_before>"""RabbitMQ Adapter between FAUCET Events and RabbitMQ"""

# Copyright (C) 2013 Nippon Telegraph and Telephone Corporation.
# Copyright (C) 2015 Brad Cowie, Christopher Lorier and Joe Stringer.
# Copyright (C) 2015 Research and Education Advanced Network New Zealand Ltd.
# Copyright (C) 2015--2019 The Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import errno
import os
import select
import socket
import sys
import time

import pika

def get_sys_prefix():
    """This was copied from faucet.valve_util.
    Returns an additional prefix for log and configuration files when used in
    a virtual environment"""

    # Find the appropriate prefix for config and log file default locations
    # in case Faucet is run in a virtual environment. virtualenv marks the
    # original path in sys.real_prefix. If this value exists, and is
    # different from sys.prefix, then we are most likely running in a
    # virtualenv. Also check for Py3.3+ pyvenv.
    sysprefix = ''
    if (getattr(sys, 'real_prefix', sys.prefix) != sys.prefix or
            getattr(sys, 'base_prefix', sys.prefix) != sys.prefix):
        sysprefix = sys.prefix

    return sysprefix

class RabbitAdapter:
    """A RabbitMQ adapter to get events from the FAUCET Unix socket and send
    them as messages to a RabbitMQ server
    """

    def __init__(self):
        super(RabbitAdapter, self).__init__()

        # get environment variables and set defaults
        self.channel = None
        self.sock = None
        self.event_sock = os.getenv('FAUCET_EVENT_SOCK', '0')
        self.host = os.getenv('FA_RABBIT_HOST', '')
        self.port = os.getenv('FA_RABBIT_PORT')
        if not self.port:
            self.port = 5672
        else:
            try:
                self.port = int(self.port)
            except ValueError:
                self.port = 5672
        self.exchange = os.getenv('FA_RABBIT_EXCHANGE')
        if not self.exchange:
            self.exchange = 'topic_recs'
        self.exchange_type = os.getenv('FA_RABBIT_EXCHANGE_TYPE')
        if not self.exchange_type:
            self.exchange_type = 'topic'
        self.routing_key = os.getenv('FA_RABBIT_ROUTING_KEY', 'FAUCET.Event')
        if not self.routing_key:
            self.routing_key = 'FAUCET.Event'

    def rabbit_conn(self):
        """Make connection to rabbit to send events"""
        # check if a rabbit host was specified
        if not self.host:
            print('Not connecting to any RabbitMQ, host is None.')
            return False

        # create connection to rabbit
        params = pika.ConnectionParameters(host=self.host,
                                           port=self.port,
                                           heartbeat=600,
                                           blocked_connection_timeout=300)
        try:
            self.channel = pika.BlockingConnection(params).channel()
            self.channel.exchange_declare(exchange=self.exchange,
                                          exchange_type=self.exchange_type)
        except (pika.exceptions.AMQPError, socket.gaierror, OSError) as err:
            print("Unable to connect to RabbitMQ at %s:%s because: %s" %
                  (self.host, self.port, err))
            self.channel = None
            return False

        print("Connected to RabbitMQ at %s:%s" % (self.host, self.port))
        return True

    def socket_conn(self):
        """Make connection to sock to receive events"""
        # check if socket events are enabled
        if self.event_sock == '0':
            print('Not connecting to any socket, FAUCET_EVENT_SOCK is none.')
            return False
        if self.event_sock == '1':
            self.event_sock = get_sys_prefix() + '/var/run/faucet/faucet.sock'
        # otherwise it's a path

        # create connection to unix socket
        try:
            self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
            self.sock.connect(self.event_sock)
            self.sock.setblocking(False)
        except socket.error as err:
            print("Failed to connect to the socket because: %s" % err)
            return False

        print("Connected to the socket at %s" % self.event_sock)
        return True

    def send_rabbit(self, buffer):
        """Send events in buffer to Rabbit."""
        buffers = buffer.strip().split(b'\n')
        if not buffers:
            return True

        if not self.channel:
            if not self.rabbit_conn():
                return False

        try:
            for buff in buffers:
                self.channel.basic_publish(exchange=self.exchange,
                                           routing_key=self.routing_key,
                                           body=buff,
                                           properties=pika.BasicProperties(
                                               delivery_mode=2))
        except pika.exceptions.AMQPError as err:
            print("Unable to send event to RabbitMQ because: %s" % err)
            self.channel = None
            return False

        return True

    def main(self):
        """Make connections to sock and rabbit and receive messages from sock
        to sent to rabbit
        """
        buffer = b''
        while True:
            if self.socket_conn():
                socket_ok = True
                while socket_ok:
                    read_ready, _, _ = select.select([self.sock], [], [])
                    if self.sock in read_ready:
                        continue_recv = True
                        while continue_recv:
                            try:
                                buffer += self.sock.recv(1024)
                            except socket.error as err:
                                if err.errno != errno.EWOULDBLOCK:
                                    socket_ok = False
                                continue_recv = False
                    if self.send_rabbit(buffer):
                        buffer = b''
                    sys.stdout.flush()
                self.sock.close()
            time.sleep(1)

if __name__ == "__main__":  # pragma: no cover
    RABBIT_ADAPTER = RabbitAdapter()
    RABBIT_ADAPTER.main()
<commit_msg>Revert "Adapter should retry socket connection (e.g if FAUCET restarted)."<commit_after>"""RabbitMQ Adapter between FAUCET Events and RabbitMQ"""

# Copyright (C) 2013 Nippon Telegraph and Telephone Corporation.
# Copyright (C) 2015 Brad Cowie, Christopher Lorier and Joe Stringer.
# Copyright (C) 2015 Research and Education Advanced Network New Zealand Ltd.
# Copyright (C) 2015--2019 The Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import errno
import os
import select
import socket
import sys

import pika

def get_sys_prefix():
    """This was copied from faucet.valve_util.
    Returns an additional prefix for log and configuration files when used in
    a virtual environment"""

    # Find the appropriate prefix for config and log file default locations
    # in case Faucet is run in a virtual environment. virtualenv marks the
    # original path in sys.real_prefix. If this value exists, and is
    # different from sys.prefix, then we are most likely running in a
    # virtualenv. Also check for Py3.3+ pyvenv.
    sysprefix = ''
    if (getattr(sys, 'real_prefix', sys.prefix) != sys.prefix or
            getattr(sys, 'base_prefix', sys.prefix) != sys.prefix):
        sysprefix = sys.prefix

    return sysprefix

class RabbitAdapter:
    """A RabbitMQ adapter to get events from the FAUCET Unix socket and send
    them as messages to a RabbitMQ server
    """

    def __init__(self):
        super(RabbitAdapter, self).__init__()

        # setup socket
        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)

        # get environment variables and set defaults
        self.channel = None
        self.event_sock = os.getenv('FAUCET_EVENT_SOCK', '0')
        self.host = os.getenv('FA_RABBIT_HOST', '')
        self.port = os.getenv('FA_RABBIT_PORT')
        if not self.port:
            self.port = 5672
        else:
            try:
                self.port = int(self.port)
            except ValueError:
                self.port = 5672
        self.exchange = os.getenv('FA_RABBIT_EXCHANGE')
        if not self.exchange:
            self.exchange = 'topic_recs'
        self.exchange_type = os.getenv('FA_RABBIT_EXCHANGE_TYPE')
        if not self.exchange_type:
            self.exchange_type = 'topic'
        self.routing_key = os.getenv('FA_RABBIT_ROUTING_KEY', 'FAUCET.Event')
        if not self.routing_key:
            self.routing_key = 'FAUCET.Event'

    def rabbit_conn(self):
        """Make connection to rabbit to send events"""
        # check if a rabbit host was specified
        if not self.host:
            print('Not connecting to any RabbitMQ, host is None.')
            return False

        # create connection to rabbit
        params = pika.ConnectionParameters(host=self.host,
                                           port=self.port,
                                           heartbeat=600,
                                           blocked_connection_timeout=300)
        try:
            self.channel = pika.BlockingConnection(params).channel()
            self.channel.exchange_declare(exchange=self.exchange,
                                          exchange_type=self.exchange_type)
        except pika.exceptions.AMQPError as err:
            print("Unable to connect to RabbitMQ at %s:%s because: %s" %
                  (self.host, self.port, err))
            return False
        except (socket.gaierror, OSError) as err:
            print("Unable to connect to RabbitMQ at %s:%s because: %s" %
                  (self.host, self.port, err))
            return False
        print("Connected to RabbitMQ at %s:%s" % (self.host, self.port))
        return True

    def socket_conn(self):
        """Make connection to sock to receive events"""
        # check if socket events are enabled
        if self.event_sock == '0':
            print('Not connecting to any socket, FA_EVENT_SOCK is none.')
            return False
        if self.event_sock == '1':
            self.event_sock = get_sys_prefix() + '/var/run/faucet/faucet.sock'
        # otherwise it's a path

        # create connection to unix socket
        try:
            self.sock.connect(self.event_sock)
        except socket.error as err:
            print("Failed to connect to the socket because: %s" % err)
            return False
        print("Connected to the socket at %s" % self.event_sock)
        return True

    def main(self):
        """Make connections to sock and rabbit and receive messages from sock
        to sent to rabbit
        """
        # ensure connections to the socket and rabbit before getting messages
        if self.rabbit_conn() and self.socket_conn():
            # get events from socket
            self.sock.setblocking(False)
            recv_data = True
            buffer = b''
            while recv_data:
                if not buffer:
                    read_ready, _, _ = select.select([self.sock], [], [])
                    if self.sock in read_ready:
                        continue_recv = True
                        while continue_recv:
                            try:
                                buffer += self.sock.recv(1024)
                            except socket.error as err:
                                if err.errno != errno.EWOULDBLOCK:
                                    recv_data = False
                                continue_recv = False
                # send events to rabbit
                try:
                    buffers = buffer.strip().split(b'\n')
                    for buff in buffers:
                        self.channel.basic_publish(exchange=self.exchange,
                                                   routing_key=self.routing_key,
                                                   body=buff,
                                                   properties=pika.BasicProperties(
                                                       delivery_mode=2,
                                                   ))
                    buffer = b''
                except pika.exceptions.AMQPError as err:
                    print("Unable to send event to RabbitMQ because: %s" % err)
                    print("The following event will be retried: %r" % buffer)
                    self.rabbit_conn()
                sys.stdout.flush()
            self.sock.close()

if __name__ == "__main__":  # pragma: no cover
    RABBIT_ADAPTER = RabbitAdapter()
    RABBIT_ADAPTER.main()
<|endoftext|>


<commit_before>"""Solvers for the linear and quadratic programs in active subspaces."""
import numpy as np
import warnings
from scipy.optimize import linprog, minimize

# checking to see if system has gurobi
try:
    HAS_GUROBI = True
    import gurobipy as gpy
except ImportError, e:
    HAS_GUROBI = False
    pass

# string constants for QP solver names
solver_SCIPY = 'SCIPY'
solver_GUROBI = 'GUROBI'

class QPSolver():
    """
    A class for solving linear and quadratic programs.

    Attributes
    ----------
    solver : str
        `solver` identifies which linear program software to use.

    Notes
    -----
    The class checks to see if Gurobi is present. If it is, it uses Gurobi to
    solve the linear and quadratic programs. Otherwise, it uses scipy
    implementations to solve the linear and quadratic programs.
    """
    solver = None

    def __init__(self, solver='GUROBI'):
        """
        Initialize a QPSolver.

        Parameters
        ----------
        solver : str, optional
            `solver` identifies which linear program software to use. Default is
            'GUROBI'. Another option is 'SCIPY'.
        """

        if solver==solver_GUROBI and HAS_GUROBI:
            self.solver = solver_GUROBI
        elif solver=='SCIPY':
            self.solver = solver_SCIPY
        else:
            warnings.warn('QP solver %s is not available. Using scipy optimization package.' % solver)
            self.solver = solver_SCIPY

    def linear_program_eq(self, c, A, b, lb, ub):
        """
        Solves an equality constrained linear program with variable bounds.

        Parameters
        ----------
        c : ndarray
            `c` is an ndarray of size m-by-1 for the linear objective function.
        A : ndarray
            `A` is an ndarray of size M-by-m that contains the coefficients
            of the linear equality constraints.
        b : ndarray
            `b` is an ndarray of size M-by-1 that is the right hand side of the
            equality constraints.
        lb : ndarray
            `lb` is an ndarray of size m-by-1 that contains the lower bounds
            on the variables.
        ub : ndarray
            `ub` is an ndarray of size m-by-1 that contains the upper bounds
            on the variables.

        Returns
        -------
        x : ndarray
            An ndarray of size m-by-1 that is the minimizer of the linear
            program.

        Notes
        -----
        This method returns the minimizer of the following linear program.

        minimize  c^T x
        subject to  A x = b
                    lb <= x <= ub
        """
        if self.solver == solver_SCIPY:
            c = c.reshape((c.size,))
            b = b.reshape((b.size,))
            return _scipy_linear_program_eq(c, A, b, lb, ub)
        elif self.solver == solver_GUROBI:
            return _gurobi_linear_program_eq(c, A, b, lb, ub)
        else:
            raise ValueError('QP solver %s not available' % self.solver)

    def linear_program_ineq(self, c, A, b):
        """
        Solves an inequality constrained linear program.

        Parameters
        ----------
        c : ndarray
            `c` is an ndarray of size m-by-1 for the linear objective function.
        A : ndarray
            `A` is an ndarray of size M-by-m that contains the coefficients
            of the linear equality constraints.
        b : ndarray
            `b` is an ndarray of size M-by-1 that is the right hand side of the
            equality constraints.

        Returns
        -------
        x : ndarray
            An ndarray of size m-by-1 that is the minimizer of the linear
            program.

        Notes
        -----
        This method returns the minimizer of the following linear program.

        minimize  c^T x
        subject to  A x >= b
        """

        if self.solver == solver_SCIPY:
            c = c.reshape((c.size,))
            b = b.reshape((b.size,))
            return _scipy_linear_program_ineq(c, A, b)
        elif self.solver == solver_GUROBI:
            return _gurobi_linear_program_ineq(c, A, b)
        else:
            raise ValueError('QP solver %s not available' % self.solver)

    def quadratic_program_bnd(self, c, Q, lb, ub):
        """
        Solves a quadratic program with variable bounds.

        Parameters
        ----------
        c : ndarray
            `c` is an ndarray of size m-by-1 that contains the coefficients of
            the linear term in the objective function.
        Q : ndarray
            `Q` is an ndarray of size m-by-m that contains the coefficients
            of the quadratic term in the objective function.
        lb : ndarray
            `lb` is an ndarray of size m-by-1 that contains the lower bounds
            on the variables.
        ub : ndarray
            `ub` is an ndarray of size m-by-1 that contains the upper bounds
            on the variables.

        Returns
        -------
        x : ndarray
            An ndarray of size m-by-1 that is the minimizer of the quadratic
            program.

        Notes
        -----
        This method returns the minimizer of the following linear program.

        minimize  c^T x + x^T Q x
        subject to  lb <= x <= ub
        """

        if self.solver == solver_SCIPY:
            return _scipy_quadratic_program_bnd(c, Q, lb, ub)
        elif self.solver == solver_GUROBI:
            return _gurobi_quadratic_program_bnd(c, Q, lb, ub)
        else:
            raise ValueError('QP solver %s not available' % self.solver)

    def quadratic_program_ineq(self, c, Q, A, b):
        """
        Solves an inequality constrained quadratic program with variable bounds.

        Parameters
        ----------
        c : ndarray
            `c` is an ndarray of size m-by-1 that contains the coefficients of
            the linear term in the objective function.
        Q : ndarray
            `Q` is an ndarray of size m-by-m that contains the coefficients
            of the quadratic term in the objective function.
        A : ndarray
            `A` is an ndarray of size M-by-m that contains the coefficients
            of the linear equality constraints.
        b : ndarray
            `b` is an ndarray of size M-by-1 that is the right hand side of the
            equality constraints.

        Returns
        -------
        x : ndarray
            An ndarray of size m-by-1 that is the minimizer of the quadratic
            program.

        Notes
        -----
        This method returns the minimizer of the following linear program.

        minimize  c^T x + x^T Q x
        subject to  A x >= b
        """

        if self.solver == solver_SCIPY:
            b = b.reshape((b.size,))
            return _scipy_quadratic_program_ineq(c, Q, A, b)
        elif self.solver == solver_GUROBI:
            return _gurobi_quadratic_program_ineq(c, Q, A, b)
        else:
            raise ValueError('QP solver %s not available' % self.solver)

def _scipy_linear_program_eq(c, A, b, lb, ub):

    # make bounds
    bounds = []
    for i in range(lb.size):
        bounds.append((lb[i,0],ub[i,0]))

    res = linprog(c, A_eq=A, b_eq=b, bounds=bounds, options={"disp": False})
    if res.success:
        return res.x.reshape((c.size,1))
    else:
        np.savez('bad_scipy_lp_eq_{:010d}'.format(np.random.randint(int(1e9))),
                c=c, A=A, b=b, lb=lb, ub=ub, res=res)
        raise Exception('Scipy did not solve the LP. Blame Scipy.')
        return None

def _scipy_linear_program_ineq(c, A, b):

    A_ub, b_ub = -A, -b
    res = linprog(c, A_ub=A_ub, b_ub=b_ub, options={"disp": False})
    if res.success:
        return res.x.reshape((c.size,1))
    else:
        np.savez('bad_scipy_lp_ineq_{:010d}'.format(np.random.randint(int(1e9))),
                c=c, A=A, b=b, res=res)
        raise Exception('Scipy did not solve the LP. Blame Scipy.')
        return None

def _scipy_quadratic_program_bnd(c, Q, lb, ub):

    # define the objective and gradient
    def fun(x):
        f = np.dot(x, c) + np.dot(x, np.dot(Q, x.T))
        return f[0]

    def jac(x):
        j = c.T + 2.0*np.dot(x, Q)
        return j[0]

    # make bounds
    bounds = []
    for i in range(lb.size):
        bounds.append((lb[i,0],ub[i,0]))

    x0 = np.zeros((c.size,))
    res = minimize(fun, x0, method='L-BFGS-B', jac=jac,
                    bounds=bounds, options={"disp": False})

    if res.success:
        xstar = res.x
        if isinstance(xstar, float):
            xstar = np.array([[xstar]])
        return xstar.reshape((c.size,1))
    else:
        np.savez('bad_scipy_qp_bnd_{:010d}'.format(np.random.randint(int(1e9))),
                c=c, Q=Q, lb=lb, ub=ub, res=res)
        raise Exception('Scipy did not solve the LP. Blame Scipy.')
        return None

def _scipy_quadratic_program_ineq(c, Q, A, b):

    # define the objective and gradient
    def fun(x):
        f = np.dot(x, c) + np.dot(x, np.dot(Q, x.T))
        return f[0]

    def jac(x):
        j = c.T + 2.0*np.dot(x, Q)
        return j[0]

    # inequality constraints
    cons = ({'type':'ineq',
            'fun' : lambda x: np.dot(A, x) - b,
            'jac' : lambda x: A})

    x0 = np.zeros((c.size,))
    res = minimize(fun, x0, method='SLSQP', jac=jac,
                    constraints=cons, options={"disp": False})

    if res.success:
        xstar = res.x
        if isinstance(xstar, float):
            xstar = np.array([[xstar]])
        return xstar.reshape((c.size,1))
    else:
        np.savez('bad_scipy_qp_ineq_{:010d}'.format(np.random.randint(int(1e9))),
                c=c, Q=Q, A=A, b=b, res=res)
        raise Exception('Scipy did not solve the LP. Blame Scipy.')
        return None

def _gurobi_linear_program_eq(c, A, b, lb, ub):

    m,n = A.shape
    model = gpy.Model()
    model.setParam('OutputFlag', 0)

    # Add variables to model
    vars = []
    for j in range(n):
        vars.append(model.addVar(lb=lb[j,0], ub=ub[j,0], vtype=gpy.GRB.CONTINUOUS))
    model.update()

    # Populate linear constraints
    for i in range(m):
        expr = gpy.LinExpr()
        for j in range(n):
            expr += A[i,j]*vars[j]
        model.addConstr(expr, gpy.GRB.EQUAL, b[i,0])

    # Populate objective
    obj = gpy.LinExpr()
    for j in range(n):
        obj += c[j,0]*vars[j]
    model.setObjective(obj)
    model.update()

    # Solve
    model.optimize()

    if model.status == gpy.GRB.OPTIMAL:
        return np.array(model.getAttr('x', vars)).reshape((n,1))
    else:
        np.savez('bad_gurobi_lp_eq_{:010d}'.format(np.random.randint(int(1e9))),
                c=c, A=A, b=b, lb=lb, ub=ub, model=model)
        raise Exception('Gurobi did not solve the LP. Blame Gurobi.')
        return None

def _gurobi_linear_program_ineq(c, A, b):

    m,n = A.shape
    model = gpy.Model()
    model.setParam('OutputFlag', 0)

    # Add variables to model
    vars = []
    for j in range(n):
        vars.append(model.addVar(lb=-gpy.GRB.INFINITY, ub=gpy.GRB.INFINITY, vtype=gpy.GRB.CONTINUOUS))
    model.update()

    # Populate linear constraints
    for i in range(m):
        expr = gpy.LinExpr()
        for j in range(n):
            expr += A[i,j]*vars[j]
        model.addConstr(expr, gpy.GRB.GREATER_EQUAL, b[i,0])

    # Populate objective
    obj = gpy.LinExpr()
    for j in range(n):
        obj += c[j,0]*vars[j]
    model.setObjective(obj)
    model.update()

    # Solve
    model.optimize()

    if model.status == gpy.GRB.OPTIMAL:
        return np.array(model.getAttr('x', vars)).reshape((n,1))
    else:
        np.savez('bad_gurobi_lp_ineq_{:010d}'.format(np.random.randint(int(1e9))),
                c=c, A=A, b=b, model=model)
        raise Exception('Gurobi did not solve the LP. Blame Gurobi.')
        return None

def _gurobi_quadratic_program_bnd(c, Q, lb, ub):

    n = Q.shape[0]
    model = gpy.Model()
    model.setParam('OutputFlag', 0)

    # Add variables to model
    vars = []
    for j in range(n):
        vars.append(model.addVar(lb=lb[j,0], ub=ub[j,0], vtype=gpy.GRB.CONTINUOUS))
    model.update()

    # Populate objective
    obj = gpy.QuadExpr()
    for i in range(n):
        for j in range(n):
            obj += Q[i,j]*vars[i]*vars[j]

    for j in range(n):
        obj += c[j,0]*vars[j]
    model.setObjective(obj)
    model.update()

    # Solve
    model.optimize()

    if model.status == gpy.GRB.OPTIMAL:
        return np.array(model.getAttr('x', vars)).reshape((n,1))
    else:
        np.savez('bad_gurobi_qp_bnd_{:010d}'.format(np.random.randint(int(1e9))),
                c=c, Q=Q, lb=lb, ub=ub, model=model)
        raise Exception('Gurobi did not solve the QP. Blame Gurobi.')
        return None

def _gurobi_quadratic_program_ineq(c, Q, A, b):

    m,n = A.shape
    model = gpy.Model()
    model.setParam('OutputFlag', 0)

    # Add variables to model
    vars = []
    for j in range(n):
        vars.append(model.addVar(lb=-gpy.GRB.INFINITY, ub=gpy.GRB.INFINITY, vtype=gpy.GRB.CONTINUOUS))
    model.update()

    # Populate linear constraints
    for i in range(m):
        expr = gpy.LinExpr()
        for j in range(n):
            expr += A[i,j]*vars[j]
        model.addConstr(expr, gpy.GRB.GREATER_EQUAL, b[i,0])

    # Populate objective
    obj = gpy.QuadExpr()
    for i in range(n):
        for j in range(n):
            obj += Q[i,j]*vars[i]*vars[j]

    for j in range(n):
        obj += c[j,0]*vars[j]
    model.setObjective(obj)
    model.update()

    # Solve
    model.optimize()

    if model.status == gpy.GRB.OPTIMAL:
        return np.array(model.getAttr('x', vars)).reshape((n,1))
    else:
        np.savez('bad_gurobi_qp_ineq_{:010d}'.format(np.random.randint(int(1e9))),
                c=c, Q=Q, A=A, b=b, model=model)
        raise Exception('Gurobi did not solve the QP. Blame Gurobi.')
        return None
<commit_msg>bug in my scipy linprog usage, fixed<commit_after>"""Solvers for the linear and quadratic programs in active subspaces."""
import numpy as np
import warnings
from scipy.optimize import linprog, minimize

# checking to see if system has gurobi
try:
    HAS_GUROBI = True
    import gurobipy as gpy
except ImportError, e:
    HAS_GUROBI = False
    pass

# string constants for QP solver names
solver_SCIPY = 'SCIPY'
solver_GUROBI = 'GUROBI'

class QPSolver():
    """
    A class for solving linear and quadratic programs.

    Attributes
    ----------
    solver : str
        `solver` identifies which linear program software to use.

    Notes
    -----
    The class checks to see if Gurobi is present. If it is, it uses Gurobi to
    solve the linear and quadratic programs. Otherwise, it uses scipy
    implementations to solve the linear and quadratic programs.
    """
    solver = None

    def __init__(self, solver='GUROBI'):
        """
        Initialize a QPSolver.

        Parameters
        ----------
        solver : str, optional
            `solver` identifies which linear program software to use. Default is
            'GUROBI'. Another option is 'SCIPY'.
        """

        if solver==solver_GUROBI and HAS_GUROBI:
            self.solver = solver_GUROBI
        elif solver=='SCIPY':
            self.solver = solver_SCIPY
        else:
            warnings.warn('QP solver %s is not available. Using scipy optimization package.' % solver)
            self.solver = solver_SCIPY

    def linear_program_eq(self, c, A, b, lb, ub):
        """
        Solves an equality constrained linear program with variable bounds.

        Parameters
        ----------
        c : ndarray
            `c` is an ndarray of size m-by-1 for the linear objective function.
        A : ndarray
            `A` is an ndarray of size M-by-m that contains the coefficients
            of the linear equality constraints.
        b : ndarray
            `b` is an ndarray of size M-by-1 that is the right hand side of the
            equality constraints.
        lb : ndarray
            `lb` is an ndarray of size m-by-1 that contains the lower bounds
            on the variables.
        ub : ndarray
            `ub` is an ndarray of size m-by-1 that contains the upper bounds
            on the variables.

        Returns
        -------
        x : ndarray
            An ndarray of size m-by-1 that is the minimizer of the linear
            program.

        Notes
        -----
        This method returns the minimizer of the following linear program.

        minimize  c^T x
        subject to  A x = b
                    lb <= x <= ub
        """
        if self.solver == solver_SCIPY:
            return _scipy_linear_program_eq(c, A, b, lb, ub)
        elif self.solver == solver_GUROBI:
            return _gurobi_linear_program_eq(c, A, b, lb, ub)
        else:
            raise ValueError('QP solver %s not available' % self.solver)

    def linear_program_ineq(self, c, A, b):
        """
        Solves an inequality constrained linear program.

        Parameters
        ----------
        c : ndarray
            `c` is an ndarray of size m-by-1 for the linear objective function.
        A : ndarray
            `A` is an ndarray of size M-by-m that contains the coefficients
            of the linear equality constraints.
        b : ndarray
            `b` is an ndarray of size M-by-1 that is the right hand side of the
            equality constraints.

        Returns
        -------
        x : ndarray
            An ndarray of size m-by-1 that is the minimizer of the linear
            program.

        Notes
        -----
        This method returns the minimizer of the following linear program.

        minimize  c^T x
        subject to  A x >= b
        """

        if self.solver == solver_SCIPY:
            return _scipy_linear_program_ineq(c, A, b)
        elif self.solver == solver_GUROBI:
            return _gurobi_linear_program_ineq(c, A, b)
        else:
            raise ValueError('QP solver %s not available' % self.solver)

    def quadratic_program_bnd(self, c, Q, lb, ub):
        """
        Solves a quadratic program with variable bounds.

        Parameters
        ----------
        c : ndarray
            `c` is an ndarray of size m-by-1 that contains the coefficients of
            the linear term in the objective function.
        Q : ndarray
            `Q` is an ndarray of size m-by-m that contains the coefficients
            of the quadratic term in the objective function.
        lb : ndarray
            `lb` is an ndarray of size m-by-1 that contains the lower bounds
            on the variables.
        ub : ndarray
            `ub` is an ndarray of size m-by-1 that contains the upper bounds
            on the variables.

        Returns
        -------
        x : ndarray
            An ndarray of size m-by-1 that is the minimizer of the quadratic
            program.

        Notes
        -----
        This method returns the minimizer of the following linear program.

        minimize  c^T x + x^T Q x
        subject to  lb <= x <= ub
        """

        if self.solver == solver_SCIPY:
            return _scipy_quadratic_program_bnd(c, Q, lb, ub)
        elif self.solver == solver_GUROBI:
            return _gurobi_quadratic_program_bnd(c, Q, lb, ub)
        else:
            raise ValueError('QP solver %s not available' % self.solver)

    def quadratic_program_ineq(self, c, Q, A, b):
        """
        Solves an inequality constrained quadratic program with variable bounds.

        Parameters
        ----------
        c : ndarray
            `c` is an ndarray of size m-by-1 that contains the coefficients of
            the linear term in the objective function.
        Q : ndarray
            `Q` is an ndarray of size m-by-m that contains the coefficients
            of the quadratic term in the objective function.
        A : ndarray
            `A` is an ndarray of size M-by-m that contains the coefficients
            of the linear equality constraints.
        b : ndarray
            `b` is an ndarray of size M-by-1 that is the right hand side of the
            equality constraints.

        Returns
        -------
        x : ndarray
            An ndarray of size m-by-1 that is the minimizer of the quadratic
            program.

        Notes
        -----
        This method returns the minimizer of the following linear program.

        minimize  c^T x + x^T Q x
        subject to  A x >= b
        """

        if self.solver == solver_SCIPY:
            return _scipy_quadratic_program_ineq(c, Q, A, b)
        elif self.solver == solver_GUROBI:
            return _gurobi_quadratic_program_ineq(c, Q, A, b)
        else:
            raise ValueError('QP solver %s not available' % self.solver)

def _scipy_linear_program_eq(c, A, b, lb, ub):

    c = c.reshape((c.size,))
    b = b.reshape((b.size,))

    # make bounds
    bounds = []
    for i in range(lb.size):
        bounds.append((lb[i,0], ub[i,0]))

    res = linprog(c, A_eq=A, b_eq=b, bounds=bounds, options={"disp": False})
    if res.success:
        return res.x.reshape((c.size,1))
    else:
        np.savez('bad_scipy_lp_eq_{:010d}'.format(np.random.randint(int(1e9))),
                c=c, A=A, b=b, lb=lb, ub=ub, res=res)
        raise Exception('Scipy did not solve the LP. Blame Scipy.')
        return None

def _scipy_linear_program_ineq(c, A, b):

    c = c.reshape((c.size,))
    b = b.reshape((b.size,))

    # make unbounded bounds
    bounds = []
    for i in range(c.size):
        bounds.append((None, None))

    A_ub, b_ub = -A, -b
    res = linprog(c, A_ub=A_ub, b_ub=b_ub, bounds=bounds, options={"disp": False})
    if res.success:
        return res.x.reshape((c.size,1))
    else:
        np.savez('bad_scipy_lp_ineq_{:010d}'.format(np.random.randint(int(1e9))),
                c=c, A=A, b=b, res=res)
        raise Exception('Scipy did not solve the LP. Blame Scipy.')
        return None

def _scipy_quadratic_program_bnd(c, Q, lb, ub):

    # define the objective and gradient
    def fun(x):
        f = np.dot(x, c) + np.dot(x, np.dot(Q, x.T))
        return f[0]

    def jac(x):
        j = c.T + 2.0*np.dot(x, Q)
        return j[0]

    # make bounds
    bounds = []
    for i in range(lb.size):
        bounds.append((lb[i,0],ub[i,0]))

    x0 = np.zeros((c.size,))
    res = minimize(fun, x0, method='L-BFGS-B', jac=jac,
                    bounds=bounds, options={"disp": False})

    if res.success:
        xstar = res.x
        if isinstance(xstar, float):
            xstar = np.array([[xstar]])
        return xstar.reshape((c.size,1))
    else:
        np.savez('bad_scipy_qp_bnd_{:010d}'.format(np.random.randint(int(1e9))),
                c=c, Q=Q, lb=lb, ub=ub, res=res)
        raise Exception('Scipy did not solve the LP. Blame Scipy.')
        return None

def _scipy_quadratic_program_ineq(c, Q, A, b):

    b = b.reshape((b.size,))

    # define the objective and gradient
    def fun(x):
        f = np.dot(x, c) + np.dot(x, np.dot(Q, x.T))
        return f[0]

    def jac(x):
        j = c.T + 2.0*np.dot(x, Q)
        return j[0]

    # inequality constraints
    cons = ({'type':'ineq',
            'fun' : lambda x: np.dot(A, x) - b,
            'jac' : lambda x: A})

    x0 = np.zeros((c.size,))
    res = minimize(fun, x0, method='SLSQP', jac=jac,
                    constraints=cons, options={"disp": False})

    if res.success:
        xstar = res.x
        if isinstance(xstar, float):
            xstar = np.array([[xstar]])
        return xstar.reshape((c.size,1))
    else:
        np.savez('bad_scipy_qp_ineq_{:010d}'.format(np.random.randint(int(1e9))),
                c=c, Q=Q, A=A, b=b, res=res)
        raise Exception('Scipy did not solve the LP. Blame Scipy.')
        return None

def _gurobi_linear_program_eq(c, A, b, lb, ub):

    m,n = A.shape
    model = gpy.Model()
    model.setParam('OutputFlag', 0)

    # Add variables to model
    vars = []
    for j in range(n):
        vars.append(model.addVar(lb=lb[j,0], ub=ub[j,0], vtype=gpy.GRB.CONTINUOUS))
    model.update()

    # Populate linear constraints
    for i in range(m):
        expr = gpy.LinExpr()
        for j in range(n):
            expr += A[i,j]*vars[j]
        model.addConstr(expr, gpy.GRB.EQUAL, b[i,0])

    # Populate objective
    obj = gpy.LinExpr()
    for j in range(n):
        obj += c[j,0]*vars[j]
    model.setObjective(obj)
    model.update()

    # Solve
    model.optimize()

    if model.status == gpy.GRB.OPTIMAL:
        return np.array(model.getAttr('x', vars)).reshape((n,1))
    else:
        np.savez('bad_gurobi_lp_eq_{:010d}'.format(np.random.randint(int(1e9))),
                c=c, A=A, b=b, lb=lb, ub=ub, model=model)
        raise Exception('Gurobi did not solve the LP. Blame Gurobi.')
        return None

def _gurobi_linear_program_ineq(c, A, b):

    m,n = A.shape
    model = gpy.Model()
    model.setParam('OutputFlag', 0)

    # Add variables to model
    vars = []
    for j in range(n):
        vars.append(model.addVar(lb=-gpy.GRB.INFINITY, ub=gpy.GRB.INFINITY, vtype=gpy.GRB.CONTINUOUS))
    model.update()

    # Populate linear constraints
    for i in range(m):
        expr = gpy.LinExpr()
        for j in range(n):
            expr += A[i,j]*vars[j]
        model.addConstr(expr, gpy.GRB.GREATER_EQUAL, b[i,0])

    # Populate objective
    obj = gpy.LinExpr()
    for j in range(n):
        obj += c[j,0]*vars[j]
    model.setObjective(obj)
    model.update()

    # Solve
    model.optimize()

    if model.status == gpy.GRB.OPTIMAL:
        return np.array(model.getAttr('x', vars)).reshape((n,1))
    else:
        np.savez('bad_gurobi_lp_ineq_{:010d}'.format(np.random.randint(int(1e9))),
                c=c, A=A, b=b, model=model)
        raise Exception('Gurobi did not solve the LP. Blame Gurobi.')
        return None

def _gurobi_quadratic_program_bnd(c, Q, lb, ub):

    n = Q.shape[0]
    model = gpy.Model()
    model.setParam('OutputFlag', 0)

    # Add variables to model
    vars = []
    for j in range(n):
        vars.append(model.addVar(lb=lb[j,0], ub=ub[j,0], vtype=gpy.GRB.CONTINUOUS))
    model.update()

    # Populate objective
    obj = gpy.QuadExpr()
    for i in range(n):
        for j in range(n):
            obj += Q[i,j]*vars[i]*vars[j]

    for j in range(n):
        obj += c[j,0]*vars[j]
    model.setObjective(obj)
    model.update()

    # Solve
    model.optimize()

    if model.status == gpy.GRB.OPTIMAL:
        return np.array(model.getAttr('x', vars)).reshape((n,1))
    else:
        np.savez('bad_gurobi_qp_bnd_{:010d}'.format(np.random.randint(int(1e9))),
                c=c, Q=Q, lb=lb, ub=ub, model=model)
        raise Exception('Gurobi did not solve the QP. Blame Gurobi.')
        return None

def _gurobi_quadratic_program_ineq(c, Q, A, b):

    m,n = A.shape
    model = gpy.Model()
    model.setParam('OutputFlag', 0)

    # Add variables to model
    vars = []
    for j in range(n):
        vars.append(model.addVar(lb=-gpy.GRB.INFINITY, ub=gpy.GRB.INFINITY, vtype=gpy.GRB.CONTINUOUS))
    model.update()

    # Populate linear constraints
    for i in range(m):
        expr = gpy.LinExpr()
        for j in range(n):
            expr += A[i,j]*vars[j]
        model.addConstr(expr, gpy.GRB.GREATER_EQUAL, b[i,0])

    # Populate objective
    obj = gpy.QuadExpr()
    for i in range(n):
        for j in range(n):
            obj += Q[i,j]*vars[i]*vars[j]

    for j in range(n):
        obj += c[j,0]*vars[j]
    model.setObjective(obj)
    model.update()

    # Solve
    model.optimize()

    if model.status == gpy.GRB.OPTIMAL:
        return np.array(model.getAttr('x', vars)).reshape((n,1))
    else:
        np.savez('bad_gurobi_qp_ineq_{:010d}'.format(np.random.randint(int(1e9))),
                c=c, Q=Q, A=A, b=b, model=model)
        raise Exception('Gurobi did not solve the QP. Blame Gurobi.')
        return None
<|endoftext|>


<commit_before># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import division
from __future__ import unicode_literals

import logging

from boto import ec2 as boto_ec2
from fabric.api import settings as fabric_settings
from fabric.context_managers import cd
from fabric.operations import run, put, sudo
from fabric.state import env

from mo_collections import UniqueIndex
from mo_dots import unwrap, wrap
from mo_dots.objects import datawrap, DataObject
from mo_files import File
from mo_logs import Log
from mo_logs import startup, constants
from pyLibrary.aws import aws_retry

@aws_retry
def _get_managed_spot_requests(ec2_conn, name):
    output = wrap([datawrap(r) for r in ec2_conn.get_all_spot_instance_requests() if not r.tags.get("Name") or r.tags.get("Name").startswith(name)])
    return output

@aws_retry
def _get_managed_instances(ec2_conn, name):
    requests = UniqueIndex(["instance_id"], data=_get_managed_spot_requests(ec2_conn, name).filter(lambda r: r.instance_id != None))
    reservations = ec2_conn.get_all_instances()

    output = []
    for res in reservations:
        for instance in res.instances:
            if instance.tags.get('Name', '').startswith(name) and instance._state.name == "running":
                instance.request = requests[instance.id]
                output.append(datawrap(instance))
    return wrap(output)

def _config_fabric(connect, instance):
    if not instance.ip_address:
        Log.error("Expecting an ip address for {{instance_id}}", instance_id=instance.id)

    for k, v in connect.items():
        env[k] = v
    env.host_string = instance.ip_address
    env.abort_exception = Log.error

def _disable_oom_on_es():

    with fabric_settings(warn_only=True):
        sudo("supervisorctl start es")

    with cd("/home/ec2-user"):
        run("mkdir -p temp")
    with cd("/home/ec2-user/temp"):
        processes = sudo("ps -eo pid,command | grep java")
        candidates = [
            line
            for line in processes.split("\n")
            if line.find("/usr/java/default/bin/java -Xms") != -1 and line.find("org.elasticsearch.bootstrap.Elasticsearch") != -1
        ]
        if not candidates:
            Log.error("Expecting to find some hint of Elasticsearch running")
        elif len(candidates) > 1:
            Log.error("Fond more than one Elasticsearch running, not sure what to do")

        pid = candidates[0].split(" ")[0].strip()
        run("echo -16 > oom_adj")
        sudo("sudo cp oom_adj /proc/" + pid + "/oom_adj")

def _refresh_indexer():
    with cd("/usr/local/elasticsearch"):
        sudo("rm -f java*.hprof")

    _disable_oom_on_es()
    with cd("/home/ec2-user/ActiveData-ETL/"):
        result = run("git pull origin push-to-es6")
        if "Already up-to-date." in result:
            Log.note("No change required")
        else:
            # RESTART ANYWAY, SO WE USE LATEST INDEX
            sudo("pip install -r requirements.txt")
            with fabric_settings(warn_only=True):
                sudo("supervisorctl stop push_to_es:*")
                sudo("supervisorctl start push_to_es:00")

def _start_supervisor():
    put("~/code/SpotManager/examples/config/es_supervisor.conf", "/etc/supervisord.conf", use_sudo=True)

    # START DAEMON (OR THROW ERROR IF RUNNING ALREADY)
    with fabric_settings(warn_only=True):
        sudo("supervisord -c /etc/supervisord.conf")

    sudo("supervisorctl reread")
    sudo("supervisorctl update")

def _run_remote(command, name):
    File("./results/temp/" + name + ".sh").write("nohup " + command + " >& /dev/null < /dev/null &\nsleep 20")
    put("./results/temp/" + name + ".sh", "" + name + ".sh")
    run("chmod u+x " + name + ".sh")
    run("./" + name + ".sh")

def main():
    try:
        settings = startup.read_settings()
        constants.set(settings.constants)
        Log.start(settings.debug)

        logging.getLogger('paramiko.transport').addHandler(LogTranslate())

        aws_args = dict(
            region_name=settings.aws.region,
            aws_access_key_id=unwrap(settings.aws.aws_access_key_id),
            aws_secret_access_key=unwrap(settings.aws.aws_secret_access_key)
        )
        ec2_conn = boto_ec2.connect_to_region(**aws_args)

        instances = _get_managed_instances(ec2_conn, settings.name)

        for i in instances:
            try:
                Log.note("Reset {{instance_id}} ({{name}}) at {{ip}}", instance_id=i.id, name=i.tags["Name"], ip=i.ip_address)
                _config_fabric(settings.fabric, i)
                _refresh_indexer()
            except Exception as e:
                Log.warning(
                    "could not refresh {{instance_id}} ({{name}}) at {{ip}}",
                    instance_id=i.id,
                    name=i.tags["Name"],
                    ip=i.ip_address,
                    cause=e
                )
    except Exception as e:
        Log.error("Problem with etl", e)
    finally:
        Log.stop()

class LogTranslate(object):

    def __init__(self, level=0):
        self.level=level

    def emit(self, record):
        Log.note("{{record}}", record=record)

    def flush(self):
        pass

    def handle(self, record):
        Log.note("{{record|json}}", record=DataObject(record))

if __name__ == "__main__":
    main()

<commit_msg>proper pip install<commit_after># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import division
from __future__ import unicode_literals

import logging

from boto import ec2 as boto_ec2
from fabric.api import settings as fabric_settings
from fabric.context_managers import cd
from fabric.operations import run, put, sudo
from fabric.state import env

from mo_collections import UniqueIndex
from mo_dots import unwrap, wrap
from mo_dots.objects import datawrap, DataObject
from mo_files import File
from mo_logs import Log
from mo_logs import startup, constants
from pyLibrary.aws import aws_retry

@aws_retry
def _get_managed_spot_requests(ec2_conn, name):
    output = wrap([datawrap(r) for r in ec2_conn.get_all_spot_instance_requests() if not r.tags.get("Name") or r.tags.get("Name").startswith(name)])
    return output

@aws_retry
def _get_managed_instances(ec2_conn, name):
    requests = UniqueIndex(["instance_id"], data=_get_managed_spot_requests(ec2_conn, name).filter(lambda r: r.instance_id != None))
    reservations = ec2_conn.get_all_instances()

    output = []
    for res in reservations:
        for instance in res.instances:
            if instance.tags.get('Name', '').startswith(name) and instance._state.name == "running":
                instance.request = requests[instance.id]
                output.append(datawrap(instance))
    return wrap(output)

def _config_fabric(connect, instance):
    if not instance.ip_address:
        Log.error("Expecting an ip address for {{instance_id}}", instance_id=instance.id)

    for k, v in connect.items():
        env[k] = v
    env.host_string = instance.ip_address
    env.abort_exception = Log.error

def _disable_oom_on_es():

    with fabric_settings(warn_only=True):
        sudo("supervisorctl start es")

    with cd("/home/ec2-user"):
        run("mkdir -p temp")
    with cd("/home/ec2-user/temp"):
        processes = sudo("ps -eo pid,command | grep java")
        candidates = [
            line
            for line in processes.split("\n")
            if line.find("/usr/java/default/bin/java -Xms") != -1 and line.find("org.elasticsearch.bootstrap.Elasticsearch") != -1
        ]
        if not candidates:
            Log.error("Expecting to find some hint of Elasticsearch running")
        elif len(candidates) > 1:
            Log.error("Fond more than one Elasticsearch running, not sure what to do")

        pid = candidates[0].split(" ")[0].strip()
        run("echo -16 > oom_adj")
        sudo("sudo cp oom_adj /proc/" + pid + "/oom_adj")

def _refresh_indexer():
    with cd("/usr/local/elasticsearch"):
        sudo("rm -f java*.hprof")

    _disable_oom_on_es()
    with cd("/home/ec2-user/ActiveData-ETL/"):
        result = run("git pull origin push-to-es6")
        if "Already up-to-date." in result:
            Log.note("No change required")
        else:
            # RESTART ANYWAY, SO WE USE LATEST INDEX
            run("~/pypy/bin/pypy -m pip install -r requirements.txt")
            with fabric_settings(warn_only=True):
                sudo("supervisorctl stop push_to_es:*")
                sudo("supervisorctl start push_to_es:00")

def _start_supervisor():
    put("~/code/SpotManager/examples/config/es_supervisor.conf", "/etc/supervisord.conf", use_sudo=True)

    # START DAEMON (OR THROW ERROR IF RUNNING ALREADY)
    with fabric_settings(warn_only=True):
        sudo("supervisord -c /etc/supervisord.conf")

    sudo("supervisorctl reread")
    sudo("supervisorctl update")

def _run_remote(command, name):
    File("./results/temp/" + name + ".sh").write("nohup " + command + " >& /dev/null < /dev/null &\nsleep 20")
    put("./results/temp/" + name + ".sh", "" + name + ".sh")
    run("chmod u+x " + name + ".sh")
    run("./" + name + ".sh")

def main():
    try:
        settings = startup.read_settings()
        constants.set(settings.constants)
        Log.start(settings.debug)

        logging.getLogger('paramiko.transport').addHandler(LogTranslate())

        aws_args = dict(
            region_name=settings.aws.region,
            aws_access_key_id=unwrap(settings.aws.aws_access_key_id),
            aws_secret_access_key=unwrap(settings.aws.aws_secret_access_key)
        )
        ec2_conn = boto_ec2.connect_to_region(**aws_args)

        instances = _get_managed_instances(ec2_conn, settings.name)

        for i in instances:
            try:
                Log.note("Reset {{instance_id}} ({{name}}) at {{ip}}", instance_id=i.id, name=i.tags["Name"], ip=i.ip_address)
                _config_fabric(settings.fabric, i)
                _refresh_indexer()
            except Exception as e:
                Log.warning(
                    "could not refresh {{instance_id}} ({{name}}) at {{ip}}",
                    instance_id=i.id,
                    name=i.tags["Name"],
                    ip=i.ip_address,
                    cause=e
                )
    except Exception as e:
        Log.error("Problem with etl", e)
    finally:
        Log.stop()

class LogTranslate(object):

    def __init__(self, level=0):
        self.level=level

    def emit(self, record):
        Log.note("{{record}}", record=record)

    def flush(self):
        pass

    def handle(self, record):
        Log.note("{{record|json}}", record=DataObject(record))

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before><commit_msg>[MERGE] forward port branch 9.0 up to b5da9d65ae2<commit_after><|endoftext|>