<commit_before>from argparse import ArgumentParser
from itertools import takewhile
from os import listdir, makedirs
from os.path import abspath, dirname, exists
from typing import Dict, List, Optional, Tuple

from transformers import is_tf_available, is_torch_available
from transformers.pipelines import Pipeline, pipeline
from transformers.tokenization_utils import BatchEncoding

class OnnxConverterArgumentParser(ArgumentParser):
    """
    Wraps all the script arguments supported to export transformers models to ONNX IR
    """

    def __init__(self):
        super(OnnxConverterArgumentParser, self).__init__("ONNX Converter")

        self.add_argument("--model", type=str, required=True, help="Model's id or path (ex: bert-base-cased)")
        self.add_argument("--tokenizer", type=str, help="Tokenizer's id or path (ex: bert-base-cased)")
        self.add_argument("--framework", type=str, choices=["pt", "tf"], help="Framework for loading the model")
        self.add_argument("--opset", type=int, default=11, help="ONNX opset to use")
        self.add_argument("--check-loading", action="store_true", help="Check ONNX is able to load the model")
        self.add_argument("--use-external-format", action="store_true", help="Allow exporting model >= than 2Gb")
        self.add_argument("output")

def ensure_valid_input(model, tokens, input_names):
    """
    Ensure input are presented in the correct order, without any None
    Args:
        model: The model used to forward the input data
        tokens: BatchEncoding holding the input data
        input_names: The name of the inputs

    Returns: Tuple

    """
    model_args_name = model.forward.__code__.co_varnames
    model_args_pos = [(model_args_name.index(name) - 1, name) for name in input_names]
    model_args = [None] * (max(map(lambda x: x[0], model_args_pos)) + 1)

    for arg_pos, arg_name in model_args_pos:
        model_args[arg_pos] = tokens[arg_name]

    model_args = tuple(model_args)  # Need to be ordered
    return tuple(takewhile(lambda arg: arg is not None, model_args))

def infer_shapes(nlp: Pipeline, framework: str) -> Tuple[List[str], List[str], Dict, BatchEncoding]:
    def build_shape_dict(tensor, is_input: bool, seq_len: int):
        if isinstance(tensor, (tuple, list)):
            return [build_shape_dict(t, is_input, seq_len) for t in tensor]

        else:
            # Let's assume batch is the first axis with only 1 element (~~ might not be always true ...)
            axes = {[axis for axis, numel in enumerate(tensor.shape) if numel == 1][0]: "batch"}
            if is_input:
                if len(tensor.shape) == 2:
                    axes[1] = "sequence"
                else:
                    raise ValueError("Unable to infer tensor axes ({})".format(len(tensor.shape)))
            else:
                seq_axes = [dim for dim, shape in enumerate(tensor.shape) if shape == seq_len]
                axes.update({dim: "sequence" for dim in seq_axes})

        return axes

    tokens = nlp.tokenizer.encode_plus("This is a sample output", return_tensors=framework)
    seq_len = tokens.input_ids.shape[-1]
    outputs = nlp.model(**tokens) if framework == "pt" else nlp.model(tokens)

    if not isinstance(outputs, (list, tuple)):
        outputs = (outputs,)

    # Generate input names & axes
    input_vars = list(tokens.keys())
    input_dynamic_axes = {k: build_shape_dict(v, True, seq_len) for k, v in tokens.items()}

    # flatten potentially grouped outputs (past for gpt2, attentions)
    outputs_flat = []
    for output in outputs:
        if isinstance(output, (tuple, list)):
            outputs_flat.extend(output)
        else:
            outputs_flat.append(output)

    # Generate output names & axes
    output_names = ["output_{}".format(i) for i in range(len(outputs_flat))]
    output_dynamic_axes = {k: build_shape_dict(v, False, seq_len) for k, v in zip(output_names, outputs_flat)}

    # Create the aggregated axes representation
    dynamic_axes = dict(input_dynamic_axes, **output_dynamic_axes)
    return input_vars, output_names, dynamic_axes, tokens

def load_graph_from_args(framework: str, model: str, tokenizer: Optional[str] = None) -> Pipeline:
    # If no tokenizer provided
    if tokenizer is None:
        tokenizer = model

    print("Loading pipeline (model: {}, tokenizer: {})".format(model, tokenizer))

    # Allocate tokenizer and model
    return pipeline("feature-extraction", model=model, framework=framework)

def convert_pytorch(nlp: Pipeline, opset: int, output: str, use_external_format: bool):
    if not is_torch_available():
        raise Exception("Cannot convert because PyTorch is not installed. Please install torch first.")

    import torch
    from torch.onnx import export

    print("PyTorch: {}".format(torch.__version__))

    with torch.no_grad():
        input_names, output_names, dynamic_axes, tokens = infer_shapes(nlp, "pt")
        model_args = ensure_valid_input(nlp.model, tokens, input_names)

        export(
            nlp.model,
            model_args,
            f=output,
            input_names=input_names,
            output_names=output_names,
            dynamic_axes=dynamic_axes,
            do_constant_folding=True,
            use_external_data_format=use_external_format,
            enable_onnx_checker=True,
            opset_version=opset,
        )

def convert_tensorflow(nlp: Pipeline, opset: int, output: str):
    if not is_tf_available():
        raise Exception(
            "Cannot convert {} because TF is not installed. Please install torch first.".format(args.model)
        )

    print("/!\\ Please note TensorFlow doesn't support exporting model > 2Gb /!\\")

    try:
        import tensorflow as tf
        from keras2onnx import convert_keras, save_model, __version__ as k2ov

        print("TensorFlow: {}, keras2onnx: {}".format(tf.version.VERSION, k2ov))

        # Build
        input_names, output_names, dynamic_axes, tokens = infer_shapes(nlp, "tf")

        # Forward
        nlp.model.predict(tokens.data)
        onnx_model = convert_keras(nlp.model, nlp.model.name, target_opset=opset)
        save_model(onnx_model, output)

    except ImportError as e:
        raise Exception(
            "Cannot import {} required to convert TF model to ONNX. Please install {} first.".format(e.name, e.name)
        )

def convert(
    framework: str,
    model: str,
    output: str,
    opset: int,
    tokenizer: Optional[str] = None,
    use_external_format: bool = False,
):
    print("ONNX opset version set to: {}".format(opset))

    # Load the pipeline
    nlp = load_graph_from_args(framework, model, tokenizer)

    parent = dirname(output)
    if not exists(parent):
        print("Creating folder {}".format(parent))
        makedirs(parent)
    elif len(listdir(parent)) > 0:
        raise Exception("Folder {} is not empty, aborting conversion".format(parent))

    # Export the graph
    if framework == "pt":
        convert_pytorch(nlp, opset, output, use_external_format)
    else:
        convert_tensorflow(nlp, opset, output)

def verify(path: str):
    from onnxruntime import InferenceSession, SessionOptions
    from onnxruntime.capi.onnxruntime_pybind11_state import RuntimeException

    print("Checking ONNX model loading from: {}".format(path))
    try:
        onnx_options = SessionOptions()
        _ = InferenceSession(path, onnx_options, providers=["CPUExecutionProvider"])
        print("Model correctly loaded")
    except RuntimeException as re:
        print("Error while loading the model: {}".format(re))

if __name__ == "__main__":
    parser = OnnxConverterArgumentParser()
    args = parser.parse_args()

    # Make sure output is absolute path
    args.output = abspath(args.output)

    try:
        # Convert
        convert(args.framework, args.model, args.output, args.opset, args.tokenizer, args.use_external_format)

        # And verify
        if args.check_loading:
            verify(args.output)
    except Exception as e:
        print("Error while converting the model: {}".format(e))
        exit(1)
<commit_msg>pass on tokenizer to pipeline (#4489)<commit_after>from argparse import ArgumentParser
from itertools import takewhile
from os import listdir, makedirs
from os.path import abspath, dirname, exists
from typing import Dict, List, Optional, Tuple

from transformers import is_tf_available, is_torch_available
from transformers.pipelines import Pipeline, pipeline
from transformers.tokenization_utils import BatchEncoding

class OnnxConverterArgumentParser(ArgumentParser):
    """
    Wraps all the script arguments supported to export transformers models to ONNX IR
    """

    def __init__(self):
        super(OnnxConverterArgumentParser, self).__init__("ONNX Converter")

        self.add_argument("--model", type=str, required=True, help="Model's id or path (ex: bert-base-cased)")
        self.add_argument("--tokenizer", type=str, help="Tokenizer's id or path (ex: bert-base-cased)")
        self.add_argument("--framework", type=str, choices=["pt", "tf"], help="Framework for loading the model")
        self.add_argument("--opset", type=int, default=11, help="ONNX opset to use")
        self.add_argument("--check-loading", action="store_true", help="Check ONNX is able to load the model")
        self.add_argument("--use-external-format", action="store_true", help="Allow exporting model >= than 2Gb")
        self.add_argument("output")

def ensure_valid_input(model, tokens, input_names):
    """
    Ensure input are presented in the correct order, without any None
    Args:
        model: The model used to forward the input data
        tokens: BatchEncoding holding the input data
        input_names: The name of the inputs

    Returns: Tuple

    """
    model_args_name = model.forward.__code__.co_varnames
    model_args_pos = [(model_args_name.index(name) - 1, name) for name in input_names]
    model_args = [None] * (max(map(lambda x: x[0], model_args_pos)) + 1)

    for arg_pos, arg_name in model_args_pos:
        model_args[arg_pos] = tokens[arg_name]

    model_args = tuple(model_args)  # Need to be ordered
    return tuple(takewhile(lambda arg: arg is not None, model_args))

def infer_shapes(nlp: Pipeline, framework: str) -> Tuple[List[str], List[str], Dict, BatchEncoding]:
    def build_shape_dict(tensor, is_input: bool, seq_len: int):
        if isinstance(tensor, (tuple, list)):
            return [build_shape_dict(t, is_input, seq_len) for t in tensor]

        else:
            # Let's assume batch is the first axis with only 1 element (~~ might not be always true ...)
            axes = {[axis for axis, numel in enumerate(tensor.shape) if numel == 1][0]: "batch"}
            if is_input:
                if len(tensor.shape) == 2:
                    axes[1] = "sequence"
                else:
                    raise ValueError("Unable to infer tensor axes ({})".format(len(tensor.shape)))
            else:
                seq_axes = [dim for dim, shape in enumerate(tensor.shape) if shape == seq_len]
                axes.update({dim: "sequence" for dim in seq_axes})

        return axes

    tokens = nlp.tokenizer.encode_plus("This is a sample output", return_tensors=framework)
    seq_len = tokens.input_ids.shape[-1]
    outputs = nlp.model(**tokens) if framework == "pt" else nlp.model(tokens)

    if not isinstance(outputs, (list, tuple)):
        outputs = (outputs,)

    # Generate input names & axes
    input_vars = list(tokens.keys())
    input_dynamic_axes = {k: build_shape_dict(v, True, seq_len) for k, v in tokens.items()}

    # flatten potentially grouped outputs (past for gpt2, attentions)
    outputs_flat = []
    for output in outputs:
        if isinstance(output, (tuple, list)):
            outputs_flat.extend(output)
        else:
            outputs_flat.append(output)

    # Generate output names & axes
    output_names = ["output_{}".format(i) for i in range(len(outputs_flat))]
    output_dynamic_axes = {k: build_shape_dict(v, False, seq_len) for k, v in zip(output_names, outputs_flat)}

    # Create the aggregated axes representation
    dynamic_axes = dict(input_dynamic_axes, **output_dynamic_axes)
    return input_vars, output_names, dynamic_axes, tokens

def load_graph_from_args(framework: str, model: str, tokenizer: Optional[str] = None) -> Pipeline:
    # If no tokenizer provided
    if tokenizer is None:
        tokenizer = model

    print("Loading pipeline (model: {}, tokenizer: {})".format(model, tokenizer))

    # Allocate tokenizer and model
    return pipeline("feature-extraction", model=model, tokenizer=tokenizer, framework=framework)

def convert_pytorch(nlp: Pipeline, opset: int, output: str, use_external_format: bool):
    if not is_torch_available():
        raise Exception("Cannot convert because PyTorch is not installed. Please install torch first.")

    import torch
    from torch.onnx import export

    print("PyTorch: {}".format(torch.__version__))

    with torch.no_grad():
        input_names, output_names, dynamic_axes, tokens = infer_shapes(nlp, "pt")
        model_args = ensure_valid_input(nlp.model, tokens, input_names)

        export(
            nlp.model,
            model_args,
            f=output,
            input_names=input_names,
            output_names=output_names,
            dynamic_axes=dynamic_axes,
            do_constant_folding=True,
            use_external_data_format=use_external_format,
            enable_onnx_checker=True,
            opset_version=opset,
        )

def convert_tensorflow(nlp: Pipeline, opset: int, output: str):
    if not is_tf_available():
        raise Exception(
            "Cannot convert {} because TF is not installed. Please install torch first.".format(args.model)
        )

    print("/!\\ Please note TensorFlow doesn't support exporting model > 2Gb /!\\")

    try:
        import tensorflow as tf
        from keras2onnx import convert_keras, save_model, __version__ as k2ov

        print("TensorFlow: {}, keras2onnx: {}".format(tf.version.VERSION, k2ov))

        # Build
        input_names, output_names, dynamic_axes, tokens = infer_shapes(nlp, "tf")

        # Forward
        nlp.model.predict(tokens.data)
        onnx_model = convert_keras(nlp.model, nlp.model.name, target_opset=opset)
        save_model(onnx_model, output)

    except ImportError as e:
        raise Exception(
            "Cannot import {} required to convert TF model to ONNX. Please install {} first.".format(e.name, e.name)
        )

def convert(
    framework: str,
    model: str,
    output: str,
    opset: int,
    tokenizer: Optional[str] = None,
    use_external_format: bool = False,
):
    print("ONNX opset version set to: {}".format(opset))

    # Load the pipeline
    nlp = load_graph_from_args(framework, model, tokenizer)

    parent = dirname(output)
    if not exists(parent):
        print("Creating folder {}".format(parent))
        makedirs(parent)
    elif len(listdir(parent)) > 0:
        raise Exception("Folder {} is not empty, aborting conversion".format(parent))

    # Export the graph
    if framework == "pt":
        convert_pytorch(nlp, opset, output, use_external_format)
    else:
        convert_tensorflow(nlp, opset, output)

def verify(path: str):
    from onnxruntime import InferenceSession, SessionOptions
    from onnxruntime.capi.onnxruntime_pybind11_state import RuntimeException

    print("Checking ONNX model loading from: {}".format(path))
    try:
        onnx_options = SessionOptions()
        _ = InferenceSession(path, onnx_options, providers=["CPUExecutionProvider"])
        print("Model correctly loaded")
    except RuntimeException as re:
        print("Error while loading the model: {}".format(re))

if __name__ == "__main__":
    parser = OnnxConverterArgumentParser()
    args = parser.parse_args()

    # Make sure output is absolute path
    args.output = abspath(args.output)

    try:
        # Convert
        convert(args.framework, args.model, args.output, args.opset, args.tokenizer, args.use_external_format)

        # And verify
        if args.check_loading:
            verify(args.output)
    except Exception as e:
        print("Error while converting the model: {}".format(e))
        exit(1)
<|endoftext|>


<commit_before> # -*- coding: utf-8 -*-
#################################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2011 Julius Network Solutions SARL <contact@julius.fr>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#################################################################################

from datetime import datetime
from osv import fields, osv
from tools.translate import _
import netsvc

class one2many_special(fields.one2many):
    def get(self, cr, obj, ids, name, user=None, offset=0, context=None, values=None):
        if not values:
            values = {}
        res = {}
        location_ids = []
        for id in ids:
            res[id] = []
            location_id = obj.pool.get('stock.tracking').read(cr, user, id, ['location_id'])['location_id']
            if location_id and location_id[0] and location_id[0] not in location_ids:
                location_ids.append(location_id[0])
        ids2 = obj.pool.get(self._obj).search(cr, user, self._domain + [(self._fields_id, 'in', ids), ('location_dest_id', 'in', location_ids)], limit=self._limit)
        for r in obj.pool.get(self._obj)._read_flat(cr, user, ids2, [self._fields_id], context=context, load='_classic_write'):
            res[r[self._fields_id]].append( r['id'] )
        return res

class stock_tracking(osv.osv):

    _inherit = 'stock.tracking'

    def hierarchy_ids(self, tracking):
        result_list = [tracking]
        for child in tracking.child_ids:
            result_list.extend(self.hierarchy_ids(child))
        return result_list

    def _get_child_products(self, cr, uid, ids, field_name, arg, context=None):
        packs = self.browse(cr, uid, ids)
        res = {}
        for pack in packs:
            res[pack.id] = []
            childs = self.hierarchy_ids(pack)
            for child in childs:
                for prod in child.product_ids:
                    res[pack.id].append(prod.id)
        return res

    def _get_child_serials(self, cr, uid, ids, field_name, arg, context=None):
        packs = self.browse(cr, uid, ids)
        res = {}
        for pack in packs:
            res[pack.id] = []
            childs = self.hierarchy_ids(pack)
            for child in childs:
                for serial in child.serial_ids:
                    res[pack.id].append(serial.id)
        return res

    _columns = {
        'parent_id': fields.many2one('stock.tracking', 'Parent'),
        'child_ids': fields.one2many('stock.tracking', 'parent_id', 'Children'),
        'ul_id': fields.many2one('product.ul', 'Logistic unit', readonly=True, states={'open':[('readonly',False)]}),
        'location_id': fields.many2one('stock.location', 'Location', required=True, readonly=True, states={'open':[('readonly',False)]}),
        'state': fields.selection([('open','Open'),('close','Close')], 'State', readonly=True),
        'product_ids': fields.one2many('product.stock.tracking', 'tracking_id', 'Products', readonly=True, states={'open':[('readonly',False)]}),
        'child_product_ids': fields.function(_get_child_products, method=True, type='one2many', obj='product.stock.tracking', string='Child Products'),
        'history_ids': fields.one2many('stock.tracking.history', 'tracking_id', 'History'),
        'current_move_ids': one2many_special('stock.move', 'tracking_id', 'Current moves', domain=[('pack_history_id', '=', False)], readonly=True),
        'name': fields.char('Pack Reference', size=64, required=True, readonly=True, states={'open':[('readonly',False)]}),
        'date': fields.datetime('Creation Date', required=True, readonly=True, states={'open':[('readonly',False)]}),
        'serial_ids': fields.one2many('serial.stock.tracking', 'tracking_id', 'Products', readonly=True, states={'open':[('readonly',False)]}),
        'child_serial_ids': fields.function(_get_child_serials, method=True, type='one2many', obj='serial.stock.tracking', string='Child Serials'),
    }

    def _check_parent_id(self, cr, uid, ids, context=None):
        lines = self.browse(cr, uid, ids, context=context)

        if lines[0].parent_id:
            if lines[0].ul_id.capacity_index > lines[0].parent_id.ul_id.capacity_index:
                return False
        return True

    _constraints = [(_check_parent_id, 'Bad parent type selection. Please try again.',['parent_id'] ),]

    _defaults = {
        'state': 'open',
    }

    def reset_open(self, cr, uid, ids, context=None):
        pack_ids = self.browse(cr, uid, ids, context)
        for pack in pack_ids:
            allowed = True
            if pack.parent_id:
                if pack.parent_id and pack.parent_id != 'open':
                    self.write(cr, uid, [pack.parent_id.id], {'state': 'open'})
#                    allowed = False
#                    raise osv.except_osv(_('Not allowed !'),_('You can\'t re-open this pack because the parent pack is close'))
            if allowed:
                for child in pack.child_ids:
                    if child.state == 'open':
                        allowed = False
                        raise osv.except_osv(_('Not allowed !'),_('You can\'t re-open this pack because there is at least one not closed child'))
                        break
            if allowed:
                self.write(cr, uid, [pack.id], {'state': 'open'})
        return True

    def set_close(self, cr, uid, ids, context=None):
        pack_ids = self.browse(cr, uid, ids, context)
        for pack in pack_ids:
            allowed = True
            for child in pack.child_ids:
                if child.state == 'open':
                    allowed = False
                    raise osv.except_osv(_('Not allowed !'),_('You can\'t close this pack because there is at least one not closed child'))
                    break
#            if allowed:
#                self.write(cr, uid, [pack.id], {'state': 'close'})
        return True

    def get_products(self, cr, uid, ids, context=None):
        pack_ids = self.browse(cr, uid, ids, context)
        stock_track = self.pool.get('product.stock.tracking')
        for pack in pack_ids:
            childs = self.hierarchy_ids(pack)
            for child in childs:
                product_ids = [x.id for x in child.product_ids]
                stock_track.unlink(cr, uid, product_ids)
                product_list = {}
                for x in child.current_move_ids:
                    if x.location_dest_id.id == child.location_id.id:
                        if x.product_id.id not in product_list.keys():
                            product_list.update({x.product_id.id:x.product_qty})
                        else:
                            product_list[x.product_id.id] += x.product_qty
                for product in product_list.keys():
                    stock_track.create(cr, uid, {'product_id': product, 'quantity': product_list[product], 'tracking_id': child.id})
        return True

    def get_serials(self, cr, uid, ids, context=None):
        pack_ids = self.browse(cr, uid, ids, context)
        serial_track = self.pool.get('serial.stock.tracking')
        serial_obj = self.pool.get('stock.production.lot')
        for pack in pack_ids:
            childs = self.hierarchy_ids(pack)
            for child in childs:
                serial_ids = [x.id for x in child.serial_ids]
                serial_track.unlink(cr, uid, serial_ids)
                serial_list = {}
                for x in child.current_move_ids:
                    if x.location_dest_id.id == child.location_id.id:
                        if x.prodlot_id.id not in serial_list.keys():
                            serial_list.update({x.prodlot_id.id:x.product_qty})
                        else:
                            serial_list[x.prodlot_id.id] += x.product_qty
                for serial in serial_list.keys():
                    if serial:
                        serial_track.create(cr, uid, {'serial_id': serial, 'quantity': serial_list[serial], 'tracking_id': child.id})
                        serial_obj.write(cr, uid, [serial], {'tracking_id': child.id})
        return True

stock_tracking()

class product_ul(osv.osv):
    _inherit = "product.ul"
    _description = "Shipping Unit"
    _columns = {
         'capacity_index': fields.integer('Capacity index'),
    }
    _order = 'capacity_index'
product_ul()

class product_stock_tracking(osv.osv):

    _name = 'product.stock.tracking'

    _columns = {
        'product_id': fields.many2one('product.product', 'Product'),
        'quantity': fields.float('Quantity'),
        'tracking_id': fields.many2one('stock.tracking', 'Tracking'),
#        'tracking_history_id': fields.many2one('stock.tracking.history', 'Tracking History'),
    }

product_stock_tracking()

class serial_stock_tracking(osv.osv):

    _name = 'serial.stock.tracking'

    _order = 'tracking_id,serial_id'

    _columns = {
        'serial_id': fields.many2one('stock.production.lot', 'Serial'),
        'product_id': fields.related('serial_id', 'product_id', type='many2one', relation='product.product', string='Product'),
        'quantity': fields.float('Quantity'),
        'tracking_id': fields.many2one('stock.tracking', 'Tracking'),
#        'tracking_history_id': fields.many2one('stock.tracking.history', 'Tracking History'),
    }

serial_stock_tracking()

class stock_tracking_history(osv.osv):

    _name = "stock.tracking.history"

    def _get_types(self, cr, uid, context={}):
#        res = [('pack_in','Add parent'),('pack_out','Unlink parent'),('move','Move')]
        res = []
        return res

#    def hierarchy_ids(self, tracking):
#        result_list = [tracking]
#        for child in tracking.child_ids:
#            result_list.extend(self.hierarchy_ids(child))
#        return result_list
#
#    def _get_child_products(self, cr, uid, ids, field_name, arg, context=None):
#        packs = self.browse(cr, uid, ids)
#        res = {}
#        for pack in packs:
#            res[pack.id] = []
#            childs = self.hierarchy_ids(pack)
#            for child in childs:
#                for prod in child.product_ids:
#                    res[pack.id].append(prod.id)
#        return res

    _columns = {
        'tracking_id': fields.many2one('stock.tracking', 'Pack', required=True),
        'type': fields.selection(_get_types, 'Type'),
#        'product_ids': fields.one2many('product.stock.tracking', 'tracking_history_id', 'Products'),
#        'child_product_ids': fields.function(_get_child_products, method=True, type='one2many', obj='product.stock.tracking', string='Child Products'),
#        'parent_hist_id': fields.many2one('stock.tracking.history', 'Parent history pack'),
#        'child_ids': fields.one2many('stock.tracking.history', 'parent_hist_id', 'Child history pack'),
    }

    _rec_name = "tracking_id"

stock_tracking_history()

'''Add a field in order to store the current pack in a production lot'''
class stock_production_lot(osv.osv):
    _inherit = 'stock.production.lot'
    _columns = {
        'tracking_id': fields.many2one('stock.tracking', 'pack'),
    }

stock_production_lot()

class product_category(osv.osv):
    _inherit = 'product.category'
    _columns = {
        'tracked': fields.boolean('Need a serial code ?'),
    }
product_category()

class stock_inventory(osv.osv):
    _inherit = 'stock.inventory'
    _defaults = {
        'name': lambda x, y, z, c: x.pool.get('ir.sequence').get(y, z, 'stock.inventory') or '/'
    }
stock_inventory()

class stock_move(osv.osv):
    _inherit = 'stock.move'
    _columns = {
#        'cancel_cascade': fields.boolean('Cancel Cascade', help='If checked, when this move is cancelled, cancel the linked move too')
    }

    def create(self, cr, uid, vals, context=None):
        production_lot_obj = self.pool.get('stock.production.lot')
        stock_tracking_obj = self.pool.get('stock.tracking')
        if vals.get('prodlot_id',False):
            production_lot_data = production_lot_obj.browse(cr, uid, vals['prodlot_id'])
            last_production_lot_move_id = self.search(cr, uid, [('prodlot_id', '=', production_lot_data.id)], limit=1, order='date')
            if last_production_lot_move_id:
                last_production_lot_move = self.browse(cr,uid,last_production_lot_move_id[0])
                if last_production_lot_move.tracking_id:
                    ids = [last_production_lot_move.tracking_id.id]
                    stock_tracking_obj.reset_open(cr, uid, ids, context=None)

        return super(stock_move,self).create(cr, uid, vals, context)

stock_move()

class split_in_production_lot(osv.osv_memory):
    _inherit = "stock.move.split"
    _columns = {
        'use_exist' : fields.boolean('Existing Lots', invisible=True),
     }
    _defaults = {
        'use_exist': lambda *a: True,
    }
    def default_get(self, cr, uid, fields, context=None):
        res = super(split_in_production_lot, self).default_get(cr, uid, fields, context)
        res.update({'use_exist': True})
        return res

split_in_production_lot()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[MRG] Merge with pierre<commit_after> # -*- coding: utf-8 -*-
#################################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2011 Julius Network Solutions SARL <contact@julius.fr>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#################################################################################

from datetime import datetime
from osv import fields, osv
from tools.translate import _
import netsvc

class one2many_special(fields.one2many):
    def get(self, cr, obj, ids, name, user=None, offset=0, context=None, values=None):
        if not values:
            values = {}
        res = {}
        location_ids = []
        for id in ids:
            res[id] = []
            location_id = obj.pool.get('stock.tracking').read(cr, user, id, ['location_id'])['location_id']
            if location_id and location_id[0] and location_id[0] not in location_ids:
                location_ids.append(location_id[0])
        ids2 = obj.pool.get(self._obj).search(cr, user, self._domain + [(self._fields_id, 'in', ids), ('location_dest_id', 'in', location_ids)], limit=self._limit)
        for r in obj.pool.get(self._obj)._read_flat(cr, user, ids2, [self._fields_id], context=context, load='_classic_write'):
            res[r[self._fields_id]].append( r['id'] )
        return res

class stock_tracking(osv.osv):

    _inherit = 'stock.tracking'

    def hierarchy_ids(self, tracking):
        result_list = [tracking]
        for child in tracking.child_ids:
            result_list.extend(self.hierarchy_ids(child))
        return result_list

    def _get_child_products(self, cr, uid, ids, field_name, arg, context=None):
        packs = self.browse(cr, uid, ids)
        res = {}
        for pack in packs:
            res[pack.id] = []
            childs = self.hierarchy_ids(pack)
            for child in childs:
                for prod in child.product_ids:
                    res[pack.id].append(prod.id)
        return res

    def _get_child_serials(self, cr, uid, ids, field_name, arg, context=None):
        packs = self.browse(cr, uid, ids)
        res = {}
        for pack in packs:
            res[pack.id] = []
            childs = self.hierarchy_ids(pack)
            for child in childs:
                for serial in child.serial_ids:
                    res[pack.id].append(serial.id)
        return res

    _columns = {
        'parent_id': fields.many2one('stock.tracking', 'Parent'),
        'child_ids': fields.one2many('stock.tracking', 'parent_id', 'Children'),
        'ul_id': fields.many2one('product.ul', 'Logistic unit', readonly=True, states={'open':[('readonly',False)]}),
        'location_id': fields.many2one('stock.location', 'Location', required=True, readonly=True, states={'open':[('readonly',False)]}),
        'state': fields.selection([('open','Open'),('close','Close')], 'State', readonly=True),
        'product_ids': fields.one2many('product.stock.tracking', 'tracking_id', 'Products', readonly=True, states={'open':[('readonly',False)]}),
        'child_product_ids': fields.function(_get_child_products, method=True, type='one2many', obj='product.stock.tracking', string='Child Products'),
        'history_ids': fields.one2many('stock.tracking.history', 'tracking_id', 'History'),
        'current_move_ids': one2many_special('stock.move', 'tracking_id', 'Current moves', domain=[('pack_history_id', '=', False)], readonly=True),
        'name': fields.char('Pack Reference', size=64, required=True, readonly=True, states={'open':[('readonly',False)]}),
        'date': fields.datetime('Creation Date', required=True, readonly=True, states={'open':[('readonly',False)]}),
        'serial_ids': fields.one2many('serial.stock.tracking', 'tracking_id', 'Products', readonly=True, states={'open':[('readonly',False)]}),
        'child_serial_ids': fields.function(_get_child_serials, method=True, type='one2many', obj='serial.stock.tracking', string='Child Serials'),
    }

    def _check_parent_id(self, cr, uid, ids, context=None):
        lines = self.browse(cr, uid, ids, context=context)

        if lines[0].parent_id:
            if lines[0].ul_id.capacity_index > lines[0].parent_id.ul_id.capacity_index:
                return False
        return True

    _constraints = [(_check_parent_id, 'Bad parent type selection. Please try again.',['parent_id'] ),]

    _defaults = {
        'state': 'open',
    }

    def reset_open(self, cr, uid, ids, context=None):
        pack_ids = self.browse(cr, uid, ids, context)
        for pack in pack_ids:
            allowed = True
            if pack.parent_id:
                if pack.parent_id and pack.parent_id != 'open':
                    self.write(cr, uid, [pack.parent_id.id], {'state': 'open'})
#                    allowed = False
#                    raise osv.except_osv(_('Not allowed !'),_('You can\'t re-open this pack because the parent pack is close'))
            if allowed:
                for child in pack.child_ids:
                    if child.state == 'open':
                        allowed = False
                        raise osv.except_osv(_('Not allowed !'),_('You can\'t re-open this pack because there is at least one not closed child'))
                        break
            if allowed:
                self.write(cr, uid, [pack.id], {'state': 'open'})
        return True

    def set_close(self, cr, uid, ids, context=None):
        pack_ids = self.browse(cr, uid, ids, context)
        for pack in pack_ids:
            allowed = True
            for child in pack.child_ids:
                if child.state == 'open':
                    allowed = False
                    raise osv.except_osv(_('Not allowed !'),_('You can\'t close this pack because there is at least one not closed child'))
                    break
#            if allowed:
#                self.write(cr, uid, [pack.id], {'state': 'close'})
        return True

    def get_products(self, cr, uid, ids, context=None):
        pack_ids = self.browse(cr, uid, ids, context)
        stock_track = self.pool.get('product.stock.tracking')
        for pack in pack_ids:
            childs = self.hierarchy_ids(pack)
            for child in childs:
                product_ids = [x.id for x in child.product_ids]
                stock_track.unlink(cr, uid, product_ids)
                product_list = {}
                for x in child.current_move_ids:
                    if x.location_dest_id.id == child.location_id.id:
                        if x.product_id.id not in product_list.keys():
                            product_list.update({x.product_id.id:x.product_qty})
                        else:
                            product_list[x.product_id.id] += x.product_qty
                for product in product_list.keys():
                    stock_track.create(cr, uid, {'product_id': product, 'quantity': product_list[product], 'tracking_id': child.id})
        return True

    def get_serials(self, cr, uid, ids, context=None):
        pack_ids = self.browse(cr, uid, ids, context)
        serial_track = self.pool.get('serial.stock.tracking')
        serial_obj = self.pool.get('stock.production.lot')
        for pack in pack_ids:
            childs = self.hierarchy_ids(pack)
            for child in childs:
                serial_ids = [x.id for x in child.serial_ids]
                serial_track.unlink(cr, uid, serial_ids)
                serial_list = {}
                for x in child.current_move_ids:
                    if x.location_dest_id.id == child.location_id.id:
                        if x.prodlot_id.id not in serial_list.keys():
                            serial_list.update({x.prodlot_id.id:x.product_qty})
                        else:
                            serial_list[x.prodlot_id.id] += x.product_qty
                for serial in serial_list.keys():
                    if serial:
                        serial_track.create(cr, uid, {'serial_id': serial, 'quantity': serial_list[serial], 'tracking_id': child.id})
                        serial_obj.write(cr, uid, [serial], {'tracking_id': child.id})
        return True

stock_tracking()

class product_ul(osv.osv):
    _inherit = "product.ul"
    _description = "Shipping Unit"
    _columns = {
         'capacity_index': fields.integer('Capacity index'),
    }
    _order = 'capacity_index'
product_ul()

class product_stock_tracking(osv.osv):

    _name = 'product.stock.tracking'

    _columns = {
        'product_id': fields.many2one('product.product', 'Product'),
        'quantity': fields.float('Quantity'),
        'tracking_id': fields.many2one('stock.tracking', 'Tracking'),
#        'tracking_history_id': fields.many2one('stock.tracking.history', 'Tracking History'),
    }

product_stock_tracking()

class serial_stock_tracking(osv.osv):

    _name = 'serial.stock.tracking'

    _order = 'tracking_id,serial_id'

    _columns = {
        'serial_id': fields.many2one('stock.production.lot', 'Serial'),
        'product_id': fields.related('serial_id', 'product_id', type='many2one', relation='product.product', string='Product'),
        'quantity': fields.float('Quantity'),
        'tracking_id': fields.many2one('stock.tracking', 'Tracking'),
#        'tracking_history_id': fields.many2one('stock.tracking.history', 'Tracking History'),
    }

serial_stock_tracking()

class stock_tracking_history(osv.osv):

    _name = "stock.tracking.history"

    def _get_types(self, cr, uid, context={}):
#        res = [('pack_in','Add parent'),('pack_out','Unlink parent'),('move','Move')]
        res = []
        return res

#    def hierarchy_ids(self, tracking):
#        result_list = [tracking]
#        for child in tracking.child_ids:
#            result_list.extend(self.hierarchy_ids(child))
#        return result_list
#
#    def _get_child_products(self, cr, uid, ids, field_name, arg, context=None):
#        packs = self.browse(cr, uid, ids)
#        res = {}
#        for pack in packs:
#            res[pack.id] = []
#            childs = self.hierarchy_ids(pack)
#            for child in childs:
#                for prod in child.product_ids:
#                    res[pack.id].append(prod.id)
#        return res

    _columns = {
        'tracking_id': fields.many2one('stock.tracking', 'Pack', required=True),
        'type': fields.selection(_get_types, 'Type'),
#        'product_ids': fields.one2many('product.stock.tracking', 'tracking_history_id', 'Products'),
#        'child_product_ids': fields.function(_get_child_products, method=True, type='one2many', obj='product.stock.tracking', string='Child Products'),
#        'parent_hist_id': fields.many2one('stock.tracking.history', 'Parent history pack'),
#        'child_ids': fields.one2many('stock.tracking.history', 'parent_hist_id', 'Child history pack'),
    }

    _rec_name = "tracking_id"

stock_tracking_history()

'''Add a field in order to store the current pack in a production lot'''
class stock_production_lot(osv.osv):
    _inherit = 'stock.production.lot'
    _columns = {
        'tracking_id': fields.many2one('stock.tracking', 'pack'),
    }

stock_production_lot()

class product_category(osv.osv):
    _inherit = 'product.category'
    _columns = {
        'tracked': fields.boolean('Need a serial code ?'),
    }
product_category()

class stock_inventory(osv.osv):
    _inherit = 'stock.inventory'
    _defaults = {
        'name': lambda x, y, z, c: x.pool.get('ir.sequence').get(y, z, 'stock.inventory') or '/'
    }
stock_inventory()

class stock_move(osv.osv):
    _inherit = 'stock.move'
    _columns = {
         'move_ori_id': fields.many2one('stock.move', 'Origin Move', select=True),
#        'cancel_cascade': fields.boolean('Cancel Cascade', help='If checked, when this move is cancelled, cancel the linked move too')
    }

    def write(self, cr, uid, ids, vals, context=None):
        result = super(stock_move,self).write(cr, uid, ids, vals, context)
        for id in ids:
            state = self.browse(cr, uid, id, context).state
            move_ori_id = self.browse(cr, uid, id, context).move_ori_id
            if state == 'done' and move_ori_id:
                self.write(cr, uid, [move_ori_id], {'state':'done'}, context)
        return result

    def create(self, cr, uid, vals, context=None):
        production_lot_obj = self.pool.get('stock.production.lot')
        stock_tracking_obj = self.pool.get('stock.tracking')
        if vals.get('prodlot_id',False):
            production_lot_data = production_lot_obj.browse(cr, uid, vals['prodlot_id'])
            last_production_lot_move_id = self.search(cr, uid, [('prodlot_id', '=', production_lot_data.id)], limit=1, order='date')
            if last_production_lot_move_id:
                last_production_lot_move = self.browse(cr,uid,last_production_lot_move_id[0])
                if last_production_lot_move.tracking_id:
                    ids = [last_production_lot_move.tracking_id.id]
                    stock_tracking_obj.reset_open(cr, uid, ids, context=None)

        return super(stock_move,self).create(cr, uid, vals, context)

stock_move()

class split_in_production_lot(osv.osv_memory):
    _inherit = "stock.move.split"
    _columns = {
        'use_exist' : fields.boolean('Existing Lots', invisible=True),
     }
    _defaults = {
        'use_exist': lambda *a: True,
    }
    def default_get(self, cr, uid, fields, context=None):
        res = super(split_in_production_lot, self).default_get(cr, uid, fields, context)
        res.update({'use_exist': True})
        return res

split_in_production_lot()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>"""Deployement."""
import logging
import os
import subprocess  # NOQA

# pylint: disable=relative-import
try:
    import config
    import curl_requests
    import misc
    import package
except ImportError:
    from . import config
    from . import curl_requests
    from . import misc
    from . import package
# pylint: enable=relative-import

LOG = logging.getLogger(__name__)

class LoopDeployment(object):
    """Contains attributes relating to deployment of packages locally."""
    def __init__(self):
        # These are used for statistics. Initialise them with '0' (int).
        self._download_size = 0
        self._downloaded_size = 0

        self._install_size = 0

    def _upd_download_size(self, size):
        """Updates the 'download_size' attribute by the specified size."""
        if isinstance(size, int):
            self._download_size += size

    def _upd_downloaded_size(self, size):
        """Updates the 'downloaded_size' attribute by the specified size."""
        if isinstance(size, int):
            self._downloaded_size += size

    def _upd_install_size(self, size):
        """Updates the 'install_size' attribute by the specified size."""
        if isinstance(size, int):
            self._install_size += size

    # pylint: disable=no-self-use
    # pylint: disable=inconsistent-return-statements
    def _download(self, pkg, counter_msg):
        """Downloads a package from the specified URL."""
        if isinstance(pkg, package.LoopPackage):
            _url = pkg.DownloadURL
            _cache_race = False  # Presume all caching server packages are completely downloaded
            _debug_msg = 'Fell back {} to {}'.format(_url, pkg.DownloadURL)

            curl = curl_requests.CURL()

            if pkg.LocalDownloadURL:
                _url = pkg.LocalDownloadURL
            elif pkg.CacheDownloadURL:
                _url = pkg.CacheDownloadURL

            # Get the status of the URL to see if it exists
            req = curl_requests.CURL(url=_url)

            # Check if a caching server package is less than the expected size,
            # if this is true, then it's likely the caching server hasn't completely
            # downloaded the file yet, and will cause problems, so fall back.
            if config.CACHING_SERVER:
                try:
                    if config.REAL_DOWNLOAD_SIZE:
                        _cache_race = req.headers['Content-Length'] < pkg.RealDownloadSize
                    else:
                        _cache_race = req.headers['Content-Length'] < pkg.DownloadSize
                except KeyError:
                    if config.REAL_DOWNLOAD_SIZE:
                        _cache_race = req.headers['content-length'] < pkg.RealDownloadSize
                    else:
                        _cache_race = req.headers['content-length'] < pkg.DownloadSize

                if _cache_race:
                    _debug_msg = '{} (Possible Caching Server race condition when downloading package)'.format(_debug_msg)

            if req.status:
                if req.status in config.HTTP_OK_STATUS:
                    curl.get(url=_url, output=pkg.DownloadPath, counter_msg=counter_msg)
                elif req.status not in config.HTTP_OK_STATUS:
                    # Fallback only if the url is either a cache or pkg server
                    if _url in [pkg.LocalDownloadURL, pkg.CacheDownloadURL] or _cache_race:
                        LOG.debug(_debug_msg)

                        _url = pkg.DownloadURL

                        curl.get(url=_url, output=pkg.DownloadPath, counter_msg=counter_msg)
            elif not req.status or req.curl_error:
                # Fallback only if the url is either a cache or pkg server
                if _url in [pkg.LocalDownloadURL, pkg.CacheDownloadURL] or _cache_race:
                    LOG.debug(_debug_msg)

                    _url = pkg.DownloadURL

                    curl.get(url=_url, output=pkg.DownloadPath, counter_msg=counter_msg)
        else:
            LOG.debug('{} is {}'.format(pkg, pkg.__class__))
            return NotImplemented
    # pylint: enable=inconsistent-return-statements

    def _installer(self, cmd):
        """'installer' command execution."""
        result = None
        msg = None

        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        p_result, p_error = process.communicate()

        if process.returncode == 0:
            msg = '  Installed _PKG_'
            LOG.info('{}: {}'.format(' '.join(cmd), p_result))
        else:
            msg = (' Error installing _PKG_. \'/var/log/install.log\' may include additional'
                   ' information.')
            LOG.debug('{}: {}'.format(' '.join(cmd), p_error))

        result = (process.returncode, msg)

        return result

    def _install(self, pkg, counter_msg):
        """Installs a package."""
        result = None
        filename = os.path.join(config.DESTINATION_PATH, pkg.DownloadPath)

        cmd = ['/usr/sbin/installer', '-pkg', filename, '-target', config.TARGET]

        if config.ALLOW_UNSECURE_PKGS:
            cmd.insert(1, '-allowUntrusted')  # Insert at index 1, shifts right

        if config.DRY_RUN:
            _msg = ' '.join(cmd)
            print('  {}'.format(_msg))
            LOG.info(_msg)
        else:
            if os.path.exists(filename):
                if not config.SILENT:
                    print('Installing {} - {}'.format(counter_msg, pkg.DownloadName))

                install_result = self._installer(cmd=cmd)
                msg = '{}'.format(install_result[1]).replace('_PKG_', pkg.DownloadName)
                result = True if install_result[0] == 0 else False

                if not config.SILENT:
                    print(msg)
            else:
                LOG.info('File not found: {}'.format(filename))
                if not config.SILENT:
                    print('File not found: {}'.format(filename))

        return result
    # pylint: enable=no-self-use

    def process(self, pkg, counter_msg):
        """Processes the download/install of packages."""
        if not config.HTTP_DMG:
            self._download(pkg=pkg, counter_msg=counter_msg)

        if config.DEPLOY_PKGS or config.FORCED_DEPLOYMENT:
            self._install(pkg=pkg, counter_msg=counter_msg)

            if not config.DRY_RUN:
                # Don't try and delete from DMG.
                if not config.HTTP_DMG:
                    misc.clean_up(file_path=pkg.DownloadPath)
<commit_msg>Include flag to dump log data to stderr for debug processes<commit_after>"""Deployement."""
import logging
import os
import subprocess  # NOQA

# pylint: disable=relative-import
try:
    import config
    import curl_requests
    import misc
    import package
except ImportError:
    from . import config
    from . import curl_requests
    from . import misc
    from . import package
# pylint: enable=relative-import

LOG = logging.getLogger(__name__)

class LoopDeployment(object):
    """Contains attributes relating to deployment of packages locally."""
    def __init__(self):
        # These are used for statistics. Initialise them with '0' (int).
        self._download_size = 0
        self._downloaded_size = 0

        self._install_size = 0

    def _upd_download_size(self, size):
        """Updates the 'download_size' attribute by the specified size."""
        if isinstance(size, int):
            self._download_size += size

    def _upd_downloaded_size(self, size):
        """Updates the 'downloaded_size' attribute by the specified size."""
        if isinstance(size, int):
            self._downloaded_size += size

    def _upd_install_size(self, size):
        """Updates the 'install_size' attribute by the specified size."""
        if isinstance(size, int):
            self._install_size += size

    # pylint: disable=no-self-use
    # pylint: disable=inconsistent-return-statements
    def _download(self, pkg, counter_msg):
        """Downloads a package from the specified URL."""
        if isinstance(pkg, package.LoopPackage):
            _url = pkg.DownloadURL
            _cache_race = False  # Presume all caching server packages are completely downloaded
            _debug_msg = 'Fell back {} to {}'.format(_url, pkg.DownloadURL)

            curl = curl_requests.CURL()

            if pkg.LocalDownloadURL:
                _url = pkg.LocalDownloadURL
            elif pkg.CacheDownloadURL:
                _url = pkg.CacheDownloadURL

            # Get the status of the URL to see if it exists
            req = curl_requests.CURL(url=_url)

            # Check if a caching server package is less than the expected size,
            # if this is true, then it's likely the caching server hasn't completely
            # downloaded the file yet, and will cause problems, so fall back.
            if config.CACHING_SERVER:
                try:
                    if config.REAL_DOWNLOAD_SIZE:
                        _cache_race = req.headers['Content-Length'] < pkg.RealDownloadSize
                    else:
                        _cache_race = req.headers['Content-Length'] < pkg.DownloadSize
                except KeyError:
                    if config.REAL_DOWNLOAD_SIZE:
                        _cache_race = req.headers['content-length'] < pkg.RealDownloadSize
                    else:
                        _cache_race = req.headers['content-length'] < pkg.DownloadSize

                if _cache_race:
                    _debug_msg = '{} (Possible Caching Server race condition when downloading package)'.format(_debug_msg)

            if req.status:
                if req.status in config.HTTP_OK_STATUS:
                    curl.get(url=_url, output=pkg.DownloadPath, counter_msg=counter_msg)
                elif req.status not in config.HTTP_OK_STATUS:
                    # Fallback only if the url is either a cache or pkg server
                    if _url in [pkg.LocalDownloadURL, pkg.CacheDownloadURL] or _cache_race:
                        LOG.debug(_debug_msg)

                        _url = pkg.DownloadURL

                        curl.get(url=_url, output=pkg.DownloadPath, counter_msg=counter_msg)
            elif not req.status or req.curl_error:
                # Fallback only if the url is either a cache or pkg server
                if _url in [pkg.LocalDownloadURL, pkg.CacheDownloadURL] or _cache_race:
                    LOG.debug(_debug_msg)

                    _url = pkg.DownloadURL

                    curl.get(url=_url, output=pkg.DownloadPath, counter_msg=counter_msg)
        else:
            LOG.debug('{} is {}'.format(pkg, pkg.__class__))
            return NotImplemented
    # pylint: enable=inconsistent-return-statements

    def _installer(self, cmd):
        """'installer' command execution."""
        result = None
        msg = None

        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        p_result, p_error = process.communicate()

        if process.returncode == 0:
            msg = '  Installed _PKG_'
            LOG.info('{}: {}'.format(' '.join(cmd), p_result))

            # Debug log installer '-dumplog' value.
            try:
                _installer_dumplog_msg = p_error.decode('utf-8')
            except Exception:
                _installer_dumplog_msg = p_error

            LOG.debug(_installer_dumplog_msg)
        else:
            msg = (' Error installing _PKG_. \'/var/log/install.log\' may include additional'
                   ' information.')
            LOG.debug('{}: {}'.format(' '.join(cmd), p_error))

        result = (process.returncode, msg)

        return result

    def _install(self, pkg, counter_msg):
        """Installs a package."""
        result = None
        filename = os.path.join(config.DESTINATION_PATH, pkg.DownloadPath)

        cmd = ['/usr/sbin/installer', '-dumplog', '-pkg', filename, '-target', config.TARGET]

        if config.ALLOW_UNSECURE_PKGS:
            cmd.insert(1, '-allowUntrusted')  # Insert at index 1, shifts right

        if config.DRY_RUN:
            _msg = ' '.join(cmd)
            print('  {}'.format(_msg))
            LOG.info(_msg)
        else:
            if os.path.exists(filename):
                if not config.SILENT:
                    print('Installing {} - {}'.format(counter_msg, pkg.DownloadName))

                install_result = self._installer(cmd=cmd)
                msg = '{}'.format(install_result[1]).replace('_PKG_', pkg.DownloadName)
                result = True if install_result[0] == 0 else False

                if not config.SILENT:
                    print(msg)
            else:
                LOG.info('File not found: {}'.format(filename))
                if not config.SILENT:
                    print('File not found: {}'.format(filename))

        return result
    # pylint: enable=no-self-use

    def process(self, pkg, counter_msg):
        """Processes the download/install of packages."""
        if not config.HTTP_DMG:
            self._download(pkg=pkg, counter_msg=counter_msg)

        if config.DEPLOY_PKGS or config.FORCED_DEPLOYMENT:
            self._install(pkg=pkg, counter_msg=counter_msg)

            if not config.DRY_RUN:
                # Don't try and delete from DMG.
                if not config.HTTP_DMG:
                    misc.clean_up(file_path=pkg.DownloadPath)
<|endoftext|>


<commit_before>from mailserver import resolvers
from mailserver.exceptions import *
from mailserver import EmailResponse, EmailIgnoreResponse
from threading import Lock
import logging
import sys

class BaseMessageHandler(object):
    initLock = Lock()

    def __init__(self):
        self._request_middleware = self._view_middleware = self._response_middleware = self._exception_middleware = None

    def __call__(self, environ, request):
        resolver = None
        try:
            mailconf = getattr(request, "mailconf", get_setting('ROOT_MAILCONF'))
            resolver = resolvers.RegexMailResolver(r'', mailconf)
            response = self.call(resolver, environ, request)
            return response
        except SystemExit:
            raise
        except:
            exc_info = sys.exc_info()
            exc = exc_info[1]
            if hasattr(exc, 'get_response'):
                response = exc.get_response(request)
                if response is not None:
                    return response
            self.handle_uncaught_exception(request, resolver, exc_info)
            raise DeliveryError("Unhandled temporary error")

    def call(self, resolver, environ, request):
        if self._request_middleware is None:
            self.initLock.acquire()
            if self._request_middleware is None:
                self.load_middleware()
            self.initLock.release()
        resp = self.get_response(resolver, request)
        # Apply response middleware
        for middleware_method in self._response_middleware:
            resp = middleware_method(request, resp)
        # Need to handle when the reply is to be sent
        # Or needs to not mark-as-delivered
        if isinstance(resp, EmailIgnoreResponse):
            pass
        elif isinstance(resp, EmailResponse):
            # Some kind of normal response, we send to the user.
            resp.send()
        else:
            raise ValueError, "Do not know how to handle response %s" % resp
        return resp

    def __repr__(self):
        return self.__class__.__name__

    def load_middleware(self):
        """
        Populate middleware lists from settings.MAIL_MIDDLEWARE_CLASSES.
        """
        from django.core import exceptions
        from mailserver import get_setting
        self._view_middleware = []
        self._response_middleware = []
        self._exception_middleware = []

        request_middleware = []
        for middleware_path in get_setting('MAIL_MIDDLEWARE'):
            try:
                dot = middleware_path.rindex('.')
            except ValueError:
                raise exceptions.ImproperlyConfigured, '%s isn\'t a middleware module' % middleware_path
            mw_module, mw_classname = middleware_path[:dot], middleware_path[dot+1:]
            try:
                mod = __import__(mw_module, {}, {}, [''])
            except ImportError, e:
                raise exceptions.ImproperlyConfigured, 'Error importing middleware %s: "%s"' % (mw_module, e)
            try:
                mw_class = getattr(mod, mw_classname)
            except AttributeError:
                raise exceptions.ImproperlyConfigured, 'Middleware module "%s" does not define a "%s" class' % (mw_module, mw_classname)

            try:
                mw_instance = mw_class()
            except exceptions.MiddlewareNotUsed:
                continue

            if hasattr(mw_instance, 'process_request'):
                request_middleware.append(mw_instance.process_request)
            if hasattr(mw_instance, 'process_view'):
                self._view_middleware.append(mw_instance.process_view)
            if hasattr(mw_instance, 'process_response'):
                self._response_middleware.insert(0, mw_instance.process_response)
            if hasattr(mw_instance, 'process_exception'):
                self._exception_middleware.insert(0, mw_instance.process_exception)

        # We only assign to this when initialization is complete as it is used
        # as a flag for initialization being complete.
        self._request_middleware = request_middleware

    def get_response(self, resolver, request):
        from django.core import exceptions
        from mailserver import get_setting

        # Apply request middleware
        for middleware_method in self._request_middleware:
            response = middleware_method(request)
            if response:
                return response

        callback, callback_args, callback_kwargs = \
            resolver.resolve(request.get_recipient_address())
        # Apply view middleware
        for middleware_method in self._view_middleware:
            response = middleware_method(request, callback, callback_args, callback_kwargs)
            if response:
                return response

        try:
            response = callback(request, *callback_args, **callback_kwargs)
        except Exception, e:
            for middleware_method in self._exception_middleware:
                response = middleware_method(request, e)
                if response:
                    return response
            raise
        # Complain if the view returned None (a common error).
        if response is None:
            try:
                view_name = callback.func_name
            except AttributeError:
                view_name = callback.__class__.__name__ + '.__call__'
            raise ValueError, "The view %s.%s didn't return a meaningfull response" % (callback.__module__, view_name)
        return response

    def handle_uncaught_exception(self, request, resolver, exc_info):
        from django.conf import settings
        from django.core.mail import mail_admins

        if settings.DEBUG_PROPAGATE_EXCEPTIONS or settings.DEBUG:
            raise

        # When DEBUG is False, send an error message to the admins.
        subject = 'Mail Error (%s)' % request['From'][1]
        try:
            request_repr = repr(request)
        except:
            request_repr = "Request repr() unavailable"
        message = "%s\n\n%s" % (self._get_traceback(exc_info), request_repr)
        mail_admins(subject, message, fail_silently=True)

    def _get_traceback(self, exc_info=None):
        "Helper function to return the traceback as a string"
        import traceback
        return '\n'.join(traceback.format_exception(*(exc_info or sys.exc_info())))

<commit_msg>Send responses that are not meant to be ignored.<commit_after>from mailserver import resolvers
from mailserver.exceptions import *
from mailserver import EmailResponse, EmailIgnoreResponse
from threading import Lock
import logging
import sys

class BaseMessageHandler(object):
    initLock = Lock()

    def __init__(self):
        self._request_middleware = self._view_middleware = self._response_middleware = self._exception_middleware = None

    def __call__(self, environ, request):
        resolver = None
        try:
            mailconf = getattr(request, "mailconf", get_setting('ROOT_MAILCONF'))
            resolver = resolvers.RegexMailResolver(r'', mailconf)
            response = self.call(resolver, environ, request)
            return self.handle_response(response)
        except SystemExit:
            raise
        except:
            exc_info = sys.exc_info()
            exc = exc_info[1]
            if hasattr(exc, 'get_response'):
                response = exc.get_response(request)
                if response is not None:
                    return self.handle_response(response)
            self.handle_uncaught_exception(request, resolver, exc_info)
            raise DeliveryError("Unhandled temporary error")

    def call(self, resolver, environ, request):
        if self._request_middleware is None:
            self.initLock.acquire()
            if self._request_middleware is None:
                self.load_middleware()
            self.initLock.release()
        resp = self.get_response(resolver, request)
        # Apply response middleware
        for middleware_method in self._response_middleware:
            resp = middleware_method(request, resp)
        # Need to handle when the reply is to be sent
        # Or needs to not mark-as-delivered
        if isinstance(resp, EmailIgnoreResponse):
            pass
        elif isinstance(resp, EmailResponse):
            # Some kind of normal response, we send to the user.
            resp.send()
        else:
            raise ValueError, "Do not know how to handle response %s" % resp
        return resp

    def __repr__(self):
        return self.__class__.__name__

    def load_middleware(self):
        """
        Populate middleware lists from settings.MAIL_MIDDLEWARE_CLASSES.
        """
        from django.core import exceptions
        from mailserver import get_setting
        self._view_middleware = []
        self._response_middleware = []
        self._exception_middleware = []

        request_middleware = []
        for middleware_path in get_setting('MAIL_MIDDLEWARE'):
            try:
                dot = middleware_path.rindex('.')
            except ValueError:
                raise exceptions.ImproperlyConfigured, '%s isn\'t a middleware module' % middleware_path
            mw_module, mw_classname = middleware_path[:dot], middleware_path[dot+1:]
            try:
                mod = __import__(mw_module, {}, {}, [''])
            except ImportError, e:
                raise exceptions.ImproperlyConfigured, 'Error importing middleware %s: "%s"' % (mw_module, e)
            try:
                mw_class = getattr(mod, mw_classname)
            except AttributeError:
                raise exceptions.ImproperlyConfigured, 'Middleware module "%s" does not define a "%s" class' % (mw_module, mw_classname)

            try:
                mw_instance = mw_class()
            except exceptions.MiddlewareNotUsed:
                continue

            if hasattr(mw_instance, 'process_request'):
                request_middleware.append(mw_instance.process_request)
            if hasattr(mw_instance, 'process_view'):
                self._view_middleware.append(mw_instance.process_view)
            if hasattr(mw_instance, 'process_response'):
                self._response_middleware.insert(0, mw_instance.process_response)
            if hasattr(mw_instance, 'process_exception'):
                self._exception_middleware.insert(0, mw_instance.process_exception)

        # We only assign to this when initialization is complete as it is used
        # as a flag for initialization being complete.
        self._request_middleware = request_middleware

    def get_response(self, resolver, request):
        from django.core import exceptions
        from mailserver import get_setting

        # Apply request middleware
        for middleware_method in self._request_middleware:
            response = middleware_method(request)
            if response:
                return response

        callback, callback_args, callback_kwargs = \
            resolver.resolve(request.get_recipient_address())
        # Apply view middleware
        for middleware_method in self._view_middleware:
            response = middleware_method(request, callback, callback_args, callback_kwargs)
            if response:
                return response

        try:
            response = callback(request, *callback_args, **callback_kwargs)
        except Exception, e:
            for middleware_method in self._exception_middleware:
                response = middleware_method(request, e)
                if response:
                    return response
            raise
        # Complain if the view returned None (a common error).
        if response is None:
            try:
                view_name = callback.func_name
            except AttributeError:
                view_name = callback.__class__.__name__ + '.__call__'
            raise ValueError, "The view %s.%s didn't return a meaningfull response" % (callback.__module__, view_name)
        return response

    def handle_response(self, response):
        if not issubclass(response.__class__, EmailIgnoreResponse):
            response.send()
        return response

    def handle_uncaught_exception(self, request, resolver, exc_info):
        from django.conf import settings
        from django.core.mail import mail_admins

        if settings.DEBUG_PROPAGATE_EXCEPTIONS or settings.DEBUG:
            raise

        # When DEBUG is False, send an error message to the admins.
        subject = 'Mail Error (%s)' % request['From'][1]
        try:
            request_repr = repr(request)
        except:
            request_repr = "Request repr() unavailable"
        message = "%s\n\n%s" % (self._get_traceback(exc_info), request_repr)
        mail_admins(subject, message, fail_silently=True)

    def _get_traceback(self, exc_info=None):
        "Helper function to return the traceback as a string"
        import traceback
        return '\n'.join(traceback.format_exception(*(exc_info or sys.exc_info())))

<|endoftext|>


<commit_before>from mailserver import resolvers
from mailserver.exceptions import *
from mailserver import EmailResponse, EmailResponseServer
from mailserver import EmailIgnoreResponse
from threading import Lock
import logging
import sys

class BaseMessageHandler(object):
    initLock = Lock()

    def __init__(self):
        self._request_middleware = self._view_middleware = self._response_middleware = self._exception_middleware = None

    def __call__(self, environ, request):
        logging.debug("processing message %s" % request)
        if self._request_middleware is None:
            self.initLock.acquire()
            if self._request_middleware is None:
                self.load_middleware()
            self.initLock.release()
        resp = self.get_response(request)
        # Need to handle when the reply is to be sent
        # Or needs to not mark-as-delivered
        if isinstance(resp, EmailResponseServer):
            # Some king of error we have to tell the MTA about.
            # (User gets nothing (from our part))
            return resp
        elif isinstance(resp, EmailIgnoreResponse):
            pass
        elif isinstance(resp, EmailResponse):
            # Some kind of normal response, we send to the user.
            resp.send()
        else:
            raise ValueError, "Do not know how to handle response %s" % resp
        return None

    def __repr__(self):
        return self.__class__.__name__

    def load_middleware(self):
        """
        Populate middleware lists from settings.MAIL_MIDDLEWARE_CLASSES.
        """
        from django.core import exceptions
        from mailserver import get_setting
        self._view_middleware = []
        self._response_middleware = []
        self._exception_middleware = []

        request_middleware = []
        for middleware_path in get_setting('MAIL_MIDDLEWARE'):
            try:
                dot = middleware_path.rindex('.')
            except ValueError:
                raise exceptions.ImproperlyConfigured, '%s isn\'t a middleware module' % middleware_path
            mw_module, mw_classname = middleware_path[:dot], middleware_path[dot+1:]
            try:
                mod = __import__(mw_module, {}, {}, [''])
            except ImportError, e:
                raise exceptions.ImproperlyConfigured, 'Error importing middleware %s: "%s"' % (mw_module, e)
            try:
                mw_class = getattr(mod, mw_classname)
            except AttributeError:
                raise exceptions.ImproperlyConfigured, 'Middleware module "%s" does not define a "%s" class' % (mw_module, mw_classname)

            try:
                mw_instance = mw_class()
            except exceptions.MiddlewareNotUsed:
                continue

            if hasattr(mw_instance, 'process_request'):
                request_middleware.append(mw_instance.process_request)
            if hasattr(mw_instance, 'process_view'):
                self._view_middleware.append(mw_instance.process_view)
            if hasattr(mw_instance, 'process_response'):
                self._response_middleware.insert(0, mw_instance.process_response)
            if hasattr(mw_instance, 'process_exception'):
                self._exception_middleware.insert(0, mw_instance.process_exception)

        # We only assign to this when initialization is complete as it is used
        # as a flag for initialization being complete.
        self._request_middleware = request_middleware

    def get_response(self, request):
        from django.core import exceptions
        from mailserver import get_setting

        # Apply request middleware
        for middleware_method in self._request_middleware:
            response = middleware_method(request)
            if response:
                return response

        mailconf = getattr(request, "mailconf", get_setting('ROOT_MAILCONF'))
        resolver = resolvers.RegexMailResolver(r'', mailconf)
        try:
            callback, callback_args, callback_kwargs = \
                resolver.resolve(request.get_recipient_address())
            # Apply view middleware
            for middleware_method in self._view_middleware:
                response = middleware_method(request, callback, callback_args, callback_kwargs)
                if response:
                    return response

            try:
                response = callback(request, *callback_args, **callback_kwargs)
            except Exception, e:
                for middleware_method in self._exception_middleware:
                    response = middleware_method(request, e)
                    if response:
                        return response
                raise
            # Complain if the view returned None (a common error).
            if response is None:
                try:
                    view_name = callback.func_name
                except AttributeError:
                    view_name = callback.__class__.__name__ + '.__call__'
                raise ValueError, "The view %s.%s didn't return a meaningfull response" % (callback.__module__, view_name)
            return response
        except exceptions.PermissionDenied:
            # How should we handle this?
            raise
        except resolvers.Resolver404, e:
            try:
                callback, param_dict = resolver.resolve404()
                return callback(request, **param_dict)
            except:
                return self.handle_uncaught_exception(
                    request, resolver, sys.exc_info())
        except SystemExit:
            raise
        except:
            exc_info = sys.exc_info()
            return self.handle_uncaught_exception(request, resolver, exc_info)

    def handle_uncaught_exception(self, request, resolver, exc_info):
        from django.conf import settings
        from django.core.mail import mail_admins

        if settings.DEBUG_PROPAGATE_EXCEPTIONS:
            raise

        # When DEBUG is False, send an error message to the admins.
        subject = 'Mail Error (%s)' % request['From'][1]
        try:
            request_repr = repr(request)
        except:
            request_repr = "Request repr() unavailable"
        message = "%s\n\n%s" % (self._get_traceback(exc_info), request_repr)
        mail_admins(subject, message, fail_silently=True)
        # Return an HttpResponse that displays a friendly error message.
        callback, param_dict = resolver.resolve500()
        return callback(request, **param_dict)

    def _get_traceback(self, exc_info=None):
        "Helper function to return the traceback as a string"
        import traceback
        return '\n'.join(traceback.format_exception(*(exc_info or sys.exc_info())))

<commit_msg>We want to provide the Response object from BaseMessageHandler even if it is to be ignored later.<commit_after>from mailserver import resolvers
from mailserver.exceptions import *
from mailserver import EmailResponse, EmailResponseServer
from mailserver import EmailIgnoreResponse
from threading import Lock
import logging
import sys

class BaseMessageHandler(object):
    initLock = Lock()

    def __init__(self):
        self._request_middleware = self._view_middleware = self._response_middleware = self._exception_middleware = None

    def __call__(self, environ, request):
        logging.debug("processing message %s" % request)
        if self._request_middleware is None:
            self.initLock.acquire()
            if self._request_middleware is None:
                self.load_middleware()
            self.initLock.release()
        resp = self.get_response(request)
        # Need to handle when the reply is to be sent
        # Or needs to not mark-as-delivered
        if isinstance(resp, EmailResponseServer):
            # Some king of error we have to tell the MTA about.
            # (User gets nothing (from our part))
            return resp
        elif isinstance(resp, EmailIgnoreResponse):
            pass
        elif isinstance(resp, EmailResponse):
            # Some kind of normal response, we send to the user.
            resp.send()
        else:
            raise ValueError, "Do not know how to handle response %s" % resp
        return resp

    def __repr__(self):
        return self.__class__.__name__

    def load_middleware(self):
        """
        Populate middleware lists from settings.MAIL_MIDDLEWARE_CLASSES.
        """
        from django.core import exceptions
        from mailserver import get_setting
        self._view_middleware = []
        self._response_middleware = []
        self._exception_middleware = []

        request_middleware = []
        for middleware_path in get_setting('MAIL_MIDDLEWARE'):
            try:
                dot = middleware_path.rindex('.')
            except ValueError:
                raise exceptions.ImproperlyConfigured, '%s isn\'t a middleware module' % middleware_path
            mw_module, mw_classname = middleware_path[:dot], middleware_path[dot+1:]
            try:
                mod = __import__(mw_module, {}, {}, [''])
            except ImportError, e:
                raise exceptions.ImproperlyConfigured, 'Error importing middleware %s: "%s"' % (mw_module, e)
            try:
                mw_class = getattr(mod, mw_classname)
            except AttributeError:
                raise exceptions.ImproperlyConfigured, 'Middleware module "%s" does not define a "%s" class' % (mw_module, mw_classname)

            try:
                mw_instance = mw_class()
            except exceptions.MiddlewareNotUsed:
                continue

            if hasattr(mw_instance, 'process_request'):
                request_middleware.append(mw_instance.process_request)
            if hasattr(mw_instance, 'process_view'):
                self._view_middleware.append(mw_instance.process_view)
            if hasattr(mw_instance, 'process_response'):
                self._response_middleware.insert(0, mw_instance.process_response)
            if hasattr(mw_instance, 'process_exception'):
                self._exception_middleware.insert(0, mw_instance.process_exception)

        # We only assign to this when initialization is complete as it is used
        # as a flag for initialization being complete.
        self._request_middleware = request_middleware

    def get_response(self, request):
        from django.core import exceptions
        from mailserver import get_setting

        # Apply request middleware
        for middleware_method in self._request_middleware:
            response = middleware_method(request)
            if response:
                return response

        mailconf = getattr(request, "mailconf", get_setting('ROOT_MAILCONF'))
        resolver = resolvers.RegexMailResolver(r'', mailconf)
        try:
            callback, callback_args, callback_kwargs = \
                resolver.resolve(request.get_recipient_address())
            # Apply view middleware
            for middleware_method in self._view_middleware:
                response = middleware_method(request, callback, callback_args, callback_kwargs)
                if response:
                    return response

            try:
                response = callback(request, *callback_args, **callback_kwargs)
            except Exception, e:
                for middleware_method in self._exception_middleware:
                    response = middleware_method(request, e)
                    if response:
                        return response
                raise
            # Complain if the view returned None (a common error).
            if response is None:
                try:
                    view_name = callback.func_name
                except AttributeError:
                    view_name = callback.__class__.__name__ + '.__call__'
                raise ValueError, "The view %s.%s didn't return a meaningfull response" % (callback.__module__, view_name)
            return response
        except exceptions.PermissionDenied:
            # How should we handle this?
            raise
        except resolvers.Resolver404, e:
            try:
                callback, param_dict = resolver.resolve404()
                return callback(request, **param_dict)
            except:
                return self.handle_uncaught_exception(
                    request, resolver, sys.exc_info())
        except SystemExit:
            raise
        except:
            exc_info = sys.exc_info()
            return self.handle_uncaught_exception(request, resolver, exc_info)

    def handle_uncaught_exception(self, request, resolver, exc_info):
        from django.conf import settings
        from django.core.mail import mail_admins

        if settings.DEBUG_PROPAGATE_EXCEPTIONS:
            raise

        # When DEBUG is False, send an error message to the admins.
        subject = 'Mail Error (%s)' % request['From'][1]
        try:
            request_repr = repr(request)
        except:
            request_repr = "Request repr() unavailable"
        message = "%s\n\n%s" % (self._get_traceback(exc_info), request_repr)
        mail_admins(subject, message, fail_silently=True)
        # Return an HttpResponse that displays a friendly error message.
        callback, param_dict = resolver.resolve500()
        return callback(request, **param_dict)

    def _get_traceback(self, exc_info=None):
        "Helper function to return the traceback as a string"
        import traceback
        return '\n'.join(traceback.format_exception(*(exc_info or sys.exc_info())))

<|endoftext|>