<commit_before># -*- coding: utf-8 -*-
from .pd_base import BaseExtClass, api, OutputHelper as Console, Jesie

class ProductProduct(BaseExtClass):
    _inherit = "product.product"

    packet_size = 500

    def is_notifiable(self):
        return self.product_tmpl_id.type == "product" and self.tag_ids and self.default_code.find('False') == -1

    fields_to_watch = ('id', 'default_code', 'name', 'barcode', 'description', 'web_global_stock',
                       'product_brand_id', 'attribute_line_ids', 'tag_ids', 'active', 'type', 'force_web')

    def set_props(self, unlink=False):
        poitem = """
                <poitem>
                  <ItemCode>{}</ItemCode>
                  <ItemName>{}</ItemName>
                  <CodeBars>{}</CodeBars>
                  <UserText>{}</UserText>
                  <Model>{}</Model>
                  <Tallaje>{}</Tallaje>
                  <Talla>{}</Talla>
                  <Color>{}</Color>
                  <Stock>{}</Stock>
                  <SWeight1>{}</SWeight1>
                  <FirmName>{}</FirmName>
                  <Price>{}</Price>
                  <ItmsGrpCod>{}</ItmsGrpCod>
                  <validFor>{}</validFor>
                </poitem>
              """

        def get_tallaje_talla_color():
            """ En base a lo hablado con Miguel, los atributos o son Color (y sólo debe haber uno) o tallaje.

                Con esta función obtenemos:
                    - el id del primer atributo de la plantilla que NO sea Color
                    - el id de la talla del tallaje obtenido
                    - el id del color
            """

            tallajeid = None
            tallajecode = None
            tallacode = None
            colorcode = None

            for attr in self.product_tmpl_id.attribute_line_ids:
                if not attr.attribute_id.is_color:
                    tallajecode = attr.attribute_id.legacy_code or '#' + str(attr.attribute_id.id)
                    tallajeid = attr.attribute_id.id
                    break

            for val in self.attribute_value_ids:
                if val.attribute_id.id == tallajeid:
                    tallacode = val.legacy_code or '#' + str(val.id)
                elif val.attribute_id.is_color:
                    colorcode = val.legacy_code or '#' + str(val.id)

                if tallacode and colorcode:
                    break

            return tallajecode, tallacode, colorcode

        if unlink or (self.force_web == 'no') or not self.active:
            is_active = 'N'
        else:
            is_active = 'N'
            for tag in self.tag_ids:
                if tag.web:
                    is_active = 'Y'
            # is_active = 'Y' if self.active else 'N'

        tallaje, talla, color = get_tallaje_talla_color()

        self.xml = poitem.format(
            self.default_code or self.id,                                   # ItemCode
            self.name,                                                      # ItemName
            self.barcode or '',                                             # CodeBars
            self.description or '',                                         # UserText
            self.product_tmpl_id.template_code or self.product_tmpl_id.default_code or self.product_tmpl_id.id,  # Model
            tallaje or '',                                                  # Tallaje
            talla or '',                                                    # Talla
            color or '',                                                    # Color
            self.web_global_stock or '0.0',                                 # Stock
            self.weight or '0.0',                                           # Sweight1
            self.product_brand_id.name or '',                               # FirmName
            self.price or '0.0',                                            # Price
            ";".join(tag.legacy_code or '#' + str(tag.id) for tag in self.tag_ids),  # ItmsGrpCod
            is_active                                                       # validFor
        )

        self.obj_type = '4'

    def get_stock_packet_size(self):
        packet_size = self.env["ir.config_parameter"].search([('key', '=', 'prestadoo.stock.packet.size')])

        if not packet_size:
            Console.debug("[Prestadoo] No se ha encontrado parámetro 'prestadoo.stock.packet.size' en 'Configuración "
                          "-> Parámetros -> Parámetros del sistema'; se utilizará 500.")
            self.packet_size = 500
        else:
            self.packet_size = int(packet_size.value)
            Console.debug("[Prestadoo] Encontrado parámetro 'prestadoo.stock.packet.size'. "
                          "Valor = {}".format(packet_size.value))

    @api.model
    def stock_2_jesie(self):
        Console.debug('[Prestadoo] Inicio de proceso de notificación de STOCK a Jesie')

        self.get_stock_packet_size()

        poitemstock = """
                        <poitemstock>
                          <ItemCode>{}</ItemCode>
                          <Stock>{}</Stock>
                        </poitemstock>
                    """

        product_list = self.search([('active', '=', True),
                                    ('sale_ok', '=', True),
                                    ('default_code', '!=', False),
                                    # ('default_code', 'not like', '%ñ%'),
                                    # ('default_code', 'not like', '%Ñ%'),
                                    ('type', '=', 'product')],
                                   order='id')

        msg_count = 0
        poitemstock_xml_list = []

        debug_sum = 0
        debug_total = str(len(product_list))

        while msg_count < len(product_list):

            products_slice = product_list[msg_count: msg_count + self.packet_size]

            poitemstock_slice_xml = "".join(poitemstock.format(product.default_code, product.web_global_stock)
                                            for product in products_slice)

            poitemstock_xml_list.append(poitemstock_slice_xml)
            msg_count += self.packet_size

            debug_sum += len(products_slice)
            Console.debug('[Prestadoo] Construído mensaje para {} items ({} de {})'.format(str(len(products_slice)),
                                                                                str(debug_sum),
                                                                                debug_total))

        Console.debug('[Prestadoo] Todos los mensajes construídos.\n\t\tComienza encolado en Jesie de {} '
                      'mensajes'.format(str(len(poitemstock_xml_list))))

        try:
            for idx, stock_xml in enumerate(poitemstock_xml_list):
                Jesie.write('A', 'STOCK', idx, stock_xml)

        except Exception as ex:
            Console.debug("[Prestadoo] Error writing stock: {}".format(ex))

        Console.debug('[Prestadoo] Fin de proceso de notificación de STOCK a Jesie')
<commit_msg>[FIX] Error en el filtro de productos creados con FALSE en el código<commit_after># -*- coding: utf-8 -*-
from .pd_base import BaseExtClass, api, OutputHelper as Console, Jesie

class ProductProduct(BaseExtClass):
    _inherit = "product.product"

    packet_size = 500

    def is_notifiable(self):
        return self.product_tmpl_id.type == "product" and \
               self.tag_ids and \
               self.default_code and \
               self.default_code.find('False') == -1

    fields_to_watch = ('id', 'default_code', 'name', 'barcode', 'description', 'web_global_stock',
                       'product_brand_id', 'attribute_line_ids', 'tag_ids', 'active', 'type', 'force_web')

    def set_props(self, unlink=False):
        poitem = """
                <poitem>
                  <ItemCode>{}</ItemCode>
                  <ItemName>{}</ItemName>
                  <CodeBars>{}</CodeBars>
                  <UserText>{}</UserText>
                  <Model>{}</Model>
                  <Tallaje>{}</Tallaje>
                  <Talla>{}</Talla>
                  <Color>{}</Color>
                  <Stock>{}</Stock>
                  <SWeight1>{}</SWeight1>
                  <FirmName>{}</FirmName>
                  <Price>{}</Price>
                  <ItmsGrpCod>{}</ItmsGrpCod>
                  <validFor>{}</validFor>
                </poitem>
              """

        def get_tallaje_talla_color():
            """ En base a lo hablado con Miguel, los atributos o son Color (y sólo debe haber uno) o tallaje.

                Con esta función obtenemos:
                    - el id del primer atributo de la plantilla que NO sea Color
                    - el id de la talla del tallaje obtenido
                    - el id del color
            """

            tallajeid = None
            tallajecode = None
            tallacode = None
            colorcode = None

            for attr in self.product_tmpl_id.attribute_line_ids:
                if not attr.attribute_id.is_color:
                    tallajecode = attr.attribute_id.legacy_code or '#' + str(attr.attribute_id.id)
                    tallajeid = attr.attribute_id.id
                    break

            for val in self.attribute_value_ids:
                if val.attribute_id.id == tallajeid:
                    tallacode = val.legacy_code or '#' + str(val.id)
                elif val.attribute_id.is_color:
                    colorcode = val.legacy_code or '#' + str(val.id)

                if tallacode and colorcode:
                    break

            return tallajecode, tallacode, colorcode

        if unlink or (self.force_web == 'no') or not self.active:
            is_active = 'N'
        else:
            is_active = 'N'
            for tag in self.tag_ids:
                if tag.web:
                    is_active = 'Y'
            # is_active = 'Y' if self.active else 'N'

        tallaje, talla, color = get_tallaje_talla_color()

        self.xml = poitem.format(
            self.default_code or self.id,                                   # ItemCode
            self.name,                                                      # ItemName
            self.barcode or '',                                             # CodeBars
            self.description or '',                                         # UserText
            self.product_tmpl_id.template_code or self.product_tmpl_id.default_code or self.product_tmpl_id.id,  # Model
            tallaje or '',                                                  # Tallaje
            talla or '',                                                    # Talla
            color or '',                                                    # Color
            self.web_global_stock or '0.0',                                 # Stock
            self.weight or '0.0',                                           # Sweight1
            self.product_brand_id.name or '',                               # FirmName
            self.price or '0.0',                                            # Price
            ";".join(tag.legacy_code or '#' + str(tag.id) for tag in self.tag_ids),  # ItmsGrpCod
            is_active                                                       # validFor
        )

        self.obj_type = '4'

    def get_stock_packet_size(self):
        packet_size = self.env["ir.config_parameter"].search([('key', '=', 'prestadoo.stock.packet.size')])

        if not packet_size:
            Console.debug("[Prestadoo] No se ha encontrado parámetro 'prestadoo.stock.packet.size' en 'Configuración "
                          "-> Parámetros -> Parámetros del sistema'; se utilizará 500.")
            self.packet_size = 500
        else:
            self.packet_size = int(packet_size.value)
            Console.debug("[Prestadoo] Encontrado parámetro 'prestadoo.stock.packet.size'. "
                          "Valor = {}".format(packet_size.value))

    @api.model
    def stock_2_jesie(self):
        Console.debug('[Prestadoo] Inicio de proceso de notificación de STOCK a Jesie')

        self.get_stock_packet_size()

        poitemstock = """
                        <poitemstock>
                          <ItemCode>{}</ItemCode>
                          <Stock>{}</Stock>
                        </poitemstock>
                    """

        product_list = self.search([('active', '=', True),
                                    ('sale_ok', '=', True),
                                    ('default_code', '!=', False),
                                    # ('default_code', 'not like', '%ñ%'),
                                    # ('default_code', 'not like', '%Ñ%'),
                                    ('type', '=', 'product')],
                                   order='id')

        msg_count = 0
        poitemstock_xml_list = []

        debug_sum = 0
        debug_total = str(len(product_list))

        while msg_count < len(product_list):

            products_slice = product_list[msg_count: msg_count + self.packet_size]

            poitemstock_slice_xml = "".join(poitemstock.format(product.default_code, product.web_global_stock)
                                            for product in products_slice)

            poitemstock_xml_list.append(poitemstock_slice_xml)
            msg_count += self.packet_size

            debug_sum += len(products_slice)
            Console.debug('[Prestadoo] Construído mensaje para {} items ({} de {})'.format(str(len(products_slice)),
                                                                                str(debug_sum),
                                                                                debug_total))

        Console.debug('[Prestadoo] Todos los mensajes construídos.\n\t\tComienza encolado en Jesie de {} '
                      'mensajes'.format(str(len(poitemstock_xml_list))))

        try:
            for idx, stock_xml in enumerate(poitemstock_xml_list):
                Jesie.write('A', 'STOCK', idx, stock_xml)

        except Exception as ex:
            Console.debug("[Prestadoo] Error writing stock: {}".format(ex))

        Console.debug('[Prestadoo] Fin de proceso de notificación de STOCK a Jesie')
<|endoftext|>


<commit_before># This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""SMV DataSet Framework's SmvInputBase interface

This module defines the abstract classes which formed the SmvDataSet's
input DS Framework for clients' projects
"""

from pyspark.sql import DataFrame
from pyspark.sql.types import StructType

import abc
import sys
import json
import re

from smv.dqm import FailParserCountPolicy
from smv.error import SmvRuntimeError
from smv.smvdataset import SmvDataSet

if sys.version_info >= (3, 4):
    ABC = abc.ABC
else:
    ABC = abc.ABCMeta('ABC', (), {})

def _smvhash(text):
    """Python's hash function will return different numbers from run to
    from, starting from 3.  Provide a deterministic hash function for
    use to calculate sourceCodeHash.
    """
    import binascii
    return binascii.crc32(text.encode())

class SmvInputBase(SmvDataSet, ABC):
    """SmvDataSet representing external input
        Concrete class need to provide:
          - readAsDF
          - dataSrcHash (optional)
          - schemaHash (optional)
    """

    def isEphemeral(self):
        return True

    def dsType(self):
        return "Input"

    def requiresDS(self):
        return []

    def run(self, df):
        """Post-processing for input data

            Args:
                df (DataFrame): input data

            Returns:
                (DataFrame): processed data
        """
        return df

    @abc.abstractmethod
    def readAsDF(self):
        """User defined data reader

            Returns:
                (DataFrame): data from raw input
        """
        pass

    def dataSrcHash(self):
        """Hash computed on data source

            Returns:
                (int): user defined hash derived from raw data (default: 0)
        """
        return 0

    def schemaHash(self):
        """Hash computed from schema

            Returns:
                (int): user defined hash derived from input schema (default: 0)
        """
        return 0

    def instanceValHash(self):
        """Hash computed based on instance values of the dataset
            Default to dataSrcHash + schemaHash
        """
        schema_hash = self.schemaHash()
        data_src_hash = self.dataSrcHash()
        res = data_src_hash + schema_hash

        self.smvApp.log.debug("{}.instanceValHash = {}".format(self.fqn(), res))

        # ensure python's numeric type can fit in a java.lang.Integer
        return int(res) & 0x7fffffff

    def doRun(self, known):
        return self.run(self.readAsDF())

class SmvInputFromFile(SmvInputBase):
    """Base class for any input based on files on HDFS or local

        Concrete class need to provide:

            * path (str): path relative to smv.inputDir, or
            * fullPath (str): file full path with protocol
            * fullSchemaPath (str): schema file's full path, or
            * schema (StructType): schema object

    """
    def path(self):
        """Relative path to smv.dataDir config parameter"""
        pass

    def fullPath(self):
        """Full path to the input (file/dir or glob pattern)"""
        return "{}/{}".format(self.smvApp.inputDir(), self.path())

    def fullSchemaPath(self):
        """Full path to the schema file"""
        know_types = ['.gz', '.csv', '.tsv', '.xml']
        base = self.fullPath()
        for t in know_types:
            base = re.sub(t + '$', '', base)
        return base + '.schema'

    def schema(self):
        """User specified schema
            Returns:
                (StructType): input data's schema
        """
        return None

    def dataSrcHash(self):
        """Hash computed on data source
            Based on file's mtime, and file's full path
        """
        full_path = self.fullPath()
        self.smvApp.log.debug("{} input path: {}".format(self.fqn(), full_path))

        path_hash = _smvhash(full_path)

        m_time = self.smvApp._jvm.SmvHDFS.modificationTime(full_path)
        self.smvApp.log.debug("{} input m_time: {}".format(self.fqn(), m_time))

        return m_time + path_hash

    def schemaHash(self):
        """Hash computed from schema
            Based on input schema as a StructType
        """
        schema = self.schema()
        self.smvApp.log.debug("{} schema: {}".format(self.fqn(), schema))

        if schema is not None:
            return _smvhash(self.schema().simpleString())
        else:
            return 0

class SmvXmlFile(SmvInputFromFile):
    """Input from file in XML format
        Concrete class need to provide:

            * rowTag (str): XML tag for identifying a row
            * path (str): File path relative to smv.InputDir. Or
            * fullPath (str): file full path with protocol
            * fullSchemaPath (str): full path of the schema JSON file or None (infer schema)
    """

    @abc.abstractmethod
    def rowTag(self):
        """XML tag for identifying a record (row)"""
        pass

    def schema(self):
        """load schema from the json file"""
        if (self.fullSchemaPath() is None):
            return None
        else:
            with open(self.fullSchemaPath(), "r") as sj:
                schema_st = sj.read()
            return StructType.fromJson(json.loads(schema_st))

    def readAsDF(self):
        """readin xml data"""

        # TODO: look for possibilities to feed to readerLogger
        reader = self.smvApp.sqlContext\
            .read.format('com.databricks.spark.xml')\
            .options(rowTag=self.rowTag())

        # If no schema specified, infer from data
        if (self.schema() is not None):
            return reader.load(self.fullPath(), schema=self.schema())
        else:
            return reader.load(self.fullPath())

class WithParser(SmvInputBase):
    """Input uses SmvSchema and Csv parser"""

    def failAtParsingError(self):
        return True

    def readerLogger(self):
        if (self.failAtParsingError()):
            return self.smvApp._jvm.SmvPythonHelper.getTerminateParserLogger()
        else:
            return self.dqmValidator.createParserValidator()

    @abc.abstractmethod
    def smvSchema(self):
        """Returns SmvSchema, as the Scala SmvSchema class
        """
        pass

    def schema(self):
        j_schema = self.smvSchema().toStructType()
        return StructType.fromJson(json.loads(j_schema.json()))

    def csvAttr(self):
        """Specifies the csv file format.  Corresponds to the CsvAttributes case class in Scala.
            Derive from smvSchema if not specified by user.

            Override this method if user want to specify CsvAttributes which is different from
            the one can be derived from smvSchema
        """
        return self.smvSchema().extractCsvAttributes()

    def userSchema(self):
        """Get user-defined schema

            Override this method to define your own schema for the target file.
            Schema declared in this way take priority over .schema files. Schema
            should be specified in the format "colName1:colType1;colName2:colType2"

            Returns:
                (string):
        """
        return None

    def schemaHash(self):
        # when smvSchema is provided, use it to calculate hash, since it also
        # includes CsvAttributes info
        return self.smvSchema().schemaHash()

class SmvCsvFile(WithParser, SmvInputFromFile):
    """Input from a file in CSV format
        Base class for CSV file input.
        User need to define path method.

        Example:
        >>> class MyCsvFile(SmvCsvFile):
        >>>     def path(self):
        >>>         return "path/relative/to/smvInputDir/file.csv"
    """

    def smvSchema(self):
        smvSchemaObj = self.smvApp.j_smvPyClient.getSmvSchema()
        if (self.userSchema() is not None):
            return smvSchemaObj.fromString(self.userSchema())
        else:
            return smvSchemaObj.fromFile(self.smvApp.j_smvApp.sc(), self.fullSchemaPath())

    def readAsDF(self):
        jdf = self.smvApp.j_smvPyClient.readCsvFromFile(
            self.fullPath(),
            self.smvSchema(),
            self.csvAttr(),
            self.readerLogger()
        )
        return DataFrame(jdf, self.smvApp.sqlContext)

class SmvSqlCsvFile(SmvCsvFile):
    """Input from a file in CSV format and using a SQL query to access it
    """

    # temporary table name
    tableName = "df"

    def query(self):
        """Query used to extract data from the table which reads the CSV file

            Override this to specify your own query (optional). Default is
            equivalent to 'select * from ' + tableName.

            Returns:
                (str): query
        """
        return "select * from " + self.tableName

    def run(self, df):
        # temporarily register DataFrame of input CSV file as a table
        df.registerTempTable(self.tableName)

        # execute the table query
        res = self.smvApp.sqlContext.sql(self.query())

        # drop the temporary table
        self.smvApp.sqlContext.sql("drop table " + self.tableName)

        return res

class SmvMultiCsvFiles(SmvCsvFile):
    """Raw input from multiple csv files sharing single schema

        Instead of a single input file, specify a data dir with files which share
        the same schema.
    """
    def path(self):
        return self.dir()

    def readAsDF(self):
        flist = self.smvApp._jvm.SmvHDFS.dirList(self.fullPath()).array()
        # ignore all hidden files in the data dir
        filesInDir = ["{}/{}".format(self.fullPath(), n) for n in flist if not n.startswith(".")]

        if (not filesInDir):
            raise SmvRuntimeError("There are no data files in {}".format(self.fullPath()))

        combinedJdf = None
        for filePath in filesInDir:
            jdf = self.smvApp.j_smvPyClient.readCsvFromFile(
                filePath,
                self.smvSchema(),
                self.csvAttr(),
                self.readerLogger()
            )
            combinedJdf = jdf if (combinedJdf is None) else combinedJdf.unionAll(jdf)

        return DataFrame(combinedJdf, self.smvApp.sqlContext)

    def description(self):
        return "Input dir: @" + self.dir()

    @abc.abstractmethod
    def dir(self):
        """Path to the directory containing the csv files and their schema

            Returns:
                (str): path
        """

class SmvCsvStringData(WithParser):
    """Input data defined by a schema string and data string
    """

    def smvSchema(self):
        smvSchemaObj = self.smvApp.j_smvPyClient.getSmvSchema()
        return smvSchemaObj.fromString(self.schemaStr())

    def readAsDF(self):
        return self.smvApp.createDFWithLogger(self.schemaStr(), self.dataStr(), self.readerLogger())

    def dataSrcHash(self):
        return _smvhash(self.dataStr())

    @abc.abstractmethod
    def schemaStr(self):
        """Smv Schema string.

            E.g. "id:String; dt:Timestamp"

            Returns:
                (str): schema
        """

    @abc.abstractmethod
    def dataStr(self):
        """Smv data string.

            E.g. "212,2016-10-03;119,2015-01-07"

            Returns:
                (str): data
        """

class SmvJdbcTable(SmvInputBase):
    """Input from a table read through JDBC
    """
    def description(self):
        return "JDBC table {}".format(self.tableName())

    def jdbcUrl(self):
        """User can override this, default use the jdbcUrl setting in smvConfig"""
        return self.smvApp.jdbcUrl()

    def readAsDF(self):
        if (self.tableQuery() is None):
            tableNameOrQuery = self.tableName()
        else:
            tableNameOrQuery = "({}) as TMP_{}".format(
                self.tableQuery(), self.tableName()
            )

        return self.smvApp.sqlContext.read\
            .format('jdbc')\
            .option('url', self.jdbcUrl())\
            .option('dbtable', tableNameOrQuery)\
            .load()

    @abc.abstractmethod
    def tableName(self):
        """User-specified name for the table to extract input from

            Override this to specify your own table name.

            Returns:
                (str): table name
        """
        pass

    def tableQuery(self):
        """Query used to extract data from Hive table

            Override this to specify your own query (optional). Default is
            equivalent to 'select * from ' + tableName().

            Returns:
                (str): query
        """
        return None

class SmvHiveTable(SmvInputBase):
    """Input from a Hive table
    """

    def readAsDF(self):
        if (self.tableQuery() is None):
            query = "select * from {}".format(self.tableName())
        else:
            query = self.tableQuery()
        return self.smvApp.sqlContext.sql(query)

    @abc.abstractmethod
    def tableName(self):
        """User-specified name Hive hive table to extract input from

            Override this to specify your own table name.

            Returns:
                (str): table name
        """
        pass

    def tableQuery(self):
        """Query used to extract data from Hive table

            Override this to specify your own query (optional). Default is
            equivalent to 'select * from ' + tableName().

            Returns:
                (str): query
        """
        return None

__all__ = [
    'SmvInputFromFile',
    'SmvXmlFile',
    'SmvMultiCsvFiles',
    'SmvCsvFile',
    'SmvSqlCsvFile',
    'SmvCsvStringData',
    'SmvJdbcTable',
    'SmvHiveTable'
]
<commit_msg> #1456 updated failAtParsingError docstr<commit_after># This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""SMV DataSet Framework's SmvInputBase interface

This module defines the abstract classes which formed the SmvDataSet's
input DS Framework for clients' projects
"""

from pyspark.sql import DataFrame
from pyspark.sql.types import StructType

import abc
import sys
import json
import re

from smv.dqm import FailParserCountPolicy
from smv.error import SmvRuntimeError
from smv.smvdataset import SmvDataSet

if sys.version_info >= (3, 4):
    ABC = abc.ABC
else:
    ABC = abc.ABCMeta('ABC', (), {})

def _smvhash(text):
    """Python's hash function will return different numbers from run to
    from, starting from 3.  Provide a deterministic hash function for
    use to calculate sourceCodeHash.
    """
    import binascii
    return binascii.crc32(text.encode())

class SmvInputBase(SmvDataSet, ABC):
    """SmvDataSet representing external input
        Concrete class need to provide:
          - readAsDF
          - dataSrcHash (optional)
          - schemaHash (optional)
    """

    def isEphemeral(self):
        return True

    def dsType(self):
        return "Input"

    def requiresDS(self):
        return []

    def run(self, df):
        """Post-processing for input data

            Args:
                df (DataFrame): input data

            Returns:
                (DataFrame): processed data
        """
        return df

    @abc.abstractmethod
    def readAsDF(self):
        """User defined data reader

            Returns:
                (DataFrame): data from raw input
        """
        pass

    def dataSrcHash(self):
        """Hash computed on data source

            Returns:
                (int): user defined hash derived from raw data (default: 0)
        """
        return 0

    def schemaHash(self):
        """Hash computed from schema

            Returns:
                (int): user defined hash derived from input schema (default: 0)
        """
        return 0

    def instanceValHash(self):
        """Hash computed based on instance values of the dataset
            Default to dataSrcHash + schemaHash
        """
        schema_hash = self.schemaHash()
        data_src_hash = self.dataSrcHash()
        res = data_src_hash + schema_hash

        self.smvApp.log.debug("{}.instanceValHash = {}".format(self.fqn(), res))

        # ensure python's numeric type can fit in a java.lang.Integer
        return int(res) & 0x7fffffff

    def doRun(self, known):
        return self.run(self.readAsDF())

class SmvInputFromFile(SmvInputBase):
    """Base class for any input based on files on HDFS or local

        Concrete class need to provide:

            * path (str): path relative to smv.inputDir, or
            * fullPath (str): file full path with protocol
            * fullSchemaPath (str): schema file's full path, or
            * schema (StructType): schema object

    """
    def path(self):
        """Relative path to smv.dataDir config parameter"""
        pass

    def fullPath(self):
        """Full path to the input (file/dir or glob pattern)"""
        return "{}/{}".format(self.smvApp.inputDir(), self.path())

    def fullSchemaPath(self):
        """Full path to the schema file"""
        know_types = ['.gz', '.csv', '.tsv', '.xml']
        base = self.fullPath()
        for t in know_types:
            base = re.sub(t + '$', '', base)
        return base + '.schema'

    def schema(self):
        """User specified schema
            Returns:
                (StructType): input data's schema
        """
        return None

    def dataSrcHash(self):
        """Hash computed on data source
            Based on file's mtime, and file's full path
        """
        full_path = self.fullPath()
        self.smvApp.log.debug("{} input path: {}".format(self.fqn(), full_path))

        path_hash = _smvhash(full_path)

        m_time = self.smvApp._jvm.SmvHDFS.modificationTime(full_path)
        self.smvApp.log.debug("{} input m_time: {}".format(self.fqn(), m_time))

        return m_time + path_hash

    def schemaHash(self):
        """Hash computed from schema
            Based on input schema as a StructType
        """
        schema = self.schema()
        self.smvApp.log.debug("{} schema: {}".format(self.fqn(), schema))

        if schema is not None:
            return _smvhash(self.schema().simpleString())
        else:
            return 0

class SmvXmlFile(SmvInputFromFile):
    """Input from file in XML format
        Concrete class need to provide:

            * rowTag (str): XML tag for identifying a row
            * path (str): File path relative to smv.InputDir. Or
            * fullPath (str): file full path with protocol
            * fullSchemaPath (str): full path of the schema JSON file or None (infer schema)
    """

    @abc.abstractmethod
    def rowTag(self):
        """XML tag for identifying a record (row)"""
        pass

    def schema(self):
        """load schema from the json file"""
        if (self.fullSchemaPath() is None):
            return None
        else:
            with open(self.fullSchemaPath(), "r") as sj:
                schema_st = sj.read()
            return StructType.fromJson(json.loads(schema_st))

    def readAsDF(self):
        """readin xml data"""

        # TODO: look for possibilities to feed to readerLogger
        reader = self.smvApp.sqlContext\
            .read.format('com.databricks.spark.xml')\
            .options(rowTag=self.rowTag())

        # If no schema specified, infer from data
        if (self.schema() is not None):
            return reader.load(self.fullPath(), schema=self.schema())
        else:
            return reader.load(self.fullPath())

class WithParser(SmvInputBase):
    """Input uses SmvSchema and Csv parser"""

    def failAtParsingError(self):
        """When set, any parsing error will throw and exception to make sure we can stop early
            to tolerant some parsing error, should

            - Override failAtParsingError to False
            - Set dqm to SmvDQM().add(FailParserCountPolicy(10))
                for tolerant <=10 parsing errors
        """
        return True

    def readerLogger(self):
        if (self.failAtParsingError()):
            return self.smvApp._jvm.SmvPythonHelper.getTerminateParserLogger()
        else:
            return self.dqmValidator.createParserValidator()

    @abc.abstractmethod
    def smvSchema(self):
        """Returns SmvSchema, as the Scala SmvSchema class
        """
        pass

    def schema(self):
        j_schema = self.smvSchema().toStructType()
        return StructType.fromJson(json.loads(j_schema.json()))

    def csvAttr(self):
        """Specifies the csv file format.  Corresponds to the CsvAttributes case class in Scala.
            Derive from smvSchema if not specified by user.

            Override this method if user want to specify CsvAttributes which is different from
            the one can be derived from smvSchema
        """
        return self.smvSchema().extractCsvAttributes()

    def userSchema(self):
        """Get user-defined schema

            Override this method to define your own schema for the target file.
            Schema declared in this way take priority over .schema files. Schema
            should be specified in the format "colName1:colType1;colName2:colType2"

            Returns:
                (string):
        """
        return None

    def schemaHash(self):
        # when smvSchema is provided, use it to calculate hash, since it also
        # includes CsvAttributes info
        return self.smvSchema().schemaHash()

class SmvCsvFile(WithParser, SmvInputFromFile):
    """Input from a file in CSV format
        Base class for CSV file input.
        User need to define path method.

        Example:
        >>> class MyCsvFile(SmvCsvFile):
        >>>     def path(self):
        >>>         return "path/relative/to/smvInputDir/file.csv"
    """

    def smvSchema(self):
        smvSchemaObj = self.smvApp.j_smvPyClient.getSmvSchema()
        if (self.userSchema() is not None):
            return smvSchemaObj.fromString(self.userSchema())
        else:
            return smvSchemaObj.fromFile(self.smvApp.j_smvApp.sc(), self.fullSchemaPath())

    def readAsDF(self):
        jdf = self.smvApp.j_smvPyClient.readCsvFromFile(
            self.fullPath(),
            self.smvSchema(),
            self.csvAttr(),
            self.readerLogger()
        )
        return DataFrame(jdf, self.smvApp.sqlContext)

class SmvSqlCsvFile(SmvCsvFile):
    """Input from a file in CSV format and using a SQL query to access it
    """

    # temporary table name
    tableName = "df"

    def query(self):
        """Query used to extract data from the table which reads the CSV file

            Override this to specify your own query (optional). Default is
            equivalent to 'select * from ' + tableName.

            Returns:
                (str): query
        """
        return "select * from " + self.tableName

    def run(self, df):
        # temporarily register DataFrame of input CSV file as a table
        df.registerTempTable(self.tableName)

        # execute the table query
        res = self.smvApp.sqlContext.sql(self.query())

        # drop the temporary table
        self.smvApp.sqlContext.sql("drop table " + self.tableName)

        return res

class SmvMultiCsvFiles(SmvCsvFile):
    """Raw input from multiple csv files sharing single schema

        Instead of a single input file, specify a data dir with files which share
        the same schema.
    """
    def path(self):
        return self.dir()

    def readAsDF(self):
        flist = self.smvApp._jvm.SmvHDFS.dirList(self.fullPath()).array()
        # ignore all hidden files in the data dir
        filesInDir = ["{}/{}".format(self.fullPath(), n) for n in flist if not n.startswith(".")]

        if (not filesInDir):
            raise SmvRuntimeError("There are no data files in {}".format(self.fullPath()))

        combinedJdf = None
        for filePath in filesInDir:
            jdf = self.smvApp.j_smvPyClient.readCsvFromFile(
                filePath,
                self.smvSchema(),
                self.csvAttr(),
                self.readerLogger()
            )
            combinedJdf = jdf if (combinedJdf is None) else combinedJdf.unionAll(jdf)

        return DataFrame(combinedJdf, self.smvApp.sqlContext)

    def description(self):
        return "Input dir: @" + self.dir()

    @abc.abstractmethod
    def dir(self):
        """Path to the directory containing the csv files and their schema

            Returns:
                (str): path
        """

class SmvCsvStringData(WithParser):
    """Input data defined by a schema string and data string
    """

    def smvSchema(self):
        smvSchemaObj = self.smvApp.j_smvPyClient.getSmvSchema()
        return smvSchemaObj.fromString(self.schemaStr())

    def readAsDF(self):
        return self.smvApp.createDFWithLogger(self.schemaStr(), self.dataStr(), self.readerLogger())

    def dataSrcHash(self):
        return _smvhash(self.dataStr())

    @abc.abstractmethod
    def schemaStr(self):
        """Smv Schema string.

            E.g. "id:String; dt:Timestamp"

            Returns:
                (str): schema
        """

    @abc.abstractmethod
    def dataStr(self):
        """Smv data string.

            E.g. "212,2016-10-03;119,2015-01-07"

            Returns:
                (str): data
        """

class SmvJdbcTable(SmvInputBase):
    """Input from a table read through JDBC
    """
    def description(self):
        return "JDBC table {}".format(self.tableName())

    def jdbcUrl(self):
        """User can override this, default use the jdbcUrl setting in smvConfig"""
        return self.smvApp.jdbcUrl()

    def readAsDF(self):
        if (self.tableQuery() is None):
            tableNameOrQuery = self.tableName()
        else:
            tableNameOrQuery = "({}) as TMP_{}".format(
                self.tableQuery(), self.tableName()
            )

        return self.smvApp.sqlContext.read\
            .format('jdbc')\
            .option('url', self.jdbcUrl())\
            .option('dbtable', tableNameOrQuery)\
            .load()

    @abc.abstractmethod
    def tableName(self):
        """User-specified name for the table to extract input from

            Override this to specify your own table name.

            Returns:
                (str): table name
        """
        pass

    def tableQuery(self):
        """Query used to extract data from Hive table

            Override this to specify your own query (optional). Default is
            equivalent to 'select * from ' + tableName().

            Returns:
                (str): query
        """
        return None

class SmvHiveTable(SmvInputBase):
    """Input from a Hive table
    """

    def readAsDF(self):
        if (self.tableQuery() is None):
            query = "select * from {}".format(self.tableName())
        else:
            query = self.tableQuery()
        return self.smvApp.sqlContext.sql(query)

    @abc.abstractmethod
    def tableName(self):
        """User-specified name Hive hive table to extract input from

            Override this to specify your own table name.

            Returns:
                (str): table name
        """
        pass

    def tableQuery(self):
        """Query used to extract data from Hive table

            Override this to specify your own query (optional). Default is
            equivalent to 'select * from ' + tableName().

            Returns:
                (str): query
        """
        return None

__all__ = [
    'SmvInputFromFile',
    'SmvXmlFile',
    'SmvMultiCsvFiles',
    'SmvCsvFile',
    'SmvSqlCsvFile',
    'SmvCsvStringData',
    'SmvJdbcTable',
    'SmvHiveTable'
]
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) 2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Run fuzz test targets.
"""

import argparse
import configparser
import os
import sys
import subprocess
import logging

# Fuzzers known to lack a seed corpus in https://github.com/bitcoin-core/qa-assets/tree/master/fuzz_seed_corpus
FUZZERS_MISSING_CORPORA = [
    "addr_info_deserialize",
    "asmap",
    "base_encode_decode",
    "block",
    "block_file_info_deserialize",
    "block_filter_deserialize",
    "block_header_and_short_txids_deserialize",
    "decode_tx",
    "fee_rate_deserialize",
    "flat_file_pos_deserialize",
    "hex",
    "integer",
    "key_origin_info_deserialize",
    "merkle_block_deserialize",
    "out_point_deserialize",
    "parse_hd_keypath",
    "parse_numbers",
    "parse_script",
    "parse_univalue",
    "partial_merkle_tree_deserialize",
    "partially_signed_transaction_deserialize",
    "prefilled_transaction_deserialize",
    "psbt_input_deserialize",
    "psbt_output_deserialize",
    "pub_key_deserialize",
    "script_deserialize",
    "strprintf",
    "sub_net_deserialize",
    "tx_in",
    "tx_in_deserialize",
    "tx_out",
]

def main():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        "-l",
        "--loglevel",
        dest="loglevel",
        default="INFO",
        help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.",
    )
    parser.add_argument(
        '--export_coverage',
        action='store_true',
        help='If true, export coverage information to files in the seed corpus',
    )
    parser.add_argument(
        '--valgrind',
        action='store_true',
        help='If true, run fuzzing binaries under the valgrind memory error detector',
    )
    parser.add_argument(
        'seed_dir',
        help='The seed corpus to run on (must contain subfolders for each fuzz target).',
    )
    parser.add_argument(
        'target',
        nargs='*',
        help='The target(s) to run. Default is to run all targets.',
    )

    args = parser.parse_args()

    # Set up logging
    logging.basicConfig(
        format='%(message)s',
        level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(),
    )

    # Read config generated by configure.
    config = configparser.ConfigParser()
    configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
    config.read_file(open(configfile, encoding="utf8"))

    if not config["components"].getboolean("ENABLE_FUZZ"):
        logging.error("Must have fuzz targets built")
        sys.exit(1)

    # Build list of tests
    test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include'))

    if not test_list_all:
        logging.error("No fuzz targets found")
        sys.exit(1)

    logging.info("Fuzz targets found: {}".format(test_list_all))

    args.target = args.target or test_list_all  # By default run all
    test_list_error = list(set(args.target).difference(set(test_list_all)))
    if test_list_error:
        logging.error("Unknown fuzz targets selected: {}".format(test_list_error))
    test_list_selection = list(set(test_list_all).intersection(set(args.target)))
    if not test_list_selection:
        logging.error("No fuzz targets selected")
    logging.info("Fuzz targets selected: {}".format(test_list_selection))

    try:
        help_output = subprocess.run(
            args=[
                os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]),
                '-help=1',
            ],
            timeout=10,
            check=True,
            stderr=subprocess.PIPE,
            universal_newlines=True,
        ).stderr
        if "libFuzzer" not in help_output:
            logging.error("Must be built with libFuzzer")
            sys.exit(1)
    except subprocess.TimeoutExpired:
        logging.error("subprocess timed out: Currently only libFuzzer is supported")
        sys.exit(1)

    run_once(
        corpus=args.seed_dir,
        test_list=test_list_selection,
        build_dir=config["environment"]["BUILDDIR"],
        export_coverage=args.export_coverage,
        use_valgrind=args.valgrind,
    )

def run_once(*, corpus, test_list, build_dir, export_coverage, use_valgrind):
    for t in test_list:
        corpus_path = os.path.join(corpus, t)
        if t in FUZZERS_MISSING_CORPORA:
            os.makedirs(corpus_path, exist_ok=True)
        args = [
            os.path.join(build_dir, 'src', 'test', 'fuzz', t),
            '-runs=1',
            corpus_path,
        ]
        if use_valgrind:
            args = ['valgrind', '--quiet', '--error-exitcode=1'] + args
        logging.debug('Run {} with args {}'.format(t, args))
        result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True)
        output = result.stderr
        logging.debug('Output: {}'.format(output))
        result.check_returncode()
        if not export_coverage:
            continue
        for l in output.splitlines():
            if 'INITED' in l:
                with open(os.path.join(corpus, t + '_coverage'), 'w', encoding='utf-8') as cov_file:
                    cov_file.write(l)
                    break

def parse_test_list(makefile):
    with open(makefile, encoding='utf-8') as makefile_test:
        test_list_all = []
        read_targets = False
        for line in makefile_test.readlines():
            line = line.strip().replace('test/fuzz/', '').replace(' \\', '')
            if read_targets:
                if not line:
                    break
                test_list_all.append(line)
                continue

            if line == 'FUZZ_TARGETS =':
                read_targets = True
    return test_list_all

if __name__ == '__main__':
    main()
<commit_msg>tests: Add support for excluding fuzz targets using -x/--exclude<commit_after>#!/usr/bin/env python3
# Copyright (c) 2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Run fuzz test targets.
"""

import argparse
import configparser
import os
import sys
import subprocess
import logging

# Fuzzers known to lack a seed corpus in https://github.com/bitcoin-core/qa-assets/tree/master/fuzz_seed_corpus
FUZZERS_MISSING_CORPORA = [
    "addr_info_deserialize",
    "asmap",
    "base_encode_decode",
    "block",
    "block_file_info_deserialize",
    "block_filter_deserialize",
    "block_header_and_short_txids_deserialize",
    "decode_tx",
    "fee_rate_deserialize",
    "flat_file_pos_deserialize",
    "hex",
    "integer",
    "key_origin_info_deserialize",
    "merkle_block_deserialize",
    "out_point_deserialize",
    "parse_hd_keypath",
    "parse_numbers",
    "parse_script",
    "parse_univalue",
    "partial_merkle_tree_deserialize",
    "partially_signed_transaction_deserialize",
    "prefilled_transaction_deserialize",
    "psbt_input_deserialize",
    "psbt_output_deserialize",
    "pub_key_deserialize",
    "script_deserialize",
    "strprintf",
    "sub_net_deserialize",
    "tx_in",
    "tx_in_deserialize",
    "tx_out",
]

def main():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        "-l",
        "--loglevel",
        dest="loglevel",
        default="INFO",
        help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.",
    )
    parser.add_argument(
        '--export_coverage',
        action='store_true',
        help='If true, export coverage information to files in the seed corpus',
    )
    parser.add_argument(
        '--valgrind',
        action='store_true',
        help='If true, run fuzzing binaries under the valgrind memory error detector',
    )
    parser.add_argument(
        '-x',
        '--exclude',
        help="A comma-separated list of targets to exclude",
    )
    parser.add_argument(
        'seed_dir',
        help='The seed corpus to run on (must contain subfolders for each fuzz target).',
    )
    parser.add_argument(
        'target',
        nargs='*',
        help='The target(s) to run. Default is to run all targets.',
    )

    args = parser.parse_args()

    # Set up logging
    logging.basicConfig(
        format='%(message)s',
        level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(),
    )

    # Read config generated by configure.
    config = configparser.ConfigParser()
    configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
    config.read_file(open(configfile, encoding="utf8"))

    if not config["components"].getboolean("ENABLE_FUZZ"):
        logging.error("Must have fuzz targets built")
        sys.exit(1)

    # Build list of tests
    test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include'))

    if not test_list_all:
        logging.error("No fuzz targets found")
        sys.exit(1)

    logging.debug("{} fuzz target(s) found: {}".format(len(test_list_all), " ".join(sorted(test_list_all))))

    args.target = args.target or test_list_all  # By default run all
    test_list_error = list(set(args.target).difference(set(test_list_all)))
    if test_list_error:
        logging.error("Unknown fuzz targets selected: {}".format(test_list_error))
    test_list_selection = list(set(test_list_all).intersection(set(args.target)))
    if not test_list_selection:
        logging.error("No fuzz targets selected")
    if args.exclude:
        for excluded_target in args.exclude.split(","):
            if excluded_target not in test_list_selection:
                logging.error("Target \"{}\" not found in current target list.".format(excluded_target))
                continue
            test_list_selection.remove(excluded_target)
    test_list_selection.sort()

    logging.info("{} of {} detected fuzz target(s) selected: {}".format(len(test_list_selection), len(test_list_all), " ".join(test_list_selection)))

    try:
        help_output = subprocess.run(
            args=[
                os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]),
                '-help=1',
            ],
            timeout=10,
            check=True,
            stderr=subprocess.PIPE,
            universal_newlines=True,
        ).stderr
        if "libFuzzer" not in help_output:
            logging.error("Must be built with libFuzzer")
            sys.exit(1)
    except subprocess.TimeoutExpired:
        logging.error("subprocess timed out: Currently only libFuzzer is supported")
        sys.exit(1)

    run_once(
        corpus=args.seed_dir,
        test_list=test_list_selection,
        build_dir=config["environment"]["BUILDDIR"],
        export_coverage=args.export_coverage,
        use_valgrind=args.valgrind,
    )

def run_once(*, corpus, test_list, build_dir, export_coverage, use_valgrind):
    for t in test_list:
        corpus_path = os.path.join(corpus, t)
        if t in FUZZERS_MISSING_CORPORA:
            os.makedirs(corpus_path, exist_ok=True)
        args = [
            os.path.join(build_dir, 'src', 'test', 'fuzz', t),
            '-runs=1',
            corpus_path,
        ]
        if use_valgrind:
            args = ['valgrind', '--quiet', '--error-exitcode=1'] + args
        logging.debug('Run {} with args {}'.format(t, args))
        result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True)
        output = result.stderr
        logging.debug('Output: {}'.format(output))
        result.check_returncode()
        if not export_coverage:
            continue
        for l in output.splitlines():
            if 'INITED' in l:
                with open(os.path.join(corpus, t + '_coverage'), 'w', encoding='utf-8') as cov_file:
                    cov_file.write(l)
                    break

def parse_test_list(makefile):
    with open(makefile, encoding='utf-8') as makefile_test:
        test_list_all = []
        read_targets = False
        for line in makefile_test.readlines():
            line = line.strip().replace('test/fuzz/', '').replace(' \\', '')
            if read_targets:
                if not line:
                    break
                test_list_all.append(line)
                continue

            if line == 'FUZZ_TARGETS =':
                read_targets = True
    return test_list_all

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# © 2014 Elico corp(www.elico-corp.com)
# Licence AGPL-3.0 or later(http://www.gnu.org/licenses/agpl.html)

{
    'name': 'Product Price Category',
    'version': '7.0.1.0.0',
    'author': 'Elico Corp',
    'website': 'http://www.elico-corp.com',
    'description' : """
    User can quickly search the product price base on the quality.
    """,
    'depends': ['base','product',],
    'category': '',
    'sequence': 16,
    'data': [
             'security/price_search_security.xml',
             'security/ir.model.access.csv',
             'product_price.xml',
        ],
    'installable': True,
    'application': False,
    'auto_install': False,
}

<commit_msg>Update __openerp__.py<commit_after># -*- coding: utf-8 -*-
# © 2014 Elico corp(www.elico-corp.com)
# Licence AGPL-3.0 or later(http://www.gnu.org/licenses/agpl.html)

{
    'name': 'Product Price Category',
    'version': '7.0.1.0.0',
    'author': 'Elico Corp',
    'website': 'http://www.elico-corp.com',
    'description' : """
    User can quickly search the product price base on the quantity.
    """,
    'depends': ['base','product',],
    'category': '',
    'sequence': 16,
    'data': [
             'security/price_search_security.xml',
             'security/ir.model.access.csv',
             'product_price.xml',
        ],
    'installable': True,
    'application': False,
    'auto_install': False,
}

<|endoftext|>


<commit_before>##############################################################################
# For copyright and license notices, see __manifest__.py file in module root
# directory
##############################################################################
from odoo import fields, models, api

class ProductTemplate(models.Model):

    _inherit = 'product.template'

    abc_sales_quantity = fields.Char(string='ABC Sales quantity')
    abc_sales_amount = fields.Char(string='ABC Sales amount')
    abc_sales_combined = fields.Char(
        compute='_compute_abc_total',
        string='ABC Sales combined')

    @api.depends()
    def _compute_abc_total(self):
        if self.abc_sales_quantity and self.abc_sales_amount:
            self.abc_sales_combined = \
                self.abc_sales_quantity + self.abc_sales_amount
        elif self.abc_sales_quantity and not self.abc_sales_amount:
            self.abc_sales_combined = self.abc_sales_quantity
        elif self.abc_sales_amount and not self.abc_sales_quantity:
            self.abc_sales_combined = self.abc_sales_amount
<commit_msg>[FIX] make _compute_abc_total api.multi<commit_after>##############################################################################
# For copyright and license notices, see __manifest__.py file in module root
# directory
##############################################################################
from odoo import fields, models, api

class ProductTemplate(models.Model):

    _inherit = 'product.template'

    abc_sales_quantity = fields.Char(string='ABC Sales quantity')
    abc_sales_amount = fields.Char(string='ABC Sales amount')
    abc_sales_combined = fields.Char(
        compute='_compute_abc_total',
        string='ABC Sales combined')

    @api.depends()
    def _compute_abc_total(self):
        for rec in self:
            if rec.abc_sales_quantity and rec.abc_sales_amount:
                rec.abc_sales_combined = \
                    rec.abc_sales_quantity + rec.abc_sales_amount
            elif rec.abc_sales_quantity and not rec.abc_sales_amount:
                rec.abc_sales_combined = rec.abc_sales_quantity
            elif rec.abc_sales_amount and not rec.abc_sales_quantity:
                rec.abc_sales_combined = rec.abc_sales_amount
<|endoftext|>


<commit_before>import collections
import re

from elasticsearch.exceptions import NotFoundError, RequestError
from six import iteritems, add_metaclass

from .field import Field
from .mapping import Mapping
from .utils import ObjectBase, AttrDict, merge
from .response import HitMeta
from .search import Search
from .connections import connections
from .exceptions import ValidationException, IllegalOperation

DELETE_META_FIELDS = frozenset((
    'id', 'parent', 'routing', 'version', 'version_type'
))

DOC_META_FIELDS = frozenset((
    'timestamp', 'ttl'
)).union(DELETE_META_FIELDS)

META_FIELDS = frozenset((
    # Elasticsearch metadata fields, except 'type'
    'index', 'using', 'score',
)).union(DOC_META_FIELDS)

class MetaField(object):
    def __init__(self, *args, **kwargs):
        self.args, self.kwargs = args, kwargs

class DocTypeMeta(type):
    def __new__(cls, name, bases, attrs):
        # DocTypeMeta filters attrs in place
        attrs['_doc_type'] = DocTypeOptions(name, bases, attrs)
        return super(DocTypeMeta, cls).__new__(cls, name, bases, attrs)

class DocTypeOptions(object):
    def __init__(self, name, bases, attrs):
        meta = attrs.pop('Meta', None)

        # default index, if not overriden by doc.meta
        self.index = getattr(meta, 'index', None)

        # default cluster alias, can be overriden in doc.meta
        self._using = getattr(meta, 'using', None)

        # get doc_type name, if not defined take the name of the class and
        # transform it to lower_case
        doc_type = getattr(meta, 'doc_type',
                re.sub(r'(.)([A-Z])', r'\1_\2', name).lower())

        # create the mapping instance
        self.mapping = getattr(meta, 'mapping', Mapping(doc_type))

        # register all declared fields into the mapping
        for name, value in list(iteritems(attrs)):
            if isinstance(value, Field):
                self.mapping.field(name, value)
                del attrs[name]

        # add all the mappings for meta fields
        for name in dir(meta):
            if isinstance(getattr(meta, name, None), MetaField):
                params = getattr(meta, name)
                self.mapping.meta(name, *params.args, **params.kwargs)

        # document inheritance - include the fields from parents' mappings and
        # index/using values
        for b in bases:
            if hasattr(b, '_doc_type') and hasattr(b._doc_type, 'mapping'):
                self.mapping.update(b._doc_type.mapping, update_only=True)
                self._using = self._using or b._doc_type._using
                self.index = self.index or b._doc_type.index

    @property
    def using(self):
        return self._using or 'default'

    @property
    def name(self):
        return self.mapping.properties.name

    @property
    def parent(self):
        if '_parent' in self.mapping._meta:
            return self.mapping._meta['_parent']['type']
        return

    def resolve_field(self, field_path):
        return self.mapping.resolve_field(field_path)

    def init(self, index=None, using=None):
        self.mapping.save(index or self.index, using=using or self.using)

    def refresh(self, index=None, using=None):
        self.mapping.update_from_es(index or self.index, using=using or self.using)

@add_metaclass(DocTypeMeta)
class DocType(ObjectBase):
    """
    Model-like class for persisting documents in elasticsearch.
    """
    def __init__(self, meta=None, **kwargs):
        meta = meta or {}
        for k in list(kwargs):
            if k.startswith('_') and k[1:] in META_FIELDS:
                meta[k] = kwargs.pop(k)

        if self._doc_type.index:
            meta.setdefault('_index', self._doc_type.index)
        super(AttrDict, self).__setattr__('meta', HitMeta(meta))

        super(DocType, self).__init__(**kwargs)

    def __getstate__(self):
        return (self.to_dict(), self.meta._d_)

    def __setstate__(self, state):
        data, meta = state
        super(AttrDict, self).__setattr__('_d_', data)
        super(AttrDict, self).__setattr__('meta', HitMeta(meta))

    def __getattr__(self, name):
        if name.startswith('_') and name[1:] in META_FIELDS:
            return getattr(self.meta, name[1:])
        return super(DocType, self).__getattr__(name)

    def __repr__(self):
        return '%s(%s)' % (
            self.__class__.__name__,
            ', '.join('%s=%r' % (key, getattr(self.meta, key)) for key in
                      ('index', 'doc_type', 'id') if key in self.meta)
        )

    def __setattr__(self, name, value):
        if name.startswith('_') and name[1:] in META_FIELDS:
            return setattr(self.meta, name[1:], value)
        return super(DocType, self).__setattr__(name, value)

    @classmethod
    def init(cls, index=None, using=None):
        """
        Create the index and populate the mappings in elasticsearch.
        """
        cls._doc_type.init(index, using)

    @classmethod
    def search(cls, using=None, index=None):
        """
        Create an :class:`~elasticsearch_dsl.Search` instance that will search
        over this ``DocType``.
        """
        return Search(
            using=using or cls._doc_type.using,
            index=index or cls._doc_type.index,
            doc_type=[cls]
        )

    @classmethod
    def get(cls, id, using=None, index=None, **kwargs):
        """
        Retrieve a single document from elasticsearch using it's ``id``.

        :arg id: ``id`` of the document to be retireved
        :arg index: elasticsearch index to use, if the ``DocType`` is
            associated with an index this can be omitted.
        :arg using: connection alias to use, defaults to ``'default'``

        Any additional keyword arguments will be passed to
        ``Elasticsearch.get`` unchanged.
        """
        es = connections.get_connection(using or cls._doc_type.using)
        doc = es.get(
            index=index or cls._doc_type.index,
            doc_type=cls._doc_type.name,
            id=id,
            **kwargs
        )
        if not doc['found']:
            return None
        return cls.from_es(doc)

    @classmethod
    def mget(cls, docs, using=None, index=None, raise_on_error=True,
             missing='none', **kwargs):
        """
        Retrieve multiple document by their ``id``\s. Returns a list of instances
        in the same order as requested.

        :arg docs: list of ``id``\s of the documents to be retireved or a list
            of document specifications as per
            https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-multi-get.html
        :arg index: elasticsearch index to use, if the ``DocType`` is
            associated with an index this can be omitted.
        :arg using: connection alias to use, defaults to ``'default'``
        :arg missing: what to do when one of the documents requested is not
            found. Valid options are ``'none'`` (use ``None``), ``'raise'`` (raise
            ``NotFoundError``) or ``'skip'`` (ignore the missing document).

        Any additional keyword arguments will be passed to
        ``Elasticsearch.mget`` unchanged.
        """
        if missing not in ('raise', 'skip', 'none'):
            raise ValueError("'missing' must be 'raise', 'skip', or 'none'.")
        es = connections.get_connection(using or cls._doc_type.using)
        body = {
            'docs': [
                doc if isinstance(doc, collections.Mapping) else {'_id': doc}
                for doc in docs
            ]
        }
        results = es.mget(
            body,
            index=index or cls._doc_type.index,
            doc_type=cls._doc_type.name,
            **kwargs
        )

        objs, error_docs, missing_docs = [], [], []
        for doc in results['docs']:
            if doc.get('found'):
                if error_docs or missing_docs:
                    # We're going to raise an exception anyway, so avoid an
                    # expensive call to cls.from_es().
                    continue

                objs.append(cls.from_es(doc))

            elif doc.get('error'):
                if raise_on_error:
                    error_docs.append(doc)
                if missing == 'none':
                    objs.append(None)

            # The doc didn't cause an error, but the doc also wasn't found.
            elif missing == 'raise':
                missing_docs.append(doc)
            elif missing == 'none':
                objs.append(None)

        if error_docs:
            error_ids = [doc['_id'] for doc in error_docs]
            message = 'Required routing/parent not provided for documents %s.'
            message %= ', '.join(error_ids)
            raise RequestError(400, message, error_docs)
        if missing_docs:
            missing_ids = [doc['_id'] for doc in missing_docs]
            message = 'Documents %s not found.' % ', '.join(missing_ids)
            raise NotFoundError(404, message, missing_docs)
        return objs

    @classmethod
    def from_es(cls, hit):
        """
        Helper method to construct an instance from a dictionary returned by
        elasticsearch.
        """
        # don't modify in place
        meta = hit.copy()
        doc = meta.pop('_source', {})

        if 'fields' in meta:
            for k, v in iteritems(meta.pop('fields')):
                if k == '_source':
                    doc.update(v)
                if k.startswith('_') and k[1:] in META_FIELDS:
                    meta[k] = v
                else:
                    doc[k] = v

        return cls(meta=meta, **doc)

    def _get_connection(self, using=None):
        return connections.get_connection(using or self._doc_type.using)
    connection = property(_get_connection)

    def _get_index(self, index=None):
        if index is None:
            index = getattr(self.meta, 'index', self._doc_type.index)
        if index is None:
            raise ValidationException('No index')
        return index

    def delete(self, using=None, index=None, **kwargs):
        """
        Delete the instance in elasticsearch.

        :arg index: elasticsearch index to use, if the ``DocType`` is
            associated with an index this can be omitted.
        :arg using: connection alias to use, defaults to ``'default'``

        Any additional keyword arguments will be passed to
        ``Elasticsearch.delete`` unchanged.
        """
        es = self._get_connection(using)
        # extract parent, routing etc from meta
        doc_meta = dict(
            (k, self.meta[k])
            for k in DELETE_META_FIELDS
            if k in self.meta
        )
        doc_meta.update(kwargs)
        es.delete(
            index=self._get_index(index),
            doc_type=self._doc_type.name,
            **doc_meta
        )

    def to_dict(self, include_meta=False):
        """
        Serialize the instance into a dictionary so that it can be saved in elasticsearch.

        :arg include_meta: if set to ``True`` will include all the metadata
            (``_index``, ``_type``, ``_id`` etc). Otherwise just the document's
            data is serialized. This is useful when passing multiple instances into
            ``elasticsearch.helpers.bulk``.
        """
        d = super(DocType, self).to_dict()
        if not include_meta:
            return d

        meta = dict(
            ('_' + k, self.meta[k])
            for k in DOC_META_FIELDS
            if k in self.meta
        )

        # in case of to_dict include the index unlike save/update/delete
        if 'index' in self.meta:
            meta['_index'] = self.meta.index
        elif self._doc_type.index:
            meta['_index'] = self._doc_type.index

        meta['_type'] = self._doc_type.name
        meta['_source'] = d
        return meta

    def update(self, using=None, index=None,  detect_noop=True, doc_as_upsert=False, **fields):
        """
        Partial update of the document, specify fields you wish to update and
        both the instance and the document in elasticsearch will be updated::

            doc = MyDocument(title='Document Title!')
            doc.save()
            doc.update(title='New Document Title!')

        :arg index: elasticsearch index to use, if the ``DocType`` is
            associated with an index this can be omitted.
        :arg using: connection alias to use, defaults to ``'default'``

        Any additional keyword arguments will be passed to
        ``Elasticsearch.update`` unchanged.
        """
        if not fields:
            raise IllegalOperation('You cannot call update() without updating individual fields. '
                                   'If you wish to update the entire object use save().')

        es = self._get_connection(using)

        # update given fields locally
        if len(fields):
            merge(self._d_, fields)

        # prepare data for ES
        doc = self.to_dict()

        # if fields were given: partial update
        if fields:
            doc = dict(
                (k, doc.get(k))
                for k in fields.keys()
            )

        # extract parent, routing etc from meta
        doc_meta = dict(
            (k, self.meta[k])
            for k in DOC_META_FIELDS
            if k in self.meta
        )
        body = {
            'doc': doc,
            'doc_as_upsert': doc_as_upsert,
            'detect_noop': detect_noop,
        }

        meta = es.update(
            index=self._get_index(index),
            doc_type=self._doc_type.name,
            body=body,
            **doc_meta
        )
        # update meta information from ES
        for k in META_FIELDS:
            if '_' + k in meta:
                setattr(self.meta, k, meta['_' + k])

    def save(self, using=None, index=None, validate=True, **kwargs):
        """
        Save the document into elasticsearch. If the document doesn't exist it
        is created, it is overwritten otherwise. Returns ``True`` if this
        operations resulted in new document being created.

        :arg index: elasticsearch index to use, if the ``DocType`` is
            associated with an index this can be omitted.
        :arg using: connection alias to use, defaults to ``'default'``
        :arg validate: set to ``False`` to skip validating the document

        Any additional keyword arguments will be passed to
        ``Elasticsearch.index`` unchanged.
        """
        if validate:
            self.full_clean()

        es = self._get_connection(using)
        # extract parent, routing etc from meta
        doc_meta = dict(
            (k, self.meta[k])
            for k in DOC_META_FIELDS
            if k in self.meta
        )
        doc_meta.update(kwargs)
        meta = es.index(
            index=self._get_index(index),
            doc_type=self._doc_type.name,
            body=self.to_dict(),
            **doc_meta
        )
        # update meta information from ES
        for k in META_FIELDS:
            if '_' + k in meta:
                setattr(self.meta, k, meta['_' + k])

        # return True/False if the document has been created/updated
        return meta['created']
<commit_msg>simplified fields merge<commit_after>import collections
import re

from elasticsearch.exceptions import NotFoundError, RequestError
from six import iteritems, add_metaclass

from .field import Field
from .mapping import Mapping
from .utils import ObjectBase, AttrDict, merge
from .response import HitMeta
from .search import Search
from .connections import connections
from .exceptions import ValidationException, IllegalOperation

DELETE_META_FIELDS = frozenset((
    'id', 'parent', 'routing', 'version', 'version_type'
))

DOC_META_FIELDS = frozenset((
    'timestamp', 'ttl'
)).union(DELETE_META_FIELDS)

META_FIELDS = frozenset((
    # Elasticsearch metadata fields, except 'type'
    'index', 'using', 'score',
)).union(DOC_META_FIELDS)

class MetaField(object):
    def __init__(self, *args, **kwargs):
        self.args, self.kwargs = args, kwargs

class DocTypeMeta(type):
    def __new__(cls, name, bases, attrs):
        # DocTypeMeta filters attrs in place
        attrs['_doc_type'] = DocTypeOptions(name, bases, attrs)
        return super(DocTypeMeta, cls).__new__(cls, name, bases, attrs)

class DocTypeOptions(object):
    def __init__(self, name, bases, attrs):
        meta = attrs.pop('Meta', None)

        # default index, if not overriden by doc.meta
        self.index = getattr(meta, 'index', None)

        # default cluster alias, can be overriden in doc.meta
        self._using = getattr(meta, 'using', None)

        # get doc_type name, if not defined take the name of the class and
        # transform it to lower_case
        doc_type = getattr(meta, 'doc_type',
                re.sub(r'(.)([A-Z])', r'\1_\2', name).lower())

        # create the mapping instance
        self.mapping = getattr(meta, 'mapping', Mapping(doc_type))

        # register all declared fields into the mapping
        for name, value in list(iteritems(attrs)):
            if isinstance(value, Field):
                self.mapping.field(name, value)
                del attrs[name]

        # add all the mappings for meta fields
        for name in dir(meta):
            if isinstance(getattr(meta, name, None), MetaField):
                params = getattr(meta, name)
                self.mapping.meta(name, *params.args, **params.kwargs)

        # document inheritance - include the fields from parents' mappings and
        # index/using values
        for b in bases:
            if hasattr(b, '_doc_type') and hasattr(b._doc_type, 'mapping'):
                self.mapping.update(b._doc_type.mapping, update_only=True)
                self._using = self._using or b._doc_type._using
                self.index = self.index or b._doc_type.index

    @property
    def using(self):
        return self._using or 'default'

    @property
    def name(self):
        return self.mapping.properties.name

    @property
    def parent(self):
        if '_parent' in self.mapping._meta:
            return self.mapping._meta['_parent']['type']
        return

    def resolve_field(self, field_path):
        return self.mapping.resolve_field(field_path)

    def init(self, index=None, using=None):
        self.mapping.save(index or self.index, using=using or self.using)

    def refresh(self, index=None, using=None):
        self.mapping.update_from_es(index or self.index, using=using or self.using)

@add_metaclass(DocTypeMeta)
class DocType(ObjectBase):
    """
    Model-like class for persisting documents in elasticsearch.
    """
    def __init__(self, meta=None, **kwargs):
        meta = meta or {}
        for k in list(kwargs):
            if k.startswith('_') and k[1:] in META_FIELDS:
                meta[k] = kwargs.pop(k)

        if self._doc_type.index:
            meta.setdefault('_index', self._doc_type.index)
        super(AttrDict, self).__setattr__('meta', HitMeta(meta))

        super(DocType, self).__init__(**kwargs)

    def __getstate__(self):
        return (self.to_dict(), self.meta._d_)

    def __setstate__(self, state):
        data, meta = state
        super(AttrDict, self).__setattr__('_d_', data)
        super(AttrDict, self).__setattr__('meta', HitMeta(meta))

    def __getattr__(self, name):
        if name.startswith('_') and name[1:] in META_FIELDS:
            return getattr(self.meta, name[1:])
        return super(DocType, self).__getattr__(name)

    def __repr__(self):
        return '%s(%s)' % (
            self.__class__.__name__,
            ', '.join('%s=%r' % (key, getattr(self.meta, key)) for key in
                      ('index', 'doc_type', 'id') if key in self.meta)
        )

    def __setattr__(self, name, value):
        if name.startswith('_') and name[1:] in META_FIELDS:
            return setattr(self.meta, name[1:], value)
        return super(DocType, self).__setattr__(name, value)

    @classmethod
    def init(cls, index=None, using=None):
        """
        Create the index and populate the mappings in elasticsearch.
        """
        cls._doc_type.init(index, using)

    @classmethod
    def search(cls, using=None, index=None):
        """
        Create an :class:`~elasticsearch_dsl.Search` instance that will search
        over this ``DocType``.
        """
        return Search(
            using=using or cls._doc_type.using,
            index=index or cls._doc_type.index,
            doc_type=[cls]
        )

    @classmethod
    def get(cls, id, using=None, index=None, **kwargs):
        """
        Retrieve a single document from elasticsearch using it's ``id``.

        :arg id: ``id`` of the document to be retireved
        :arg index: elasticsearch index to use, if the ``DocType`` is
            associated with an index this can be omitted.
        :arg using: connection alias to use, defaults to ``'default'``

        Any additional keyword arguments will be passed to
        ``Elasticsearch.get`` unchanged.
        """
        es = connections.get_connection(using or cls._doc_type.using)
        doc = es.get(
            index=index or cls._doc_type.index,
            doc_type=cls._doc_type.name,
            id=id,
            **kwargs
        )
        if not doc['found']:
            return None
        return cls.from_es(doc)

    @classmethod
    def mget(cls, docs, using=None, index=None, raise_on_error=True,
             missing='none', **kwargs):
        """
        Retrieve multiple document by their ``id``\s. Returns a list of instances
        in the same order as requested.

        :arg docs: list of ``id``\s of the documents to be retireved or a list
            of document specifications as per
            https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-multi-get.html
        :arg index: elasticsearch index to use, if the ``DocType`` is
            associated with an index this can be omitted.
        :arg using: connection alias to use, defaults to ``'default'``
        :arg missing: what to do when one of the documents requested is not
            found. Valid options are ``'none'`` (use ``None``), ``'raise'`` (raise
            ``NotFoundError``) or ``'skip'`` (ignore the missing document).

        Any additional keyword arguments will be passed to
        ``Elasticsearch.mget`` unchanged.
        """
        if missing not in ('raise', 'skip', 'none'):
            raise ValueError("'missing' must be 'raise', 'skip', or 'none'.")
        es = connections.get_connection(using or cls._doc_type.using)
        body = {
            'docs': [
                doc if isinstance(doc, collections.Mapping) else {'_id': doc}
                for doc in docs
            ]
        }
        results = es.mget(
            body,
            index=index or cls._doc_type.index,
            doc_type=cls._doc_type.name,
            **kwargs
        )

        objs, error_docs, missing_docs = [], [], []
        for doc in results['docs']:
            if doc.get('found'):
                if error_docs or missing_docs:
                    # We're going to raise an exception anyway, so avoid an
                    # expensive call to cls.from_es().
                    continue

                objs.append(cls.from_es(doc))

            elif doc.get('error'):
                if raise_on_error:
                    error_docs.append(doc)
                if missing == 'none':
                    objs.append(None)

            # The doc didn't cause an error, but the doc also wasn't found.
            elif missing == 'raise':
                missing_docs.append(doc)
            elif missing == 'none':
                objs.append(None)

        if error_docs:
            error_ids = [doc['_id'] for doc in error_docs]
            message = 'Required routing/parent not provided for documents %s.'
            message %= ', '.join(error_ids)
            raise RequestError(400, message, error_docs)
        if missing_docs:
            missing_ids = [doc['_id'] for doc in missing_docs]
            message = 'Documents %s not found.' % ', '.join(missing_ids)
            raise NotFoundError(404, message, missing_docs)
        return objs

    @classmethod
    def from_es(cls, hit):
        """
        Helper method to construct an instance from a dictionary returned by
        elasticsearch.
        """
        # don't modify in place
        meta = hit.copy()
        doc = meta.pop('_source', {})

        if 'fields' in meta:
            for k, v in iteritems(meta.pop('fields')):
                if k == '_source':
                    doc.update(v)
                if k.startswith('_') and k[1:] in META_FIELDS:
                    meta[k] = v
                else:
                    doc[k] = v

        return cls(meta=meta, **doc)

    def _get_connection(self, using=None):
        return connections.get_connection(using or self._doc_type.using)
    connection = property(_get_connection)

    def _get_index(self, index=None):
        if index is None:
            index = getattr(self.meta, 'index', self._doc_type.index)
        if index is None:
            raise ValidationException('No index')
        return index

    def delete(self, using=None, index=None, **kwargs):
        """
        Delete the instance in elasticsearch.

        :arg index: elasticsearch index to use, if the ``DocType`` is
            associated with an index this can be omitted.
        :arg using: connection alias to use, defaults to ``'default'``

        Any additional keyword arguments will be passed to
        ``Elasticsearch.delete`` unchanged.
        """
        es = self._get_connection(using)
        # extract parent, routing etc from meta
        doc_meta = dict(
            (k, self.meta[k])
            for k in DELETE_META_FIELDS
            if k in self.meta
        )
        doc_meta.update(kwargs)
        es.delete(
            index=self._get_index(index),
            doc_type=self._doc_type.name,
            **doc_meta
        )

    def to_dict(self, include_meta=False):
        """
        Serialize the instance into a dictionary so that it can be saved in elasticsearch.

        :arg include_meta: if set to ``True`` will include all the metadata
            (``_index``, ``_type``, ``_id`` etc). Otherwise just the document's
            data is serialized. This is useful when passing multiple instances into
            ``elasticsearch.helpers.bulk``.
        """
        d = super(DocType, self).to_dict()
        if not include_meta:
            return d

        meta = dict(
            ('_' + k, self.meta[k])
            for k in DOC_META_FIELDS
            if k in self.meta
        )

        # in case of to_dict include the index unlike save/update/delete
        if 'index' in self.meta:
            meta['_index'] = self.meta.index
        elif self._doc_type.index:
            meta['_index'] = self._doc_type.index

        meta['_type'] = self._doc_type.name
        meta['_source'] = d
        return meta

    def update(self, using=None, index=None,  detect_noop=True, doc_as_upsert=False, **fields):
        """
        Partial update of the document, specify fields you wish to update and
        both the instance and the document in elasticsearch will be updated::

            doc = MyDocument(title='Document Title!')
            doc.save()
            doc.update(title='New Document Title!')

        :arg index: elasticsearch index to use, if the ``DocType`` is
            associated with an index this can be omitted.
        :arg using: connection alias to use, defaults to ``'default'``

        Any additional keyword arguments will be passed to
        ``Elasticsearch.update`` unchanged.
        """
        if not fields:
            raise IllegalOperation('You cannot call update() without updating individual fields. '
                                   'If you wish to update the entire object use save().')

        es = self._get_connection(using)

        # update given fields locally
        merge(self._d_, fields)

        # prepare data for ES
        doc = self.to_dict()

        # if fields were given: partial update
        if fields:
            doc = dict(
                (k, doc.get(k))
                for k in fields.keys()
            )

        # extract parent, routing etc from meta
        doc_meta = dict(
            (k, self.meta[k])
            for k in DOC_META_FIELDS
            if k in self.meta
        )
        body = {
            'doc': doc,
            'doc_as_upsert': doc_as_upsert,
            'detect_noop': detect_noop,
        }

        meta = es.update(
            index=self._get_index(index),
            doc_type=self._doc_type.name,
            body=body,
            **doc_meta
        )
        # update meta information from ES
        for k in META_FIELDS:
            if '_' + k in meta:
                setattr(self.meta, k, meta['_' + k])

    def save(self, using=None, index=None, validate=True, **kwargs):
        """
        Save the document into elasticsearch. If the document doesn't exist it
        is created, it is overwritten otherwise. Returns ``True`` if this
        operations resulted in new document being created.

        :arg index: elasticsearch index to use, if the ``DocType`` is
            associated with an index this can be omitted.
        :arg using: connection alias to use, defaults to ``'default'``
        :arg validate: set to ``False`` to skip validating the document

        Any additional keyword arguments will be passed to
        ``Elasticsearch.index`` unchanged.
        """
        if validate:
            self.full_clean()

        es = self._get_connection(using)
        # extract parent, routing etc from meta
        doc_meta = dict(
            (k, self.meta[k])
            for k in DOC_META_FIELDS
            if k in self.meta
        )
        doc_meta.update(kwargs)
        meta = es.index(
            index=self._get_index(index),
            doc_type=self._doc_type.name,
            body=self.to_dict(),
            **doc_meta
        )
        # update meta information from ES
        for k in META_FIELDS:
            if '_' + k in meta:
                setattr(self.meta, k, meta['_' + k])

        # return True/False if the document has been created/updated
        return meta['created']
<|endoftext|>


<commit_before>#!/bin/env python

import re
import logging as log
from datetime import datetime

# regex to find datetime text & associated strptime pattern for parsing
known_datetime_patterns = (
    # standard for "strace -tt"
    ('\d+:\d+:\d+\.\d+', '%H:%M:%S.%f'),

    # ftrace uptime epoch style
    ('\d+\.\d{5}', 'epoch'),

    # Local time format
    ('[A-Z][a-z]{2} [A-Z][a-z]{2} \d+ \d+:\d+:\d+ \d{4}', '%c'),
)

class TraceData(object):
    "Manage strace output"

    def __init__(self, filename):
        try:
            self.__infh = open(filename)

            # Lets start by reading some of the beginning lines
            self.beginning = self.__infh.read(1000)
            self.__infh.seek(0)

            self.init()
            self.epoch = datetime.fromtimestamp(0)

        except (IOError, OSError) as e:
            log.error('Could not open filename "%s": %s', filename, str(e))

    def init(self):
        """Initialize the trace such as establishing datetime regexs, etc"""
        self.datetime_re = None
        for pat, strp in known_datetime_patterns:
            if re.search(pat, self.beginning, re.M):
                self.datetime_re = re.compile(pat)
                self.strptime = strp
                break

    def __iter__(self):
        """Yield lines to caller"""
        for line in self.__infh:
            yield line.rstrip()

    def parse_datetime(self, data, flags=0):
        """Use initilized datetime regex and strptime patterns to return
        first valid datetime object hit within data"""
        if self.datetime_re is None:
            return self.epoch

        hits = self.datetime_re.findall(data, flags)
        if hits:
            if self.strptime == 'epoch':
                return datetime.fromtimestamp(float(hits[0]))
            else:
                return datetime.strptime(hits[0], self.strptime)
        else:
            return self.epoch

    def datetime_parsed(self):
        """Yield parsed datetime objects with each line"""
        for line in self.__infh:
            yield self.parse_datetime(line), line.rstrip()
<commit_msg>tracedata.py: Parse simple HH:MM:SS<commit_after>#!/bin/env python

import re
import logging as log
from datetime import datetime

# regex to find datetime text & associated strptime pattern for parsing
known_datetime_patterns = (
    # standard for "strace -tt"
    ('\d+:\d+:\d+\.\d+', '%H:%M:%S.%f'),

    # ftrace uptime epoch style
    ('\d+\.\d{5}', 'epoch'),

    # Local time format
    ('[A-Z][a-z]{2} [A-Z][a-z]{2} \d+ \d+:\d+:\d+ \d{4}', '%c'),

    # Only HH:MM:SS
    ('\d+:\d+:\d+', '%H:%M:%S'),
)

class TraceData(object):
    "Manage strace output"

    def __init__(self, filename):
        try:
            self.__infh = open(filename)

            # Lets start by reading some of the beginning lines
            self.beginning = self.__infh.read(1000)
            self.__infh.seek(0)

            self.init()
            self.epoch = datetime.fromtimestamp(0)

        except (IOError, OSError) as e:
            log.error('Could not open filename "%s": %s', filename, str(e))

    def init(self):
        """Initialize the trace such as establishing datetime regexs, etc"""
        self.datetime_re = None
        for pat, strp in known_datetime_patterns:
            if re.search(pat, self.beginning, re.M):
                self.datetime_re = re.compile(pat)
                self.strptime = strp
                break

    def __iter__(self):
        """Yield lines to caller"""
        for line in self.__infh:
            yield line.rstrip()

    def parse_datetime(self, data, flags=0):
        """Use initilized datetime regex and strptime patterns to return
        first valid datetime object hit within data"""
        if self.datetime_re is None:
            return self.epoch

        hits = self.datetime_re.findall(data, flags)
        if hits:
            if self.strptime == 'epoch':
                return datetime.fromtimestamp(float(hits[0]))
            else:
                return datetime.strptime(hits[0], self.strptime)
        else:
            return self.epoch

    def datetime_parsed(self):
        """Yield parsed datetime objects with each line"""
        for line in self.__infh:
            yield self.parse_datetime(line), line.rstrip()
<|endoftext|>