<commit_before># =============================================================================
#
# Copyright (c) 2016, Cisco Systems
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.
# =============================================================================

"""
NCS6K

Production Packages

External Names                                Internal Names
ncs6k-doc.pkg-5.2.4                           ncs6k-doc-5.2.4
ncs6k-li.pkg-5.2.4                            ncs6k-li-5.2.4
ncs6k-mcast.pkg-5.2.4                         ncs6k-mcast-5.2.4
ncs6k-mgbl.pkg-5.2.4                          ncs6k-mgbl-5.2.4
ncs6k-mini-x.iso-5.2.4                        ncs6k-xr-5.2.4
ncs6k-mpls.pkg-5.2.4                          ncs6k-mpls-5.2.4
ncs6k-sysadmin.iso-5.2.4                      ncs6k-sysadmin-5.2.4
ncs6k-full-x.iso-5.2.4                        ncs6k-xr-5.2.4
ncs6k-5.2.5.CSCuy47880.smu                    ncs6k-5.2.5.CSCuy47880-1.0.0  <- subversion added

Engineering Packages

External Names                                Internal Names
ncs6k-mcast.pkg-5.2.5.47I.DT_IMAGE            ncs6k-mcast-5.2.5.47I
ncs6k-mini-x.iso-6.1.0.07I.DT_IMAGE           ncs6k-xr-5.2.5.47I
ncs6k-5.2.5.47I.CSCuy47880-0.0.4.i.smu        ncs6k-5.2.5.47I.CSCuy47880-0.0.4.i

ASR9K-64

Production Packages - not finalized yet

External Names                                Internal Names
asr9k-mcast-x64-2.0.0.0-r611.x86_64.rpm       asr9k-mcast-x64-2.0.0.0-r611
asr9k-bgp-x64-1.0.0.0-r611.x86_64.rpm         asr9k-bgp-x64-1.0.0.0-r611
asr9k-mgbl-x64-3.0.0.0-r611.x86_64.rpm        asr9k-mgbl-x64-3.0.0.0-r611
asr9k-full-x64.iso-6.1.1                      asr9k-xr-6.1.1
asr9k-mini-x64.iso-6.1.1                      asr9k-xr-6.1.1

Engineering Packages

External Names                                                          Internal Names
asr9k-mcast-x64-2.0.0.0-r61116I.x86_64.rpm-6.1.1.16I.DT_IMAGE           asr9k-mcast-x64-2.0.0.0-r61116I
asr9k-bgp-x64-1.0.0.0-r61116I.x86_64.rpm-6.1.1.16I.DT_IMAGE             asr9k-bgp-x64-1.0.0.0-r61116I
asr9k-mgbl-x64-3.0.0.0-r61116I.x86_64.rpm-6.1.1.16I.DT_IMAGE            asr9k-mgbl-x64-3.0.0.0-r61116I
asr9k-full-x64.iso-6.1.1.16I.DT_IMAGE                                   asr9k-xr-6.1.1.16I
asr9k-mini-x64.iso-6.1.1.16I.DT_IMAGE                                   asr9k-xr-6.1.1.16I

NCS5K

Production Packages

External Names                                Internal Names
ncs5k-sysadmin.iso-6.0.1                      ncs5k-sysadmin-6.0.1
ncs5k-full-x.iso-6.0.1                        ncs5k-xr-6.0.1
ncs5k-mini-x.iso-6.0.1                        ncs5k-xr-6.0.1
ncs5k-mcast-2.0.0.0-r601.x86_64.rpm-6.0.1     ncs5k-mcast-2.0.0.0-r601
ncs5k-mgbl-2.0.0.0-r601.x86_64.rpm-6.0.1      ncs5k-mgbl-2.0.0.0-r601
ncs5k-mpls-2.0.0.0-r601.x86_64.rpm-6.0.1      ncs5k-mpls-2.0.0.0-r601
ncs5k-k9sec-2.0.0.0-r601.x86_64.rpm-6.0.1     ncs5k-k9sec-2.0.0.0-r601
ncs5k-isis-2.0.0.0-r601.x86_64.rpm-6.0.1      ncs5k-isis-2.0.0.0-r601
ncs5k-ospf-2.0.0.0-r601.x86_64.rpm-6.0.1      ncs5k-ospf-2.0.0.0-r601

Engineering Packages

External Names                                                Internal Names
ncs5k-mgbl-x64-3.0.0.0-r61116I.x86_64.rpm-6.0.1.16I.DT_IMAGE  ncs5k-mgbl-3.0.0.0-r60116I
ncs5k-sysadmin.iso-6.0.1                                      ncs5k-sysadmin-6.0.1.26I
ncs5k-full-x.iso-6.0.1.16I.DT_IMAGE                           ncs5k-xr-6.0.1.16I

NCS5500

Production Packages

External Names                                                Internal Names
ncs5500-eigrp-2.0.0.0-r601.x86_64.rpm-6.0.1                   ncs5500-eigrp-2.0.0.0-r601
ncs5500-isis-2.0.0.0-r601.x86_64.rpm-6.0.1                    ncs5500-isis-2.0.0.0-r601
ncs5500-k9sec-2.0.0.0-r601.x86_64.rpm-6.0.1                   ncs5500-k9sec-2.0.0.0-r601
ncs5500-m2m-2.0.0.0-r601.x86_64.rpm-6.0.1                     ncs5500-m2m-2.0.0.0-r601
ncs5500-mgbl-3.0.0.0-r601.x86_64.rpm-6.0.1                    ncs5500-mgbl-3.0.0.0-r601
ncs5500-mini-x.iso-6.0.1                                      ncs5500-xr-6.0.1
ncs5500-mpls-te-rsvp-2.0.0.0-r601.x86_64.rpm-6.0.1            ncs5500-mpls-te-rsvp-2.0.0.0-r601
ncs5500-mpls-2.0.0.0-r601.x86_64.rpm-6.0.1                    ncs5500-mpls-2.0.0.0-r601
ncs5500-ospf-1.0.0.0-r601.x86_64.rpm-6.0.1                    ncs5500-ospf-1.0.0.0-r601
ncs5500-parser-1.0.0.0-r601.x86_64.rpm-6.0.1                  ncs5500-parser-1.0.0.0-r601
"""
import re

platforms = ['asr9k', 'ncs1k', 'ncs5k', 'ncs5500', 'ncs6k']
# This list will cover all eXR platforms.  They are arranged in alphabetical order for human consumption.
package_types = [
    "bgp",
    "diags",
    "doc",
    "eigrp",
    "full",
    "fwding",
    "infra",
    "infra-test",
    "isis",
    "k9sec",
    "li",
    "m2m",
    "mcast",
    "mgbl",
    "mini",
    "mpls",
    "mpls-te-rsvp",
    "optic",
    "ospf",
    "os-support",
    "parser",
    "rm",
    "sysadmin",
    "xr"
]

version_dict = {"asr9k, ncs1k, ncs5k, ncs5500":  # 61117I or 611 or 6.1.1.17I or 6.1.1
                re.compile("(?P<VERSION>(\d+\d+\d+(\d+\w+)?)|(\d+\.\d+\.\d+(\.\d+\w+)?)(?!\.\d)(?!-))"),
                "ncs6k":                         # 5.2.4 or 5.2.4.47I
                re.compile("(?P<VERSION>\d+\.\d+\.\d+(\.\d+\w+)?)"),
                }

smu_re = re.compile("(?P<SMU>CSC[a-z]{2}\d{5})")

subversion_dict = {"asr9k, ncs1k, ncs5k, ncs5500":
                   re.compile("-(?P<SUBVERSION>\d+\.\d+\.\d+\.\d+)-"),  # 2.0.0.0
                   "ncs6k":
                   re.compile("CSC.*(?P<SUBVERSION>\d+\.\d+\.\d+?)"),   # 0.0.4
                   }

class SoftwarePackage(object):
    def __init__(self, package_name):
        self.package_name = package_name

    @property
    def platform(self):
        for platform in platforms:
            if platform + "-" in self.package_name:
                return platform
        else:
            return None

    @property
    def package_type(self):
        for package_type in package_types:
            if "-" + package_type + "-" in self.package_name:
                return package_type
        else:
            return None

    @property
    def version(self):
        if not self.platform or not self.get_values(version_dict, self.platform):
            return None

        result = re.search(self.get_values(version_dict, self.platform), self.package_name)

        return result.group("VERSION") if result else None

    @property
    def smu(self):
        result = re.search(smu_re, self.package_name)
        return result.group("SMU") if result else None

    @property
    def subversion(self):
        if not self.platform or not self.get_values(subversion_dict, self.platform):
            return None

        # For NCS6K, only need to consider subversion if it is a SMU.
        if self.platform in ["asr9k", "ncs1k", "ncs5k", "ncs5500"] or self.smu:
            result = re.search(self.get_values(subversion_dict, self.platform), self.package_name)
            return result.group("SUBVERSION") if result else None

        return None

    def get_values(self, dictionary, key):
        for keys in dictionary.keys():
            if key in keys.split(','):
                return dictionary.get(keys)
        return None

    def is_valid(self):
        return self.platform and self.version and (self.package_type or self.smu)

    def __eq__(self, other):
        package_type_same = False
        if (self.package_type == "xr" or self.package_type == "mini" or self.package_type == "full") and \
           (other.package_type == "xr" or other.package_type == "mini" or other.package_type == "full"):
            package_type_same = True

        result = self.platform == other.platform and \
            (package_type_same or self.package_type == other.package_type) and \
            self.version == other.version and \
            self.smu == other.smu and \
            (self.subversion == other.subversion if self.subversion and other.subversion else True)

        return result

    def __hash__(self):
        return hash("{}{}{}{}{}".format(
            self.platform, self.package_type, self.version, self.smu, self.subversion))

    @staticmethod
    def from_show_cmd(cmd):
        software_packages = set()
        data = cmd.split()

        for line in data:
            software_package = SoftwarePackage(line)
            if software_package.is_valid():
                software_packages.add(software_package)
        return software_packages

    @staticmethod
    def from_package_list(pkg_list):
        software_packages = set()
        for pkg in pkg_list:
            software_package = SoftwarePackage(pkg)
            if software_package.is_valid():
                software_packages.add(software_package)
        return software_packages

    def __repr__(self):
        return self.package_name

    def __str__(self):
        return self.__repr__()
<commit_msg>Fix silly error<commit_after># =============================================================================
#
# Copyright (c) 2016, Cisco Systems
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.
# =============================================================================

"""
NCS6K

Production Packages

External Names                                Internal Names
ncs6k-doc.pkg-5.2.4                           ncs6k-doc-5.2.4
ncs6k-li.pkg-5.2.4                            ncs6k-li-5.2.4
ncs6k-mcast.pkg-5.2.4                         ncs6k-mcast-5.2.4
ncs6k-mgbl.pkg-5.2.4                          ncs6k-mgbl-5.2.4
ncs6k-mini-x.iso-5.2.4                        ncs6k-xr-5.2.4
ncs6k-mpls.pkg-5.2.4                          ncs6k-mpls-5.2.4
ncs6k-sysadmin.iso-5.2.4                      ncs6k-sysadmin-5.2.4
ncs6k-full-x.iso-5.2.4                        ncs6k-xr-5.2.4
ncs6k-5.2.5.CSCuy47880.smu                    ncs6k-5.2.5.CSCuy47880-1.0.0  <- subversion added

Engineering Packages

External Names                                Internal Names
ncs6k-mcast.pkg-5.2.5.47I.DT_IMAGE            ncs6k-mcast-5.2.5.47I
ncs6k-mini-x.iso-6.1.0.07I.DT_IMAGE           ncs6k-xr-5.2.5.47I
ncs6k-5.2.5.47I.CSCuy47880-0.0.4.i.smu        ncs6k-5.2.5.47I.CSCuy47880-0.0.4.i

ASR9K-64

Production Packages - not finalized yet

External Names                                Internal Names
asr9k-mcast-x64-2.0.0.0-r611.x86_64.rpm       asr9k-mcast-x64-2.0.0.0-r611
asr9k-bgp-x64-1.0.0.0-r611.x86_64.rpm         asr9k-bgp-x64-1.0.0.0-r611
asr9k-mgbl-x64-3.0.0.0-r611.x86_64.rpm        asr9k-mgbl-x64-3.0.0.0-r611
asr9k-full-x64.iso-6.1.1                      asr9k-xr-6.1.1
asr9k-mini-x64.iso-6.1.1                      asr9k-xr-6.1.1

Engineering Packages

External Names                                                          Internal Names
asr9k-mcast-x64-2.0.0.0-r61116I.x86_64.rpm-6.1.1.16I.DT_IMAGE           asr9k-mcast-x64-2.0.0.0-r61116I
asr9k-bgp-x64-1.0.0.0-r61116I.x86_64.rpm-6.1.1.16I.DT_IMAGE             asr9k-bgp-x64-1.0.0.0-r61116I
asr9k-mgbl-x64-3.0.0.0-r61116I.x86_64.rpm-6.1.1.16I.DT_IMAGE            asr9k-mgbl-x64-3.0.0.0-r61116I
asr9k-full-x64.iso-6.1.1.16I.DT_IMAGE                                   asr9k-xr-6.1.1.16I
asr9k-mini-x64.iso-6.1.1.16I.DT_IMAGE                                   asr9k-xr-6.1.1.16I

NCS5K

Production Packages

External Names                                Internal Names
ncs5k-sysadmin.iso-6.0.1                      ncs5k-sysadmin-6.0.1
ncs5k-full-x.iso-6.0.1                        ncs5k-xr-6.0.1
ncs5k-mini-x.iso-6.0.1                        ncs5k-xr-6.0.1
ncs5k-mcast-2.0.0.0-r601.x86_64.rpm-6.0.1     ncs5k-mcast-2.0.0.0-r601
ncs5k-mgbl-2.0.0.0-r601.x86_64.rpm-6.0.1      ncs5k-mgbl-2.0.0.0-r601
ncs5k-mpls-2.0.0.0-r601.x86_64.rpm-6.0.1      ncs5k-mpls-2.0.0.0-r601
ncs5k-k9sec-2.0.0.0-r601.x86_64.rpm-6.0.1     ncs5k-k9sec-2.0.0.0-r601
ncs5k-isis-2.0.0.0-r601.x86_64.rpm-6.0.1      ncs5k-isis-2.0.0.0-r601
ncs5k-ospf-2.0.0.0-r601.x86_64.rpm-6.0.1      ncs5k-ospf-2.0.0.0-r601

Engineering Packages

External Names                                                Internal Names
ncs5k-mgbl-x64-3.0.0.0-r61116I.x86_64.rpm-6.0.1.16I.DT_IMAGE  ncs5k-mgbl-3.0.0.0-r60116I
ncs5k-sysadmin.iso-6.0.1                                      ncs5k-sysadmin-6.0.1.26I
ncs5k-full-x.iso-6.0.1.16I.DT_IMAGE                           ncs5k-xr-6.0.1.16I

NCS5500

Production Packages

External Names                                                Internal Names
ncs5500-eigrp-2.0.0.0-r601.x86_64.rpm-6.0.1                   ncs5500-eigrp-2.0.0.0-r601
ncs5500-isis-2.0.0.0-r601.x86_64.rpm-6.0.1                    ncs5500-isis-2.0.0.0-r601
ncs5500-k9sec-2.0.0.0-r601.x86_64.rpm-6.0.1                   ncs5500-k9sec-2.0.0.0-r601
ncs5500-m2m-2.0.0.0-r601.x86_64.rpm-6.0.1                     ncs5500-m2m-2.0.0.0-r601
ncs5500-mgbl-3.0.0.0-r601.x86_64.rpm-6.0.1                    ncs5500-mgbl-3.0.0.0-r601
ncs5500-mini-x.iso-6.0.1                                      ncs5500-xr-6.0.1
ncs5500-mpls-te-rsvp-2.0.0.0-r601.x86_64.rpm-6.0.1            ncs5500-mpls-te-rsvp-2.0.0.0-r601
ncs5500-mpls-2.0.0.0-r601.x86_64.rpm-6.0.1                    ncs5500-mpls-2.0.0.0-r601
ncs5500-ospf-1.0.0.0-r601.x86_64.rpm-6.0.1                    ncs5500-ospf-1.0.0.0-r601
ncs5500-parser-1.0.0.0-r601.x86_64.rpm-6.0.1                  ncs5500-parser-1.0.0.0-r601
"""
import re

platforms = ['asr9k', 'ncs1k', 'ncs5k', 'ncs5500', 'ncs6k']
# This list will cover all eXR platforms.  They are arranged in alphabetical order for human consumption.
package_types = [
    "bgp",
    "diags",
    "doc",
    "eigrp",
    "full",
    "fwding",
    "infra",
    "infra-test",
    "isis",
    "k9sec",
    "li",
    "m2m",
    "mcast",
    "mgbl",
    "mini",
    "mpls",
    "mpls-te-rsvp",
    "optic",
    "ospf",
    "os-support",
    "parser",
    "rm",
    "sysadmin",
    "xr"
]

version_dict = {"asr9k ncs1k ncs5k ncs5500":  # 61117I or 611 or 6.1.1.17I or 6.1.1
                re.compile("(?P<VERSION>(\d+\d+\d+(\d+\w+)?)|(\d+\.\d+\.\d+(\.\d+\w+)?)(?!\.\d)(?!-))"),
                "ncs6k":                         # 5.2.4 or 5.2.4.47I
                re.compile("(?P<VERSION>\d+\.\d+\.\d+(\.\d+\w+)?)"),
                }

smu_re = re.compile("(?P<SMU>CSC[a-z]{2}\d{5})")

subversion_dict = {"asr9k ncs1k ncs5k ncs5500":
                   re.compile("-(?P<SUBVERSION>\d+\.\d+\.\d+\.\d+)-"),  # 2.0.0.0
                   "ncs6k":
                   re.compile("CSC.*(?P<SUBVERSION>\d+\.\d+\.\d+?)"),   # 0.0.4
                   }

class SoftwarePackage(object):
    def __init__(self, package_name):
        self.package_name = package_name

    @property
    def platform(self):
        for platform in platforms:
            if platform + "-" in self.package_name:
                return platform
        else:
            return None

    @property
    def package_type(self):
        for package_type in package_types:
            if "-" + package_type + "-" in self.package_name:
                return package_type
        else:
            return None

    @property
    def version(self):
        if not self.platform or not self.get_values(version_dict, self.platform):
            return None

        result = re.search(self.get_values(version_dict, self.platform), self.package_name)
        return result.group("VERSION") if result else None

    @property
    def smu(self):
        result = re.search(smu_re, self.package_name)
        return result.group("SMU") if result else None

    @property
    def subversion(self):
        if not self.platform or not self.get_values(subversion_dict, self.platform):
            return None

        # For NCS6K, only need to consider subversion if it is a SMU.
        if self.platform in ["asr9k", "ncs1k", "ncs5k", "ncs5500"] or self.smu:
            result = re.search(self.get_values(subversion_dict, self.platform), self.package_name)
            return result.group("SUBVERSION") if result else None

        return None

    def get_values(self, dictionary, key):
        for keys in dictionary.keys():
            if key in keys.split():
                return dictionary.get(keys)
        return None

    def is_valid(self):
        return self.platform and self.version and (self.package_type or self.smu)

    def __eq__(self, other):
        package_type_same = False
        if (self.package_type == "xr" or self.package_type == "mini" or self.package_type == "full") and \
           (other.package_type == "xr" or other.package_type == "mini" or other.package_type == "full"):
            package_type_same = True

        result = self.platform == other.platform and \
            (package_type_same or self.package_type == other.package_type) and \
            self.version == other.version and \
            self.smu == other.smu and \
            (self.subversion == other.subversion if self.subversion and other.subversion else True)

        return result

    def __hash__(self):
        return hash("{}{}{}{}{}".format(
            self.platform, self.package_type, self.version, self.smu, self.subversion))

    @staticmethod
    def from_show_cmd(cmd):
        software_packages = set()
        data = cmd.split()

        for line in data:
            software_package = SoftwarePackage(line)
            if software_package.is_valid():
                software_packages.add(software_package)
        return software_packages

    @staticmethod
    def from_package_list(pkg_list):
        software_packages = set()
        for pkg in pkg_list:
            software_package = SoftwarePackage(pkg)
            if software_package.is_valid():
                software_packages.add(software_package)
        return software_packages

    def __repr__(self):
        return self.package_name

    def __str__(self):
        return self.__repr__()
<|endoftext|>


<commit_before># =============================================================================
#
# Copyright (c) 2016, Cisco Systems
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.
# =============================================================================

# from package_lib import SoftwarePackage
import re
from time import time
from datetime import datetime
from csmpe.plugins import CSMPlugin
from install import expand_subpkgs
from install import install_activate_reload
from install import install_activate_write_memory
from install import install_activate_issu
from csmpe.core_plugins.csm_get_software_packages.ios_xe.plugin import get_package
from csmpe.core_plugins.csm_install_operations.utils import update_device_info_udi
from utils import remove_exist_image
from utils import xe_show_platform
from utils import install_add_remove
from utils import installed_package_name

class Plugin(CSMPlugin):
    """This plugin Activates packages on the device."""
    name = "Install Activate Plugin"
    platforms = {'ASR900'}
    phases = {'Activate'}
    os = {'XE'}

    def run(self):
        """
        Performs install activate operation
        """

        rsp_count = int(self.ctx.load_data('xe_rsp_count')[0])
        pkg = self.ctx.load_data('xe_activate_pkg')[0]
        mode = self.ctx.load_data('xe_boot_mode')[0]
        folder = self.ctx.load_data('xe_install_folder')[0]

        self.ctx.info("Activate number of RSP = {}".format(rsp_count))
        self.ctx.info("Activate package = {}".format(pkg))
        self.ctx.info("Activate package mode = {}".format(mode))
        self.ctx.info("Install folder = {}".format(folder))

        self.ctx.info("Activate package(s) pending")
        self.ctx.post_status("Activate Package(s) Pending")

        prompt = self.ctx._connection.hostname

        # issu: need to copy the consolidated image to the installed folder
        if mode == 'issu':
            cmd = 'copy bootflash:' + pkg + ' ' + folder + '/' + pkg
            install_add_remove(self.ctx, cmd)
            # package = 'bootflash:' + pkg
            # remove_exist_image(self.ctx, package)

        # subpackage: need to expand the consolidated image to the installed folder
        if mode == 'subpackage':
            result = expand_subpkgs(self.ctx, rsp_count, folder, pkg)
            if not result:
                self.ctx.error("Error in extracting sub-images from the consolidated "
                               "image {}".format(pkg))
                return

        # configurations
        cmd = "configure terminal"
        self.ctx.send(cmd, wait_for_string=prompt)
        cmd = "config-register 0x2102"
        self.ctx.send(cmd, wait_for_string=prompt)

        if mode == 'issu':
            cmd = 'redundancy'
            self.ctx.send(cmd, wait_for_string=prompt)
            cmd = 'mode sso'
            self.ctx.send(cmd, wait_for_string=prompt)
        else:
            cmd = "no boot system"
            self.ctx.send(cmd, wait_for_string=prompt)
            if mode == 'consolidated':
                cmd = "boot system bootflash:" + pkg
            else:
                cmd = 'boot system ' + folder + '/packages.conf'
            self.ctx.send(cmd, wait_for_string=prompt)

        self.ctx.send('end', wait_for_string=prompt)

        cmd = "write memory"
        install_activate_write_memory(self.ctx, cmd, self.ctx._connection.hostname)
        # self.ctx.send(cmd, timeout=300, wait_for_string=prompt)

        # Start activation
        if mode == 'issu':
            cmd = 'request platform software package install node file ' + \
                  folder + '/' + pkg + ' interface-module-delay 160'
            install_activate_issu(self.ctx, cmd)
        else:
            install_activate_reload(self.ctx)

        self.ctx.info("Activate package done")

        # Verify the version
        activate_success = True
        if self.ctx._connection.os_version not in pkg:
            activate_success = False
            self.ctx.warning('The post-activate OS Version: {} while Activate package = '
                             '{}'.format(self.ctx._connection.os_version, pkg))

        # Verify the Image type
        if mode == 'issu' or mode == 'subpackage':
            pkg_conf = folder + '/packages.conf'
            image_type = installed_package_name(self.ctx, pkg_conf)
            if not image_type:
                activate_success = False
                self.ctx.warning('{} does not exist.'.format(pkg_conf))

            if image_type not in pkg:
                activate_success = False
                self.ctx.warning('The post-activate image type: {} while Activate package = '
                                 '{}'.format(image_type, pkg))
        else:
            # mode is consolidated
            output = self.ctx.send('show version | include ^System image')
            if output:
                m = re.search('(asr.*\.bin)', output)
                if m:
                    image_type = m.group(0)
                    if image_type not in pkg:
                        self.ctx.error('The post-activate image type: {} while Activate package = '
                                       '{}'.format(image_type, pkg))
                else:
                    activate_success = False
                    self.ctx.warning('System image not found in show version: {}'.format(output))
            else:
                activate_success = False
                self.ctx.warning('System image not found in show version: {}'.format(output))

        if not activate_success:
            self.ctx.error('Activte image {} has failed'.format(pkg))
        else:
            # Refresh package information
            get_package(self.ctx)
            update_device_info_udi(self.ctx)

        if mode == 'issu':
            # Remove all-in-one image from the installed folder
            package = folder + '/' + pkg
            remove_exist_image(self.ctx, package)
            if rsp_count == 2:
                package = 'stby-' + package
                remove_exist_image(self.ctx, package)

        # Verify the status of RP and SIP
        previous_data, timestamp = self.ctx.load_data('xe_show_platform')
        self.ctx.info("Pre-Activate data collected on {}".format(
            datetime.fromtimestamp(int(timestamp)).strftime('%Y-%m-%d %H:%M:%S')))
        if timestamp < time() - (60 * 60 * 2):  # two hours
            self.ctx.warning("Pre-Activate phase 'show platform' "
                             "data older than 2 hours")

        current_data = xe_show_platform(self.ctx)
        if not current_data:
            self.ctx.error("The CLI 'show platform' is not able to determine the status of RP and SIP ")
            return

        for Slot, Status in previous_data.items():
            Type = Status[0]
            previous_state = Status[1]
            current_state = current_data[Slot][1]
            if previous_state != current_state:
                if previous_state == 'ok, active' and current_state == 'ok, standby' \
                        or previous_state == 'ok, standby' and current_state == 'ok, active':
                    continue
                self.ctx.warning("Slot {} Type {} state changes after Activation".format(Slot, Type))
                self.ctx.warning("\t Pre-Activate State = {} vs. Post-Activate State = "
                                 "{}".format(previous_state, current_state))
            if 'ok' not in current_state:
                self.ctx.warning("Slot {} Type {} is not in 'ok' state after "
                                 "activation".format(Slot, Type))

        self.ctx.info("The status of RP and SIP has been verified. Please check any warnings in plugins.log")
<commit_msg>fix software version after activation<commit_after># =============================================================================
#
# Copyright (c) 2016, Cisco Systems
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE.
# =============================================================================

# from package_lib import SoftwarePackage
import re
from time import time
from datetime import datetime
from csmpe.plugins import CSMPlugin
from install import expand_subpkgs
from install import install_activate_reload
from install import install_activate_write_memory
from install import install_activate_issu
from csmpe.core_plugins.csm_get_software_packages.ios_xe.plugin import get_package
from csmpe.core_plugins.csm_install_operations.utils import update_device_info_udi
from utils import remove_exist_image
from utils import xe_show_platform
from utils import install_add_remove
from utils import installed_package_name

class Plugin(CSMPlugin):
    """This plugin Activates packages on the device."""
    name = "Install Activate Plugin"
    platforms = {'ASR900'}
    phases = {'Activate'}
    os = {'XE'}

    def run(self):
        """
        Performs install activate operation
        """

        rsp_count = int(self.ctx.load_data('xe_rsp_count')[0])
        pkg = self.ctx.load_data('xe_activate_pkg')[0]
        mode = self.ctx.load_data('xe_boot_mode')[0]
        folder = self.ctx.load_data('xe_install_folder')[0]

        self.ctx.info("Activate number of RSP = {}".format(rsp_count))
        self.ctx.info("Activate package = {}".format(pkg))
        self.ctx.info("Activate package mode = {}".format(mode))
        self.ctx.info("Install folder = {}".format(folder))

        self.ctx.info("Activate package(s) pending")
        self.ctx.post_status("Activate Package(s) Pending")

        prompt = self.ctx._connection.hostname

        # issu: need to copy the consolidated image to the installed folder
        if mode == 'issu':
            cmd = 'copy bootflash:' + pkg + ' ' + folder + '/' + pkg
            install_add_remove(self.ctx, cmd)
            # package = 'bootflash:' + pkg
            # remove_exist_image(self.ctx, package)

        # subpackage: need to expand the consolidated image to the installed folder
        if mode == 'subpackage':
            result = expand_subpkgs(self.ctx, rsp_count, folder, pkg)
            if not result:
                self.ctx.error("Error in extracting sub-images from the consolidated "
                               "image {}".format(pkg))
                return

        # configurations
        cmd = "configure terminal"
        self.ctx.send(cmd, wait_for_string=prompt)
        cmd = "config-register 0x2102"
        self.ctx.send(cmd, wait_for_string=prompt)

        if mode == 'issu':
            cmd = 'redundancy'
            self.ctx.send(cmd, wait_for_string=prompt)
            cmd = 'mode sso'
            self.ctx.send(cmd, wait_for_string=prompt)
        else:
            cmd = "no boot system"
            self.ctx.send(cmd, wait_for_string=prompt)
            if mode == 'consolidated':
                cmd = "boot system bootflash:" + pkg
            else:
                cmd = 'boot system ' + folder + '/packages.conf'
            self.ctx.send(cmd, wait_for_string=prompt)

        self.ctx.send('end', wait_for_string=prompt)

        cmd = "write memory"
        install_activate_write_memory(self.ctx, cmd, self.ctx._connection.hostname)
        # self.ctx.send(cmd, timeout=300, wait_for_string=prompt)

        # Start activation
        if mode == 'issu':
            cmd = 'request platform software package install node file ' + \
                  folder + '/' + pkg + ' interface-module-delay 160'
            install_activate_issu(self.ctx, cmd)
        else:
            install_activate_reload(self.ctx)

        self.ctx.info("Activate package done")

        # Refresh package information
        get_package(self.ctx)
        update_device_info_udi(self.ctx)

        # Verify the version
        activate_success = True
        if self.ctx._connection.os_version not in pkg:
            activate_success = False
            self.ctx.warning('The post-activate OS Version: {} while Activate package = '
                             '{}'.format(self.ctx._connection.os_version, pkg))

        # Verify the Image type
        if mode == 'issu' or mode == 'subpackage':
            pkg_conf = folder + '/packages.conf'
            image_type = installed_package_name(self.ctx, pkg_conf)
            if not image_type:
                activate_success = False
                self.ctx.warning('{} does not exist.'.format(pkg_conf))

            if image_type not in pkg:
                activate_success = False
                self.ctx.warning('The post-activate image type: {} while Activate package = '
                                 '{}'.format(image_type, pkg))
        else:
            # mode is consolidated
            output = self.ctx.send('show version | include ^System image')
            if output:
                m = re.search('(asr.*\.bin)', output)
                if m:
                    image_type = m.group(0)
                    if image_type not in pkg:
                        self.ctx.error('The post-activate image type: {} while Activate package = '
                                       '{}'.format(image_type, pkg))
                else:
                    activate_success = False
                    self.ctx.warning('System image not found in show version: {}'.format(output))
            else:
                activate_success = False
                self.ctx.warning('System image not found in show version: {}'.format(output))

        if not activate_success:
            # Refresh package information
            get_package(self.ctx)
            update_device_info_udi(self.ctx)
            self.ctx.error('Activte image {} has failed'.format(pkg))

        if mode == 'issu':
            # Remove all-in-one image from the installed folder
            package = folder + '/' + pkg
            remove_exist_image(self.ctx, package)
            if rsp_count == 2:
                package = 'stby-' + package
                remove_exist_image(self.ctx, package)

        # Verify the status of RP and SIP
        previous_data, timestamp = self.ctx.load_data('xe_show_platform')
        self.ctx.info("Pre-Activate data collected on {}".format(
            datetime.fromtimestamp(int(timestamp)).strftime('%Y-%m-%d %H:%M:%S')))
        if timestamp < time() - (60 * 60 * 2):  # two hours
            self.ctx.warning("Pre-Activate phase 'show platform' "
                             "data older than 2 hours")

        current_data = xe_show_platform(self.ctx)
        if not current_data:
            self.ctx.error("The CLI 'show platform' is not able to determine the status of RP and SIP ")
            return

        for Slot, Status in previous_data.items():
            Type = Status[0]
            previous_state = Status[1]
            current_state = current_data[Slot][1]
            if previous_state != current_state:
                if previous_state == 'ok, active' and current_state == 'ok, standby' \
                        or previous_state == 'ok, standby' and current_state == 'ok, active':
                    continue
                self.ctx.warning("Slot {} Type {} state changes after Activation".format(Slot, Type))
                self.ctx.warning("\t Pre-Activate State = {} vs. Post-Activate State = "
                                 "{}".format(previous_state, current_state))
            if 'ok' not in current_state:
                self.ctx.warning("Slot {} Type {} is not in 'ok' state after "
                                 "activation".format(Slot, Type))

        self.ctx.info("The status of RP and SIP has been verified. Please check any warnings in plugins.log")
<|endoftext|>


<commit_before>"""
Life of a webhook
-----------------
The WebhookService is started by ./inbox and runs continuously as its own
greenlet.

Now say an API client registers a webhook W. The API server calls
WebhookService.register_hook via ZeroRPC with the webhook data. We then insert
a row into the WebhookParameters table of the database, containing the webhook
data. We start a corresponding WebhookWorker, which also runs continuously as
its own greenlet.

The WebhookService asynchronously consumes the transaction log, and pushes any
events that match the filter parameters for W to the corresponding
WebhookWorker's queue. The WebhookWorker continuously monitors its queue; when
it receives a transaction, it attempts to post to the stored callback_url. If
this succeeds, we increment the min_processed_id column of the associated
WebhookParameters row, indicating that the webhook has successfully processed
all transaction log entries with id less than or equal to min_processed_id.

If the post request fails, the worker places the transaction in a bounded-size
retry queue. If the retry queue fills up, we suspend the worker to avoid
consuming arbitrarily large amounts of memory.

If the webhook service dies and is restarted, it resumes consuming the
transaction log at the minimum id across all stored min_processed_id values.
This ensures that we guarantee at-least-once delivery for webhooks in the face
of service restarts (unless the POST request fails on all retries). However, we
cannot also reasonably guarantee exactly-once delivery, and so clients that
require exactly-once semantics must deduplicate received data themselves.
"""

import calendar
import copy
from collections import defaultdict
import itertools
import json
import time
import urlparse

import gevent
import gevent.queue
import requests
from sqlalchemy import asc

from inbox.server.log import get_logger, log_uncaught_errors
from inbox.server.models import session_scope
from inbox.server.models.kellogs import cereal
from inbox.server.models.tables.base import Transaction, WebhookParameters

class EventData(object):
    """Keeps track of the data from a single transaction log entry, together
    with some bookkeeping for retrying."""
    def __init__(self, transaction):
        self.data = transaction.delta.copy()
        if transaction.additional_data is not None:
            self.data.update(transaction.additional_data)
        self.id = transaction.id

        self.retry_ts = 0
        self.retry_count = 0

    def __cmp__(self, other):
        return cmp(self.retry_ts, other.retry_ts)

    def note_failure(self, retry_interval):
        self.retry_count += 1
        self.retry_ts = int(time.time() + retry_interval)

def format_output(transaction_data, include_body):
    response = {
        'id': transaction_data['public_id'],
        'object': 'message',
        'ns': transaction_data['namespace_public_id'],
        'subject': transaction_data['subject'],
        'from': transaction_data['from_addr'],
        'to': transaction_data['to_addr'],
        'cc': transaction_data['cc_addr'],
        'bcc': transaction_data['bcc_addr'],
        'date': transaction_data['received_date'],
        'thread': transaction_data['thread']['id'],
        'size': transaction_data['size'],
        # TODO(emfree): also store block/attachment info.
    }
    if include_body:
        response['body'] = transaction_data['sanitized_body']
    return cereal(response)

def format_failure_output(hook_id, timestamp, status_code):
    response = {
        'webhook_id': hook_id,
        'timestamp': timestamp,
        'status_code': status_code
    }
    return json.dumps(response)

class WebhookService(gevent.Greenlet):
    """Asynchronously consumes the transaction log and executes registered
    webhooks."""
    def __init__(self, poll_interval=1, chunk_size=22, run_immediately=True):
        self.hooks = defaultdict(set)
        self.log = get_logger(purpose='webhooks')
        self.poll_interval = poll_interval
        self.chunk_size = chunk_size
        self.minimum_id = -1
        gevent.Greenlet.__init__(self)
        if run_immediately:
            self.start()

    def register_hook(self, namespace_id, parameter_dict):
        """Register a new webhook.

        Parameters
        ----------
        namespace_id: int
            ID for the namespace to apply the webhook on.
        hook_parameters: dictionary
            Dictionary of the hook parameters.
        """
        with session_scope() as db_session:
            hook_params = WebhookParameters(namespace_id=namespace_id,
                                            min_processed_id=self.minimum_id,
                                            **parameter_dict)
            db_session.add(hook_params)
            db_session.commit()
            hook = WebhookWorker(hook_params)
            # TODO(emfree) handle inactive webhooks
        self.hooks[namespace_id].add(hook)
        if not hook.started:
            hook.start()

    def _run(self):
        self.log.info("Running the webhook service")
        log_uncaught_errors(self._run_impl, self.log)()

    def _run_impl(self):
        self.load_hooks()
        for worker in itertools.chain(*self.hooks.values()):
            if not worker.started:
                worker.start()
        # Needed for workers to actually start
        gevent.sleep(0)
        while True:
            self.process_log()
            gevent.sleep(self.poll_interval)

    def process_log(self):
        """Scan the transaction log `self.chunk_size` entries at a time,
        publishing matching events to registered hooks."""
        with session_scope() as db_session:
            self.log.info("scanning tx log from id: {}".
                          format(self.minimum_id))
            query = db_session.query(Transaction). \
                filter(Transaction.table_name == 'message',
                       Transaction.id > self.minimum_id). \
                order_by(asc(Transaction.id)).yield_per(self.chunk_size)
            for transaction in query:
                namespace_id = transaction.namespace_id
                event_data = EventData(transaction)
                for hook in self.hooks[namespace_id]:
                    if hook.match(event_data):
                        # It's important to put a separate class instance on
                        # each queue.
                        hook.enqueue(copy.copy(event_data))
                self.minimum_id = transaction.id

    def load_hooks(self):
        """Load stored hook parameters from the database. Run once on
        startup."""
        with session_scope() as db_session:
            stored_hook_params = db_session.query(WebhookParameters).all()
            for hook_params in stored_hook_params:
                namespace_id = hook_params.namespace_id
                self.hooks[namespace_id].add(WebhookWorker(hook_params))
            if stored_hook_params:
                self.minimum_id = min(params.min_processed_id for params in
                                      stored_hook_params)

class WebhookWorker(gevent.Greenlet):
    def __init__(self, hook_parameters, max_queue_size=22):
        self.id = hook_parameters.id
        self.public_id = hook_parameters.public_id
        self.filter = hook_parameters.create_filter()
        self.min_processed_id = hook_parameters.min_processed_id
        self.include_body = hook_parameters.include_body
        self.callback_url = hook_parameters.callback_url
        self.failure_notify_url = hook_parameters.failure_notify_url
        self.max_retries = hook_parameters.max_retries
        self.retry_interval = hook_parameters.retry_interval

        self.suspended = False

        self.retry_queue = gevent.queue.Queue(max_queue_size)
        self.queue = gevent.queue.Queue(max_queue_size)
        self.log = get_logger()
        gevent.Greenlet.__init__(self)

    def _run(self):
        gevent.spawn(log_uncaught_errors(self.retry_failed, self.log))
        log_uncaught_errors(self._run_impl, self.log)()

    def _run_impl(self):
        self.log.info("Starting worker for hook id {}".format(self.id))
        while True:
            gevent.sleep(0)
            event = self.queue.get()
            self.execute(event)

    def retry_failed(self):
        """Periodically poll the retry queue, attempting to reexecute failed
        hooks. Runs in its own child greenlet."""
        while True:
            gevent.sleep(self.retry_interval)
            retry_ts = self.retry_queue.peek().retry_ts
            if retry_ts < time.time():
                event = self.retry_queue.get()
                result = self.execute(event)
                if result:
                    self.suspended = False

    def enqueue(self, data):
        if not self.suspended:
            self.queue.put(data)

    def execute(self, event):
        """Attempts to post event data to callback_url. Returns True on
        success, false otherwise. On failure, puts the event on the retry
        queue.

        Parameters
        ----------
        event: EventData
        """
        assert urlparse.urlparse.scheme == 'https', \
            'callback_url MUST be https!'
        # OMG WTF TODO (emfree): Do NOT set verify=False in prod!
        r = requests.post(self.callback_url,
                          data=format_output(event.data, self.include_body),
                          verify=False)
        if r.status_code == requests.status_codes.codes.ok:
            if self.retry_queue.empty():
                self.set_min_processed_id(event.id)
        else:
            self.log.info('Hook {0} failed at transaction {1}'.
                          format(self.id, event.id))
            if self.failure_notify_url is not None:
                timestamp = int(time.time())
                failure_output = format_failure_output(
                    hook_id=self.public_id,
                    timestamp=timestamp,
                    status_code=r.status_code)
                requests.post(self.failure_notify_url,
                              data=failure_output)
            event.note_failure()
            if event.retry_count < self.max_retries:
                try:
                    self.retry_queue.put_nowait(event)
                except gevent.queue.Full:
                    self.suspended = True
            return False

    def match(self, event):
        try:
            return self.filter.match(event.data)
        except KeyError:
            self.log.error("Could not filter data for transaction {}".
                           format(event.id))

    def set_min_processed_id(self, new_id):
        if new_id <= self.min_processed_id:
            return
        self.min_processed_id = new_id
        with session_scope() as db_session:
            stored_params = db_session.query(WebhookParameters). \
                filter_by(id=self.id).one()
            stored_params.min_processed_id = self.min_processed_id
            db_session.commit()
<commit_msg>fix a tx bug and add some logging<commit_after>"""
Life of a webhook
-----------------
The WebhookService is started by ./inbox and runs continuously as its own
greenlet.

Now say an API client registers a webhook W. The API server calls
WebhookService.register_hook via ZeroRPC with the webhook data. We then insert
a row into the WebhookParameters table of the database, containing the webhook
data. We start a corresponding WebhookWorker, which also runs continuously as
its own greenlet.

The WebhookService asynchronously consumes the transaction log, and pushes any
events that match the filter parameters for W to the corresponding
WebhookWorker's queue. The WebhookWorker continuously monitors its queue; when
it receives a transaction, it attempts to post to the stored callback_url. If
this succeeds, we increment the min_processed_id column of the associated
WebhookParameters row, indicating that the webhook has successfully processed
all transaction log entries with id less than or equal to min_processed_id.

If the post request fails, the worker places the transaction in a bounded-size
retry queue. If the retry queue fills up, we suspend the worker to avoid
consuming arbitrarily large amounts of memory.

If the webhook service dies and is restarted, it resumes consuming the
transaction log at the minimum id across all stored min_processed_id values.
This ensures that we guarantee at-least-once delivery for webhooks in the face
of service restarts (unless the POST request fails on all retries). However, we
cannot also reasonably guarantee exactly-once delivery, and so clients that
require exactly-once semantics must deduplicate received data themselves.
"""

import calendar
import copy
from collections import defaultdict
import itertools
import json
import time
import urlparse

import gevent
import gevent.queue
import requests
from sqlalchemy import asc

from inbox.server.log import get_logger, log_uncaught_errors
from inbox.server.models import session_scope
from inbox.server.models.kellogs import cereal
from inbox.server.models.tables.base import Transaction, WebhookParameters

class EventData(object):
    """Keeps track of the data from a single transaction log entry, together
    with some bookkeeping for retrying."""
    def __init__(self, transaction):
        self.id = transaction.id
        self.retry_ts = 0
        self.retry_count = 0

        if transaction.delta:
            self.data = transaction.delta.copy()
            if transaction.additional_data is not None:
                self.data.update(transaction.additional_data)

    def __cmp__(self, other):
        return cmp(self.retry_ts, other.retry_ts)

    def note_failure(self, retry_interval):
        self.retry_count += 1
        self.retry_ts = int(time.time() + retry_interval)

def format_output(transaction_data, include_body):
    response = {
        'id': transaction_data['public_id'],
        'object': 'message',
        'ns': transaction_data['namespace_public_id'],
        'subject': transaction_data['subject'],
        'from': transaction_data['from_addr'],
        'to': transaction_data['to_addr'],
        'cc': transaction_data['cc_addr'],
        'bcc': transaction_data['bcc_addr'],
        'date': transaction_data['received_date'],
        'thread': transaction_data['thread']['id'],
        'size': transaction_data['size'],
        # TODO(emfree): also store block/attachment info.
    }
    if include_body:
        response['body'] = transaction_data['sanitized_body']
    return cereal(response)

def format_failure_output(hook_id, timestamp, status_code):
    response = {
        'webhook_id': hook_id,
        'timestamp': timestamp,
        'status_code': status_code
    }
    return json.dumps(response)

class WebhookService(gevent.Greenlet):
    """Asynchronously consumes the transaction log and executes registered
    webhooks."""
    def __init__(self, poll_interval=1, chunk_size=22, run_immediately=True):
        self.hooks = defaultdict(set)
        self.log = get_logger(purpose='webhooks')
        self.poll_interval = poll_interval
        self.chunk_size = chunk_size
        self.minimum_id = -1
        gevent.Greenlet.__init__(self)
        if run_immediately:
            self.start()

    def register_hook(self, namespace_id, parameter_dict):
        """Register a new webhook.

        Parameters
        ----------
        namespace_id: int
            ID for the namespace to apply the webhook on.
        hook_parameters: dictionary
            Dictionary of the hook parameters.
        """
        with session_scope() as db_session:
            hook_params = WebhookParameters(namespace_id=namespace_id,
                                            min_processed_id=self.minimum_id,
                                            **parameter_dict)
            db_session.add(hook_params)
            db_session.commit()
            hook = WebhookWorker(hook_params)
            # TODO(emfree) handle inactive webhooks
        self.hooks[namespace_id].add(hook)
        if not hook.started:
            hook.start()

    def _run(self):
        self.log.info("Running the webhook service")
        log_uncaught_errors(self._run_impl, self.log)()

    def _run_impl(self):
        self.load_hooks()
        for worker in itertools.chain(*self.hooks.values()):
            if not worker.started:
                worker.start()
        # Needed for workers to actually start
        gevent.sleep(0)
        while True:
            self.process_log()
            gevent.sleep(self.poll_interval)

    def process_log(self):
        """Scan the transaction log `self.chunk_size` entries at a time,
        publishing matching events to registered hooks."""
        with session_scope() as db_session:
            self.log.info("scanning tx log from id: {}".
                          format(self.minimum_id))
            query = db_session.query(Transaction). \
                filter(Transaction.table_name == 'message',
                       Transaction.id > self.minimum_id). \
                order_by(asc(Transaction.id)).yield_per(self.chunk_size)
            self.log.debug("Total of {0} transactions to process".format(query.count()))
            for transaction in query:
                namespace_id = transaction.namespace_id
                event_data = EventData(transaction)
                for hook in self.hooks[namespace_id]:
                    if hook.match(event_data):
                        # It's important to put a separate class instance on
                        # each queue.
                        hook.enqueue(copy.copy(event_data))
                self.minimum_id = transaction.id
            self.log.debug("processed tx. setting min id to {0}".format(self.minimum_id))

    def load_hooks(self):
        """Load stored hook parameters from the database. Run once on
        startup."""
        with session_scope() as db_session:
            stored_hook_params = db_session.query(WebhookParameters).all()
            for hook_params in stored_hook_params:
                namespace_id = hook_params.namespace_id
                self.hooks[namespace_id].add(WebhookWorker(hook_params))
            if stored_hook_params:
                self.minimum_id = min(params.min_processed_id for params in
                                      stored_hook_params)

class WebhookWorker(gevent.Greenlet):
    def __init__(self, hook_parameters, max_queue_size=22):
        self.id = hook_parameters.id
        self.public_id = hook_parameters.public_id
        self.filter = hook_parameters.create_filter()
        self.min_processed_id = hook_parameters.min_processed_id
        self.include_body = hook_parameters.include_body
        self.callback_url = hook_parameters.callback_url
        self.failure_notify_url = hook_parameters.failure_notify_url
        self.max_retries = hook_parameters.max_retries
        self.retry_interval = hook_parameters.retry_interval

        self.suspended = False

        self.retry_queue = gevent.queue.Queue(max_queue_size)
        self.queue = gevent.queue.Queue(max_queue_size)
        self.log = get_logger()
        gevent.Greenlet.__init__(self)

    def _run(self):
        gevent.spawn(log_uncaught_errors(self.retry_failed, self.log))
        log_uncaught_errors(self._run_impl, self.log)()

    def _run_impl(self):
        self.log.info("Starting worker for hook id {}".format(self.id))
        while True:
            gevent.sleep(0)
            event = self.queue.get()
            self.execute(event)

    def retry_failed(self):
        """Periodically poll the retry queue, attempting to reexecute failed
        hooks. Runs in its own child greenlet."""
        while True:
            gevent.sleep(self.retry_interval)
            retry_ts = self.retry_queue.peek().retry_ts
            if retry_ts < time.time():
                event = self.retry_queue.get()
                result = self.execute(event)
                if result:
                    self.suspended = False

    def enqueue(self, data):
        if not self.suspended:
            self.queue.put(data)

    def execute(self, event):
        """Attempts to post event data to callback_url. Returns True on
        success, false otherwise. On failure, puts the event on the retry
        queue.

        Parameters
        ----------
        event: EventData
        """
        assert urlparse.urlparse.scheme == 'https', \
            'callback_url MUST be https!'
        # OMG WTF TODO (emfree): Do NOT set verify=False in prod!
        r = requests.post(self.callback_url,
                          data=format_output(event.data, self.include_body),
                          verify=False)
        if r.status_code == requests.status_codes.codes.ok:
            if self.retry_queue.empty():
                self.set_min_processed_id(event.id)
        else:
            self.log.info('Hook {0} failed at transaction {1}'.
                          format(self.id, event.id))
            if self.failure_notify_url is not None:
                timestamp = int(time.time())
                failure_output = format_failure_output(
                    hook_id=self.public_id,
                    timestamp=timestamp,
                    status_code=r.status_code)
                requests.post(self.failure_notify_url,
                              data=failure_output)
            event.note_failure()
            if event.retry_count < self.max_retries:
                try:
                    self.retry_queue.put_nowait(event)
                except gevent.queue.Full:
                    self.suspended = True
            return False

    def match(self, event):
        try:
            return self.filter.match(event.data)
        except KeyError:
            self.log.error("Could not filter data for transaction {}".
                           format(event.id))

    def set_min_processed_id(self, new_id):
        if new_id <= self.min_processed_id:
            return
        self.min_processed_id = new_id
        with session_scope() as db_session:
            stored_params = db_session.query(WebhookParameters). \
                filter_by(id=self.id).one()
            stored_params.min_processed_id = self.min_processed_id
            db_session.commit()
<|endoftext|>


<commit_before>import os
from urlparse import urlparse, ParseResult
from urllib import quote

from .models import Ingest, Source, Layer

layer_s3_bucket = os.getenv('TILE_SERVER_BUCKET')

def get_safe_uri(uri):
    """Parse URI and ensure the path is escaped properly

    Fixes a case where the URIs with the `|` are not escaped properly and break
    ingests

    Args:
        uri (str): uri to check for validity

    Returns:
        str
    """
    parsed = urlparse(uri)
    safe = ParseResult(parsed.scheme, parsed.netloc,
                       quote(parsed.path), parsed.params,
                       parsed.query, parsed.fragment)
    return safe.geturl()

def get_source_definition(image, extent, crs=None):
    """Create source definition from an image

    Args:
        image (Image): image to get source definition from
        extent (List[float]): extent of scene
        crs (str): optional crs of image

    Return:
        Source
    """

    uri = get_safe_uri(image.sourceUri)
    band_maps = [{'source': band.number, 'target': band.number} for band in image.bands]
    cell_size = {'width': image.resolutionMeters, 'height': image.resolutionMeters}
    extent_crs = 'epsg:4326'
    return Source(uri, extent, band_maps, cell_size, extent_crs, crs)

def get_ingest_layer(scene):
    """Create a layer definition for ingest

    Args:
        scene (Scene): scene to extract layer from

    Return:
        Layer
    """
    extent = scene.get_extent()
    sources = [get_source_definition(image, extent) for image in scene.images]
    highest_resolution_meters = min([image.resolutionMeters for image in scene.images])
    cell_size = {'width': highest_resolution_meters, 'height': highest_resolution_meters}
    output_uri = 's3://{}/layers'.format(layer_s3_bucket)
    return Layer(scene.id, output_uri, sources, cell_size)

def create_ingest_definition(scene):
    """Create ingest definition for a multiband geotiff

    Args:
         scene (Scene): scene to create an ingest definition for

    Return:
        Ingest
    """
    layer = get_ingest_layer(scene)
    id = scene.id
    return Ingest(id, [layer])
<commit_msg>Add min resolution to produce reasonable tile zoom levels<commit_after>import os
from urlparse import urlparse, ParseResult
from urllib import quote

from .models import Ingest, Source, Layer

layer_s3_bucket = os.getenv('TILE_SERVER_BUCKET')

# 1 cm resolution is as high as we go, otherwise zoom levels get ridiculous
MIN_RESOLUTION_METERS = .1

def get_safe_uri(uri):
    """Parse URI and ensure the path is escaped properly

    Fixes a case where the URIs with the `|` are not escaped properly and break
    ingests

    Args:
        uri (str): uri to check for validity

    Returns:
        str
    """
    parsed = urlparse(uri)
    safe = ParseResult(parsed.scheme, parsed.netloc,
                       quote(parsed.path), parsed.params,
                       parsed.query, parsed.fragment)
    return safe.geturl()

def get_source_definition(image, extent, crs=None):
    """Create source definition from an image

    Args:
        image (Image): image to get source definition from
        extent (List[float]): extent of scene
        crs (str): optional crs of image

    Return:
        Source
    """

    uri = get_safe_uri(image.sourceUri)

    band_maps = [{'source': band.number, 'target': band.number} for band in image.bands]
    if image.resolutionMeters < MIN_RESOLUTION_METERS:
        width = MIN_RESOLUTION_METERS
        height = MIN_RESOLUTION_METERS
    else:
        width = image.resolutionMeters
        height = image.resolutionMeters
    cell_size = {'width': height, 'height': width}
    extent_crs = 'epsg:4326'
    return Source(uri, extent, band_maps, cell_size, extent_crs, crs)

def get_ingest_layer(scene):
    """Create a layer definition for ingest

    Args:
        scene (Scene): scene to extract layer from

    Return:
        Layer
    """
    extent = scene.get_extent()
    sources = [get_source_definition(image, extent) for image in scene.images]
    highest_resolution_meters = min([image.resolutionMeters for image in scene.images])
    if highest_resolution_meters < MIN_RESOLUTION_METERS:
        width = MIN_RESOLUTION_METERS
        height = MIN_RESOLUTION_METERS
    else:
        width = highest_resolution_meters
        height = highest_resolution_meters

    cell_size = {'width': width, 'height': height}
    output_uri = 's3://{}/layers'.format(layer_s3_bucket)
    return Layer(scene.id, output_uri, sources, cell_size)

def create_ingest_definition(scene):
    """Create ingest definition for a multiband geotiff

    Args:
         scene (Scene): scene to create an ingest definition for

    Return:
        Ingest
    """
    layer = get_ingest_layer(scene)
    id = scene.id
    return Ingest(id, [layer])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Guewen Baconnier
#    Copyright 2013 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

# TODO multi-records:
# 2 external records may be converted in 1 openerp record and
# conversely. A processor must accept in input a dict where the keys are
# the name of the external / openerp model and the values are the
# records. And the same for the output records.

from ..connector import ConnectorUnit, MetaConnectorUnit, Environment
from ..exception import MappingError
from .binder import Binder

def mapping(func):
    """ Decorator declarating a mapping for a field """
    func.is_mapping = True
    return func

def changed_by(*args):
    """ Decorator for the mappings. When fields are modified, we want to modify
    only the modified fields. Using this decorator, we can specify which fields
    updates should trigger which mapping.

    If ``changed_by`` is empty, the mapping is always active.
    As far as possible, it should be used, thus, when we do an update on
    only a small number of fields on a record, the size of the output
    record will be limited to only the fields really having to be
    modified.

    :param *args: field names which trigger the mapping when modified
    """
    def register_mapping(func):
        func.changed_by = args
        return func
    return register_mapping

class MetaMapper(MetaConnectorUnit):
    """ Metaclass for Mapper """

    def __new__(meta, name, bases, attrs):
        if attrs.get('_map_methods') is None:
            attrs['_map_methods'] = {}

        cls = super(MetaMapper, meta).__new__(meta, name, bases, attrs)

        for base in bases:
            for attr_name, changed_by in getattr(base, '_map_methods', {}).iteritems():
                cls._map_methods.setdefault(attr_name, set()).update(changed_by)

        for attr_name, attr in attrs.iteritems():
            mapping = getattr(attr, 'is_mapping', None)
            if mapping:
                changed_by = getattr(attr, 'changed_by', set())
                cls._map_methods.setdefault(attr_name, set()).update(changed_by)
        return cls

class Mapper(ConnectorUnit):
    """ Transform a record to a defined output """

    __metaclass__ = MetaMapper

    # name of the OpenERP model, to be defined in concrete classes
    _model_name = None

    direct = []  # direct conversion of a field to another (from_attr, to_attr)
    children = []  # conversion of sub-records (from_attr, to_attr, model)

    _map_methods = None

    def _map_direct(self, record, from_attr, to_attr):
        raise NotImplementedError

    def _map_children(self, record, attr, model):
        raise NotImplementedError

    @property
    def map_methods(self):
        for meth, changed_by in self._map_methods.iteritems():
            yield getattr(self, meth), changed_by

    def convert(self, record, fields=None, parent_values=None):
        """ Transform an external record to an OpenERP record or the opposite

        :param record: record to transform
        :param parent_values: openerp record of the containing object
            (e.g. sale_order for a sale_order_line)
        """
        if fields is None:
            fields = {}

        result = {}

        for from_attr, to_attr in self.direct:
            if (not fields or from_attr in fields):
                # XXX not compatible with all
                # record type (wrap
                # records in a standard class representation?)
                value = self._map_direct(record,
                                         from_attr,
                                         to_attr)
                result[to_attr] = value

        for meth, changed_by in self.map_methods:
            if changed_by.intersection(fields):
                values = meth(record)
                if not values:
                    continue
                if isinstance(values, dict):
                    result.update(values)
                else:
                    raise ValueError('%s: invalid return value for the '
                                     'mapping method %s' % (values, meth))

        for from_attr, to_attr, model in self.children:
            if (not fields or from_attr in fields):
                values = self._map_children(record, from_attr, model)
                result[to_attr] = values

        return result

    def _sub_convert(self, records, processor, parent_values=None):
        """ return values of a one2many to put in the main record
        do not create the records!
        """
        raise NotImplementedError

class ImportMapper(Mapper):
    """ Transform a record from a backend to an OpenERP record """

    def _map_direct(self, record, from_attr, to_attr):
        value = record[from_attr]
        if not value:
            return False

        column = self.model._all_columns[to_attr].column
        if column._type == 'many2one':
            rel_id = record[from_attr]
            model_name = column._obj
            env = Environment(self.backend_record,
                              self.session,
                              model_name)
            binder = env.get_connector_unit(Binder)
            value = binder.to_openerp(rel_id)

            if not value:
                raise MappingError("Can not find an existing %s for external "
                                   "record %s" % (model_name, rel_id))
        return value

    def _map_children(self, record, attr, model_name):
        env = Environment(self.backend_record,
                          self.session,
                          model_name)
        mapper = env.get_connector_unit(ImportMapper)
        child_records = record[attr]  # XXX not compatible with
                                     # all record types
        return self._sub_convert(child_records, mapper,
                                 parent_values=record)

    def _sub_convert(self, records, mapper, parent_values=None):
        """ return values of a one2many to put in the main record
        do not create the records!
        """
        result = []
        for record in records:
            vals = mapper.convert(record,
                                  parent_values=parent_values)
            result.append((0, 0, vals))
        return result

class ExportMapper(Mapper):
    """ Transform a record from OpenERP to a backend record """

    def _map_direct(self, record, from_attr, to_attr):
        value = record[from_attr]
        if not value:
            return False

        column = self.model._all_columns[from_attr].column
        if column._type == 'many2one':
            rel_id = record[from_attr].id
            model_name = column._obj
            env = Environment(self.backend_record,
                              self.session,
                              model_name)
            binder = env.get_connector_unit(Binder)
            value = binder.to_backend(rel_id)

            if not value:
                raise MappingError("Can not find an external id for record "
                                   "%s in model %s" % (rel_id, model_name))
        return value

    def _map_children(self, record, attr, model_name):
        env = Environment(self.backend_record,
                          self.session,
                          model_name)
        mapper = env.get_connector_unit(ExportMapper)
        child_records = record[attr]  # XXX not compatible with
                                      # all record types
        return self._sub_convert(child_records, mapper,
                                 parent_values=record)

    def _sub_convert(self, records, mapper, parent_values=None):
        """ return values of a one2many to put in the main record
        do not create the records!
        """
        result = []
        for record in records:
            vals = mapper.convert(record,
                                  parent_values=parent_values)
            result.append(vals)
        return result
<commit_msg>[FIX] previous broken fix on unit/mapper.py<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Guewen Baconnier
#    Copyright 2013 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

# TODO multi-records:
# 2 external records may be converted in 1 openerp record and
# conversely. A processor must accept in input a dict where the keys are
# the name of the external / openerp model and the values are the
# records. And the same for the output records.

from ..connector import ConnectorUnit, MetaConnectorUnit, Environment
from ..exception import MappingError
from .binder import Binder

def mapping(func):
    """ Decorator declarating a mapping for a field """
    func.is_mapping = True
    return func

def changed_by(*args):
    """ Decorator for the mappings. When fields are modified, we want to modify
    only the modified fields. Using this decorator, we can specify which fields
    updates should trigger which mapping.

    If ``changed_by`` is empty, the mapping is always active.
    As far as possible, it should be used, thus, when we do an update on
    only a small number of fields on a record, the size of the output
    record will be limited to only the fields really having to be
    modified.

    :param *args: field names which trigger the mapping when modified
    """
    def register_mapping(func):
        func.changed_by = args
        return func
    return register_mapping

class MetaMapper(MetaConnectorUnit):
    """ Metaclass for Mapper """

    def __new__(meta, name, bases, attrs):
        if attrs.get('_map_methods') is None:
            attrs['_map_methods'] = {}

        cls = super(MetaMapper, meta).__new__(meta, name, bases, attrs)

        for base in bases:
            for attr_name, changed_by in getattr(base, '_map_methods', {}).iteritems():
                cls._map_methods.setdefault(attr_name, set()).update(changed_by)

        for attr_name, attr in attrs.iteritems():
            mapping = getattr(attr, 'is_mapping', None)
            if mapping:
                changed_by = getattr(attr, 'changed_by', set())
                cls._map_methods.setdefault(attr_name, set()).update(changed_by)
        return cls

class Mapper(ConnectorUnit):
    """ Transform a record to a defined output """

    __metaclass__ = MetaMapper

    # name of the OpenERP model, to be defined in concrete classes
    _model_name = None

    direct = []  # direct conversion of a field to another (from_attr, to_attr)
    children = []  # conversion of sub-records (from_attr, to_attr, model)

    _map_methods = None

    def _map_direct(self, record, from_attr, to_attr):
        raise NotImplementedError

    def _map_children(self, record, attr, model):
        raise NotImplementedError

    @property
    def map_methods(self):
        for meth, changed_by in self._map_methods.iteritems():
            yield getattr(self, meth), changed_by

    def convert(self, record, fields=None, parent_values=None):
        """ Transform an external record to an OpenERP record or the opposite

        :param record: record to transform
        :param parent_values: openerp record of the containing object
            (e.g. sale_order for a sale_order_line)
        """
        if fields is None:
            fields = {}

        result = {}

        for from_attr, to_attr in self.direct:
            if (not fields or from_attr in fields):
                # XXX not compatible with all
                # record type (wrap
                # records in a standard class representation?)
                value = self._map_direct(record,
                                         from_attr,
                                         to_attr)
                result[to_attr] = value

        for meth, changed_by in self.map_methods:

            if (not fields or not changed_by or
                    changed_by.intersection(fields)):
                values = meth(record)
                if not values:
                    continue
                if isinstance(values, dict):
                    result.update(values)
                else:
                    raise ValueError('%s: invalid return value for the '
                                     'mapping method %s' % (values, meth))

        for from_attr, to_attr, model in self.children:
            if (not fields or from_attr in fields):
                values = self._map_children(record, from_attr, model)
                result[to_attr] = values

        return result

    def _sub_convert(self, records, processor, parent_values=None):
        """ return values of a one2many to put in the main record
        do not create the records!
        """
        raise NotImplementedError

class ImportMapper(Mapper):
    """ Transform a record from a backend to an OpenERP record """

    def _map_direct(self, record, from_attr, to_attr):
        value = record[from_attr]
        if not value:
            return False

        column = self.model._all_columns[to_attr].column
        if column._type == 'many2one':
            rel_id = record[from_attr]
            model_name = column._obj
            env = Environment(self.backend_record,
                              self.session,
                              model_name)
            binder = env.get_connector_unit(Binder)
            value = binder.to_openerp(rel_id)

            if not value:
                raise MappingError("Can not find an existing %s for external "
                                   "record %s" % (model_name, rel_id))
        return value

    def _map_children(self, record, attr, model_name):
        env = Environment(self.backend_record,
                          self.session,
                          model_name)
        mapper = env.get_connector_unit(ImportMapper)
        child_records = record[attr]  # XXX not compatible with
                                     # all record types
        return self._sub_convert(child_records, mapper,
                                 parent_values=record)

    def _sub_convert(self, records, mapper, parent_values=None):
        """ return values of a one2many to put in the main record
        do not create the records!
        """
        result = []
        for record in records:
            vals = mapper.convert(record,
                                  parent_values=parent_values)
            result.append((0, 0, vals))
        return result

class ExportMapper(Mapper):
    """ Transform a record from OpenERP to a backend record """

    def _map_direct(self, record, from_attr, to_attr):
        value = record[from_attr]
        if not value:
            return False

        column = self.model._all_columns[from_attr].column
        if column._type == 'many2one':
            rel_id = record[from_attr].id
            model_name = column._obj
            env = Environment(self.backend_record,
                              self.session,
                              model_name)
            binder = env.get_connector_unit(Binder)
            value = binder.to_backend(rel_id)

            if not value:
                raise MappingError("Can not find an external id for record "
                                   "%s in model %s" % (rel_id, model_name))
        return value

    def _map_children(self, record, attr, model_name):
        env = Environment(self.backend_record,
                          self.session,
                          model_name)
        mapper = env.get_connector_unit(ExportMapper)
        child_records = record[attr]  # XXX not compatible with
                                      # all record types
        return self._sub_convert(child_records, mapper,
                                 parent_values=record)

    def _sub_convert(self, records, mapper, parent_values=None):
        """ return values of a one2many to put in the main record
        do not create the records!
        """
        result = []
        for record in records:
            vals = mapper.convert(record,
                                  parent_values=parent_values)
            result.append(vals)
        return result
<|endoftext|>


<commit_before>import os
import sys

from collections import defaultdict, OrderedDict

from marisa_trie import BytesTrie

this_dir = os.path.realpath(os.path.dirname(__file__))
sys.path.append(os.path.realpath(os.path.join(os.pardir, os.pardir)))

sys.path.append(os.path.realpath(os.path.join(os.pardir, os.pardir, os.pardir, 'python')))

from geodata.encoding import safe_decode
from geodata.i18n.unicode_paths import DATA_DIR
from geodata.i18n.normalize import strip_accents
from geodata.i18n.unicode_properties import get_chars_by_script, get_script_languages
from address_normalizer.text.normalize import PhraseFilter
from address_normalizer.text.tokenize import *

WELL_REPRESENTED_LANGUAGES = set(['en', 'fr', 'it', 'de', 'nl', 'es'])

DICTIONARIES_DIR = os.path.join(DATA_DIR, 'dictionaries')

PREFIX_KEY = u'\x02'
SUFFIX_KEY = u'\x03'

POSSIBLE_ROMAN_NUMERALS = set(['i', 'ii', 'iii', 'iv', 'v', 'vi', 'vii', 'viii', 'ix',
                               'x', 'xi', 'xii', 'xiii', 'xiv', 'xv', 'xvi', 'xvii', 'xviii', 'xix',
                               'xx', 'xxx', 'xl', 'l', 'lx', 'lxx', 'lxxx', 'xc',
                               'c', 'cc', 'ccc', 'cd', 'd', 'dc', 'dcc', 'dccc', 'cm',
                               'm', 'mm', 'mmm', 'mmmm'])

class DictionaryPhraseFilter(PhraseFilter):
    def __init__(self, *dictionaries):
        self.dictionaries = dictionaries

    def serialize(self, s):
        return s

    def deserialize(self, s):
        return s

    def configure(self, base_dir=DICTIONARIES_DIR):
        kvs = defaultdict(OrderedDict)
        for lang in os.listdir(DICTIONARIES_DIR):
            for filename in self.dictionaries:
                is_suffix_dictionary = 'suffixes' in filename
                is_prefix_dictionary = 'prefixes' in filename

                path = os.path.join(DICTIONARIES_DIR, lang, filename)
                if not os.path.exists(path):
                    continue

                for line in open(path):
                    line = line.strip()
                    if not line:
                        continue

                    phrases = safe_decode(line).split(u'|')
                    if not phrases:
                        continue
                    canonical = strip_accents(phrases[0])

                    for phrase in phrases:

                        if phrase in POSSIBLE_ROMAN_NUMERALS:
                            continue

                        is_canonical = strip_accents(phrase) == canonical

                        if is_suffix_dictionary:
                            phrase = SUFFIX_KEY + phrase[::-1]
                        elif is_prefix_dictionary:
                            phrase = PREFIX_KEY + phrase

                        kvs[phrase][lang] = is_canonical

        kvs = [(k, '|'.join([v, str(int(c))])) for k, vals in kvs.iteritems() for v, c in vals.iteritems()]

        self.trie = BytesTrie(kvs)
        self.configured = True

    def search_substring(self, s):
        if len(s) == 0:
            return None, 0

        for i in xrange(len(s) + 1):
            if not self.trie.has_keys_with_prefix(s[:i]):
                i -= 1
                break
        if i > 0:
            return (self.trie.get(s[:i]), i)
        else:
            return None, 0

    def basic_filter(self, tokens):
        return super(DictionaryPhraseFilter, self).filter(tokens)

    def filter(self, tokens):
        for c, t, data in self.basic_filter(tokens):
            if c != token_types.PHRASE:
                token = t[1]
                token_len = len(token)
                suffix_search, suffix_len = self.search_substring(SUFFIX_KEY + token[::-1])

                if suffix_search and self.trie.get(token[token_len - (suffix_len - len(SUFFIX_KEY)):].rstrip('.')):
                    yield (token_types.PHRASE, [(c,) + t], suffix_search)
                    continue
                prefix_search, prefix_len = self.search_substring(PREFIX_KEY + token)
                if prefix_search and self.trie.get(token[:(prefix_len - len(PREFIX_KEY))]):
                    yield (token_types.PHRASE, [(c,) + t], prefix_search)
                    continue
            yield c, t, data

street_types_gazetteer = DictionaryPhraseFilter('street_types.txt',
                                                'directionals.txt',
                                                'concatenated_suffixes_separable.txt',
                                                'concatenated_suffixes_inseparable.txt',
                                                'concatenated_prefixes_separable.txt',
                                                'stopwords.txt',)

char_scripts = get_chars_by_script()
script_languages = get_script_languages()

UNKNOWN_SCRIPT = 'Unknown'
COMMON_SCRIPT = 'Common'
MAX_ASCII = 127

def get_string_script(s):
    s = safe_decode(s)
    script = last_script = UNKNOWN_SCRIPT
    is_ascii = True
    script_len = 0
    for c in s:
        script = char_scripts[ord(c)]
        if script == COMMON_SCRIPT and last_script != UNKNOWN_SCRIPT:
            script = last_script
        if last_script != script and last_script != UNKNOWN_SCRIPT and last_script != COMMON_SCRIPT:
            break
        is_ascii = is_ascii and ord(c) <= MAX_ASCII
        script_len += 1
        if script != UNKNOWN_SCRIPT:
            last_script = script
    return (last_script, script_len, is_ascii)

UNKNOWN_LANGUAGE = 'unk'
AMBIGUOUS_LANGUAGE = 'xxx'

def disambiguate_language(text, languages):
    num_defaults = sum((1 for lang, default in languages if default))
    valid_languages = OrderedDict(languages)
    tokens = tokenize(safe_decode(text).replace(u'-', u' ').lower())

    current_lang = None
    possible_lang = None

    seen_languages = set()

    for c, t, data in street_types_gazetteer.filter(tokens):
        if c == token_types.PHRASE:
            valid = []

            for d in data:
                lang, canonical = d.split('|')
                canonical = int(canonical)
                if lang not in valid_languages:
                    continue
                is_default = valid_languages[lang]

                if canonical or is_default:
                    valid.append(lang)
                elif not seen_languages and len(t[0][1]) > 1:
                    possible_lang = lang if possible_lang is None or possible_lang == lang else None

            if seen_languages and valid and not any((l in seen_languages for l in valid)):
                return AMBIGUOUS_LANGUAGE

            if len(valid) == 1:
                current_lang = valid[0]
            else:
                valid = [l for l in valid if valid_languages.get(l)]
                if len(valid) == 1 and current_lang is not None and valid[0] != current_lang:
                    return AMBIGUOUS_LANGUAGE
                elif len(valid) == 1:
                    current_lang = valid[0]

            seen_languages.update(valid)

    if current_lang is not None:
        return current_lang
    elif possible_lang is not None:
        return possible_lang
    return UNKNOWN_LANGUAGE
<commit_msg>[languages] Adding non-canonicals only for streets, prefixes and suffixes. Better handling of default langauges, abbreviations and ambiguity<commit_after>import os
import sys

from collections import defaultdict, OrderedDict

from marisa_trie import BytesTrie

this_dir = os.path.realpath(os.path.dirname(__file__))
sys.path.append(os.path.realpath(os.path.join(os.pardir, os.pardir)))

sys.path.append(os.path.realpath(os.path.join(os.pardir, os.pardir, os.pardir, 'python')))

from geodata.encoding import safe_decode
from geodata.i18n.unicode_paths import DATA_DIR
from geodata.i18n.normalize import strip_accents
from geodata.i18n.unicode_properties import get_chars_by_script, get_script_languages
from address_normalizer.text.normalize import PhraseFilter
from address_normalizer.text.tokenize import *

WELL_REPRESENTED_LANGUAGES = set(['en', 'fr', 'it', 'de', 'nl', 'es'])

DICTIONARIES_DIR = os.path.join(DATA_DIR, 'dictionaries')

PREFIX_KEY = u'\x02'
SUFFIX_KEY = u'\x03'

POSSIBLE_ROMAN_NUMERALS = set(['i', 'ii', 'iii', 'iv', 'v', 'vi', 'vii', 'viii', 'ix',
                               'x', 'xi', 'xii', 'xiii', 'xiv', 'xv', 'xvi', 'xvii', 'xviii', 'xix',
                               'xx', 'xxx', 'xl', 'l', 'lx', 'lxx', 'lxxx', 'xc',
                               'c', 'cc', 'ccc', 'cd', 'd', 'dc', 'dcc', 'dccc', 'cm',
                               'm', 'mm', 'mmm', 'mmmm'])

class DictionaryPhraseFilter(PhraseFilter):
    def __init__(self, *dictionaries):
        self.dictionaries = dictionaries

    def serialize(self, s):
        return s

    def deserialize(self, s):
        return s

    def configure(self, base_dir=DICTIONARIES_DIR):
        kvs = defaultdict(OrderedDict)
        for lang in os.listdir(DICTIONARIES_DIR):
            for filename in self.dictionaries:
                is_suffix_dictionary = 'suffixes' in filename
                is_prefix_dictionary = 'prefixes' in filename
                is_street_types_dictionary = 'street_types' in filename

                path = os.path.join(DICTIONARIES_DIR, lang, filename)
                if not os.path.exists(path):
                    continue

                for line in open(path):
                    line = line.strip()
                    if not line:
                        continue

                    phrases = safe_decode(line).split(u'|')
                    if not phrases:
                        continue
                    canonical = strip_accents(phrases[0])

                    for phrase in phrases:

                        if phrase in POSSIBLE_ROMAN_NUMERALS:
                            continue

                        is_canonical = strip_accents(phrase) == canonical

                        if is_suffix_dictionary:
                            phrase = SUFFIX_KEY + phrase[::-1]
                        elif is_prefix_dictionary:
                            phrase = PREFIX_KEY + phrase

                        if is_canonical or is_street_types_dictionary or is_prefix_dictionary or is_suffix_dictionary:
                            kvs[phrase][lang] = is_canonical

        kvs = [(k, '|'.join([v, str(int(c))])) for k, vals in kvs.iteritems() for v, c in vals.iteritems()]

        self.trie = BytesTrie(kvs)
        self.configured = True

    def search_substring(self, s):
        if len(s) == 0:
            return None, 0

        for i in xrange(len(s) + 1):
            if not self.trie.has_keys_with_prefix(s[:i]):
                i -= 1
                break
        if i > 0:
            return (self.trie.get(s[:i]), i)
        else:
            return None, 0

    def basic_filter(self, tokens):
        return super(DictionaryPhraseFilter, self).filter(tokens)

    def filter(self, tokens):
        for c, t, data in self.basic_filter(tokens):
            if c != token_types.PHRASE:
                token = t[1]
                token_len = len(token)
                suffix_search, suffix_len = self.search_substring(SUFFIX_KEY + token[::-1])

                if suffix_search and self.trie.get(token[token_len - (suffix_len - len(SUFFIX_KEY)):].rstrip('.')):
                    yield (token_types.PHRASE, [(c,) + t], suffix_search)
                    continue
                prefix_search, prefix_len = self.search_substring(PREFIX_KEY + token)
                if prefix_search and self.trie.get(token[:(prefix_len - len(PREFIX_KEY))]):
                    yield (token_types.PHRASE, [(c,) + t], prefix_search)
                    continue
            yield c, t, data

street_types_gazetteer = DictionaryPhraseFilter('street_types.txt',
                                                'directionals.txt',
                                                'concatenated_suffixes_separable.txt',
                                                'concatenated_suffixes_inseparable.txt',
                                                'concatenated_prefixes_separable.txt',
                                                'stopwords.txt',)

char_scripts = get_chars_by_script()
script_languages = get_script_languages()

UNKNOWN_SCRIPT = 'Unknown'
COMMON_SCRIPT = 'Common'
MAX_ASCII = 127

def get_string_script(s):
    s = safe_decode(s)
    script = last_script = UNKNOWN_SCRIPT
    is_ascii = True
    script_len = 0
    for c in s:
        script = char_scripts[ord(c)]
        if script == COMMON_SCRIPT and last_script != UNKNOWN_SCRIPT:
            script = last_script
        if last_script != script and last_script != UNKNOWN_SCRIPT and last_script != COMMON_SCRIPT:
            break
        is_ascii = is_ascii and ord(c) <= MAX_ASCII
        script_len += 1
        if script != UNKNOWN_SCRIPT:
            last_script = script
    return (last_script, script_len, is_ascii)

UNKNOWN_LANGUAGE = 'unk'
AMBIGUOUS_LANGUAGE = 'xxx'

def disambiguate_language(text, languages):
    num_defaults = sum((1 for lang, default in languages if default))
    valid_languages = OrderedDict(languages)
    tokens = tokenize(safe_decode(text).replace(u'-', u' ').lower())

    current_lang = None
    possible_lang = None

    seen_languages = set()

    for c, t, data in street_types_gazetteer.filter(tokens):
        if c == token_types.PHRASE:
            valid = []

            for d in data:
                lang, canonical = d.split('|')
                canonical = int(canonical)
                if lang not in valid_languages:
                    continue
                is_default = valid_languages[lang]

                if canonical or (is_default and not current_lang):
                    valid.append(lang)
                elif is_default and num_defaults > 1 and current_lang != lang:
                    return AMBIGUOUS_LANGUAGE
                elif not seen_languages and len(t[0][1]) > 1:
                    possible_lang = lang if possible_lang is None or possible_lang == lang else None

            if seen_languages and valid and not any((l in seen_languages for l in valid)):
                return AMBIGUOUS_LANGUAGE

            if len(valid) == 1:
                current_lang = valid[0]
            else:
                valid = [l for l in valid if valid_languages.get(l)]
                if len(valid) == 1 and current_lang is not None and valid[0] != current_lang:
                    return AMBIGUOUS_LANGUAGE
                elif len(valid) == 1:
                    current_lang = valid[0]

            seen_languages.update(valid)

    if current_lang is not None:
        return current_lang
    elif possible_lang is not None:
        return possible_lang
    return UNKNOWN_LANGUAGE
<|endoftext|>