<commit_before>from django.conf import settings
from django.test import SimpleTestCase
from django.test.utils import override_settings
from corehq.pillows.utils import get_all_expected_es_indices

class ProdIndexManagementTest(SimpleTestCase):
    @classmethod
    def setUpClass(cls):
        cls._PILLOWTOPS = settings.PILLOWTOPS
        if not settings.PILLOWTOPS:
            # assumes HqTestSuiteRunner, which blanks this out and saves a copy here
            settings.PILLOWTOPS = settings._PILLOWTOPS

    @classmethod
    def tearDownClass(cls):
        settings.PILLOWTOPS = cls._PILLOWTOPS

    @override_settings(SERVER_ENVIRONMENT='production')
    def test_prod_config(self):
        found_prod_indices = [info.to_json() for info in get_all_expected_es_indices()]
        for info in found_prod_indices:
            # for now don't test these two properties, just ensure they exist
            self.assertTrue(info['mapping'])
            del info['mapping']
        found_prod_indices = sorted(found_prod_indices, key=lambda info: info['index'])
        self.assertEqual(EXPECTED_PROD_INDICES, found_prod_indices)

EXPECTED_PROD_INDICES = [
    {
        "alias": "case_search",
        "index": "test_case_search_2016-03-15",
        "type": "case",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "whitespace"
                        },
                        "sortable_exact": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "keyword"
                        }
                    }
                }
            }
        }
    },
    {
        "alias": "hqapps",
        "index": "test_hqapps_2016-10-20_1835",
        "type": "app",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "type": "custom",
                            "tokenizer": "whitespace",
                            "filter": ["lowercase"]
                        },
                    }
                }
            }
        }
    },
    {
        "alias": "hqcases",
        "index": "test_hqcases_2017-03-28",
        "type": "case",
        "meta": {
            "settings": {
                'number_of_shards': 10,
                "analysis": {
                    "analyzer": {
                        "default": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "whitespace"
                        },
                        "sortable_exact": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "keyword"
                        }
                    }
                }
            }
        }
    },
    {
        "alias": "hqdomains",
        "index": "test_hqdomains_2016-08-08",
        "type": "hqdomain",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "type": "custom",
                            "tokenizer": "whitespace",
                            "filter": ["lowercase"]
                        },
                        "comma": {
                            "type": "pattern",
                            "pattern": "\s*,\s*"
                        },
                    }
                }
            }
        }
    },
    {
        "alias": "hqgroups",
        "index": "test_hqgroups_20150403_1501",
        "type": "group",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "whitespace"
                        },
                        "sortable_exact": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "keyword"
                        }
                    }
                }
            }
        }
    },
    {
        "alias": "hqusers",
        "index": "test_hqusers_2017-02-03",
        "type": "user",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "type": "custom",
                            "tokenizer": "whitespace",
                            "filter": ["lowercase"]
                        },
                    }
                }
            }
        }
    },
    {
        "alias": "ledgers",
        "index": "test_ledgers_2016-03-15",
        "type": "ledger",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "whitespace"
                        },
                        "sortable_exact": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "keyword"
                        }
                    }
                }
            }
        }
    },
    {
        "alias": "report_cases",
        "index": "test_report_cases_czei39du507m9mmpqk3y01x72a3ux4p0",
        "type": "report_case",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "whitespace"
                        },
                        "sortable_exact": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "keyword"
                        }
                    }
                }
            }
        }
    },
    {
        "alias": "report_xforms",
        "index": "test_report_xforms_20160824_1708",
        "type": "report_xform",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "whitespace"
                        },
                        "sortable_exact": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "keyword"
                        }
                    }
                }
            }
        }
    },
    {
        "alias": "smslogs",
        "index": "test_smslogs_2017-02-09",
        "type": "sms",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "whitespace"
                        },
                        "sortable_exact": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "keyword"
                        }
                    }
                }
            }
        }
    },
    {
        "alias": "xforms",
        "index": "test_xforms_2017-03-28",
        "type": "xform",
        "meta": {
            "settings": {
                'number_of_shards': 10,
                "analysis": {
                    "analyzer": {
                        "default": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "whitespace"
                        },
                        "sortable_exact": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "keyword"
                        }
                    }
                }
            }
        }
    }
]
<commit_msg>fix comment [ci skip]<commit_after>from django.conf import settings
from django.test import SimpleTestCase
from django.test.utils import override_settings
from corehq.pillows.utils import get_all_expected_es_indices

class ProdIndexManagementTest(SimpleTestCase):
    @classmethod
    def setUpClass(cls):
        cls._PILLOWTOPS = settings.PILLOWTOPS
        if not settings.PILLOWTOPS:
            # assumes HqTestSuiteRunner, which blanks this out and saves a copy here
            settings.PILLOWTOPS = settings._PILLOWTOPS

    @classmethod
    def tearDownClass(cls):
        settings.PILLOWTOPS = cls._PILLOWTOPS

    @override_settings(SERVER_ENVIRONMENT='production')
    def test_prod_config(self):
        found_prod_indices = [info.to_json() for info in get_all_expected_es_indices()]
        for info in found_prod_indices:
            # for now don't test this property, just ensure it exist
            self.assertTrue(info['mapping'])
            del info['mapping']
        found_prod_indices = sorted(found_prod_indices, key=lambda info: info['index'])
        self.assertEqual(EXPECTED_PROD_INDICES, found_prod_indices)

EXPECTED_PROD_INDICES = [
    {
        "alias": "case_search",
        "index": "test_case_search_2016-03-15",
        "type": "case",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "whitespace"
                        },
                        "sortable_exact": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "keyword"
                        }
                    }
                }
            }
        }
    },
    {
        "alias": "hqapps",
        "index": "test_hqapps_2016-10-20_1835",
        "type": "app",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "type": "custom",
                            "tokenizer": "whitespace",
                            "filter": ["lowercase"]
                        },
                    }
                }
            }
        }
    },
    {
        "alias": "hqcases",
        "index": "test_hqcases_2017-03-28",
        "type": "case",
        "meta": {
            "settings": {
                'number_of_shards': 10,
                "analysis": {
                    "analyzer": {
                        "default": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "whitespace"
                        },
                        "sortable_exact": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "keyword"
                        }
                    }
                }
            }
        }
    },
    {
        "alias": "hqdomains",
        "index": "test_hqdomains_2016-08-08",
        "type": "hqdomain",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "type": "custom",
                            "tokenizer": "whitespace",
                            "filter": ["lowercase"]
                        },
                        "comma": {
                            "type": "pattern",
                            "pattern": "\s*,\s*"
                        },
                    }
                }
            }
        }
    },
    {
        "alias": "hqgroups",
        "index": "test_hqgroups_20150403_1501",
        "type": "group",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "whitespace"
                        },
                        "sortable_exact": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "keyword"
                        }
                    }
                }
            }
        }
    },
    {
        "alias": "hqusers",
        "index": "test_hqusers_2017-02-03",
        "type": "user",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "type": "custom",
                            "tokenizer": "whitespace",
                            "filter": ["lowercase"]
                        },
                    }
                }
            }
        }
    },
    {
        "alias": "ledgers",
        "index": "test_ledgers_2016-03-15",
        "type": "ledger",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "whitespace"
                        },
                        "sortable_exact": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "keyword"
                        }
                    }
                }
            }
        }
    },
    {
        "alias": "report_cases",
        "index": "test_report_cases_czei39du507m9mmpqk3y01x72a3ux4p0",
        "type": "report_case",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "whitespace"
                        },
                        "sortable_exact": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "keyword"
                        }
                    }
                }
            }
        }
    },
    {
        "alias": "report_xforms",
        "index": "test_report_xforms_20160824_1708",
        "type": "report_xform",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "whitespace"
                        },
                        "sortable_exact": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "keyword"
                        }
                    }
                }
            }
        }
    },
    {
        "alias": "smslogs",
        "index": "test_smslogs_2017-02-09",
        "type": "sms",
        "meta": {
            "settings": {
                "analysis": {
                    "analyzer": {
                        "default": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "whitespace"
                        },
                        "sortable_exact": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "keyword"
                        }
                    }
                }
            }
        }
    },
    {
        "alias": "xforms",
        "index": "test_xforms_2017-03-28",
        "type": "xform",
        "meta": {
            "settings": {
                'number_of_shards': 10,
                "analysis": {
                    "analyzer": {
                        "default": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "whitespace"
                        },
                        "sortable_exact": {
                            "filter": [
                                "lowercase"
                            ],
                            "type": "custom",
                            "tokenizer": "keyword"
                        }
                    }
                }
            }
        }
    }
]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# FOGLAMP_BEGIN
# See: http://foglamp.readthedocs.io/
# FOGLAMP_END

""" System tests that obtain the set of available packages for the current platform
    It then installs each of those plugin packages via REST API endpoints
"""
import os
import subprocess
import http.client
import json
import pytest
import py

__author__ = "Ashish Jabble"
__copyright__ = "Copyright (c) 2019 Dianomic Systems Inc."
__license__ = "Apache 2.0"
__version__ = "${VERSION}"

available_pkg = []
# plugins installed by default 4
counter = 4

@pytest.fixture
def reset_packages():
    try:
        subprocess.run(["$FOGLAMP_ROOT/tests/system/lab/remove"], shell=True, check=True)
    except subprocess.CalledProcessError:
        assert False, "reset package script failed"

@pytest.fixture
def setup_package(package_build_version):
    try:
        subprocess.run(["$FOGLAMP_ROOT/tests/system/python/scripts/package/setup {}".format(package_build_version)],
                       shell=True, check=True)
    except subprocess.CalledProcessError:
        assert False, "setup package script failed"

def load_data_from_json():
    _dir = os.path.dirname(os.path.realpath(__file__))
    file_path = py.path.local(_dir).join('/').join('data/package_list.json')
    with open(str(file_path)) as data_file:
        json_data = json.load(data_file)
    return json_data

class TestPackages:

    def test_reset_and_setup(self, reset_packages, setup_package):
        # TODO: Remove this workaround
        # Use better setup & teardown methods
        pass

    def test_ping(self, foglamp_url):
        conn = http.client.HTTPConnection(foglamp_url)
        conn.request("GET", '/foglamp/ping')
        r = conn.getresponse()
        assert 200 == r.status
        r = r.read().decode()
        jdoc = json.loads(r)
        assert len(jdoc), "No data found"
        assert 1 < jdoc['uptime']
        assert isinstance(jdoc['uptime'], int)
        assert 0 == jdoc['dataRead']
        assert 0 == jdoc['dataSent']
        assert 0 == jdoc['dataPurged']
        assert 'FogLAMP' == jdoc['serviceName']
        assert 'green' == jdoc['health']
        assert jdoc['authenticationOptional'] is True
        assert jdoc['safeMode'] is False

    def test_available_plugin_packages(self, foglamp_url):
        conn = http.client.HTTPConnection(foglamp_url)
        conn.request("GET", '/foglamp/plugins/available')
        r = conn.getresponse()
        assert 200 == r.status
        r = r.read().decode()
        jdoc = json.loads(r)
        assert len(jdoc), "No data found"
        global available_pkg
        plugins = available_pkg = jdoc['plugins']
        assert len(plugins), "No plugin found"
        assert 'link' in jdoc
        assert 'foglamp-filter-python35' in plugins
        assert 'foglamp-north-http-north' in plugins
        assert 'foglamp-north-kafka' in plugins
        assert 'foglamp-notify-python35' in plugins
        assert 'foglamp-rule-outofbound' in plugins
        assert 'foglamp-south-modbus' in plugins
        assert 'foglamp-south-playback' in plugins

    def test_available_service_packages(self, foglamp_url):
        conn = http.client.HTTPConnection(foglamp_url)
        conn.request("GET", '/foglamp/service/available')
        r = conn.getresponse()
        assert 200 == r.status
        r = r.read().decode()
        jdoc = json.loads(r)
        assert len(jdoc), "No data found"
        assert 1 == len(jdoc['services'])
        assert 'foglamp-service-notification' == jdoc['services'][0]
        assert 'link' in jdoc

    def test_install_service_package(self, foglamp_url):
        conn = http.client.HTTPConnection(foglamp_url)
        data = {"format": "repository", "name": "foglamp-service-notification"}
        conn.request("POST", '/foglamp/service?action=install', json.dumps(data))
        r = conn.getresponse()
        assert 200 == r.status
        r = r.read().decode()
        jdoc = json.loads(r)
        assert {'message': '{} is successfully installed'.format(data['name'])} == jdoc

        # verify service installed
        conn.request("GET", '/foglamp/service/installed')
        r = conn.getresponse()
        assert 200 == r.status
        r = r.read().decode()
        jdoc = json.loads(r)
        assert len(jdoc), "No data found"
        assert 3 == len(jdoc['services'])
        assert 'notification' in jdoc['services']

    def test_install_plugin_package(self, foglamp_url, package_build_source_list, package_build_list):
        # FIXME: FOGL-3276 (Remove below once we have dedicated Rpi with sensehat device attached; otherwise its discovery fails)
        if 'foglamp-south-sensehat' in available_pkg:
            available_pkg.remove('foglamp-south-sensehat')

        # When "package_build_source_list" is true then it will install all available packages
        # Otherwise install from list as we defined in JSON file
        if package_build_source_list.lower() == 'true':
            for pkg_name in available_pkg:
                self._verify_and_install_package(foglamp_url, pkg_name)
        else:
            json_data = load_data_from_json()
            # If 'all' in 'package_build_list' then it will iterate each key in JSON file
            if 'all' in package_build_list:
                package_build_list = ",".join(json_data.keys())
            my_list = package_build_list.split(",")

            for pkg_list_cat in my_list:
                for k, pkg_list_name in json_data[pkg_list_cat][0].items():
                    for pkg_name in pkg_list_name:
                        full_pkg_name = 'foglamp-{}-{}'.format(k, pkg_name)
                        if full_pkg_name in available_pkg:
                            self._verify_and_install_package(foglamp_url, full_pkg_name)
                        else:
                            # TODO: Add them into some csv (report)
                            print("{} not found in available package list".format(full_pkg_name))

    def _verify_and_install_package(self, foglamp_url, pkg_name):
        global counter
        conn = http.client.HTTPConnection(foglamp_url)
        data = {"format": "repository", "name": pkg_name}
        conn.request("POST", '/foglamp/plugins', json.dumps(data))
        r = conn.getresponse()
        assert 200 == r.status
        r = r.read().decode()
        jdoc = json.loads(r)
        assert '{} is successfully installed'.format(pkg_name) == jdoc['message']
        assert 'link' in jdoc
        # Special case: On flirax8 package installation this installs modbus package too as it depends upon
        # available package list always in alphabetically sorted order
        if pkg_name == 'foglamp-south-flirax8':
            available_pkg.remove('foglamp-south-modbus')
            counter += 1
        counter += 1
        conn.request("GET", '/foglamp/plugins/installed')
        r = conn.getresponse()
        assert 200 == r.status
        r = r.read().decode()
        jdoc = json.loads(r)
        assert len(jdoc), "No data found"
        assert counter == len(jdoc['plugins'])
<commit_msg>assertion fixes for install service<commit_after># -*- coding: utf-8 -*-

# FOGLAMP_BEGIN
# See: http://foglamp.readthedocs.io/
# FOGLAMP_END

""" System tests that obtain the set of available packages for the current platform
    It then installs each of those plugin packages via REST API endpoints
"""
import os
import subprocess
import http.client
import json
import pytest
import py

__author__ = "Ashish Jabble"
__copyright__ = "Copyright (c) 2019 Dianomic Systems Inc."
__license__ = "Apache 2.0"
__version__ = "${VERSION}"

available_pkg = []
# plugins installed by default 4
counter = 4

@pytest.fixture
def reset_packages():
    try:
        subprocess.run(["$FOGLAMP_ROOT/tests/system/lab/remove"], shell=True, check=True)
    except subprocess.CalledProcessError:
        assert False, "reset package script failed"

@pytest.fixture
def setup_package(package_build_version):
    try:
        subprocess.run(["$FOGLAMP_ROOT/tests/system/python/scripts/package/setup {}".format(package_build_version)],
                       shell=True, check=True)
    except subprocess.CalledProcessError:
        assert False, "setup package script failed"

def load_data_from_json():
    _dir = os.path.dirname(os.path.realpath(__file__))
    file_path = py.path.local(_dir).join('/').join('data/package_list.json')
    with open(str(file_path)) as data_file:
        json_data = json.load(data_file)
    return json_data

class TestPackages:

    def test_reset_and_setup(self, reset_packages, setup_package):
        # TODO: Remove this workaround
        # Use better setup & teardown methods
        pass

    def test_ping(self, foglamp_url):
        conn = http.client.HTTPConnection(foglamp_url)
        conn.request("GET", '/foglamp/ping')
        r = conn.getresponse()
        assert 200 == r.status
        r = r.read().decode()
        jdoc = json.loads(r)
        assert len(jdoc), "No data found"
        assert 1 < jdoc['uptime']
        assert isinstance(jdoc['uptime'], int)
        assert 0 == jdoc['dataRead']
        assert 0 == jdoc['dataSent']
        assert 0 == jdoc['dataPurged']
        assert 'FogLAMP' == jdoc['serviceName']
        assert 'green' == jdoc['health']
        assert jdoc['authenticationOptional'] is True
        assert jdoc['safeMode'] is False

    def test_available_plugin_packages(self, foglamp_url):
        conn = http.client.HTTPConnection(foglamp_url)
        conn.request("GET", '/foglamp/plugins/available')
        r = conn.getresponse()
        assert 200 == r.status
        r = r.read().decode()
        jdoc = json.loads(r)
        assert len(jdoc), "No data found"
        global available_pkg
        plugins = available_pkg = jdoc['plugins']
        assert len(plugins), "No plugin found"
        assert 'link' in jdoc
        assert 'foglamp-filter-python35' in plugins
        assert 'foglamp-north-http-north' in plugins
        assert 'foglamp-north-kafka' in plugins
        assert 'foglamp-notify-python35' in plugins
        assert 'foglamp-rule-outofbound' in plugins
        assert 'foglamp-south-modbus' in plugins
        assert 'foglamp-south-playback' in plugins

    def test_available_service_packages(self, foglamp_url):
        conn = http.client.HTTPConnection(foglamp_url)
        conn.request("GET", '/foglamp/service/available')
        r = conn.getresponse()
        assert 200 == r.status
        r = r.read().decode()
        jdoc = json.loads(r)
        assert len(jdoc), "No data found"
        assert 1 == len(jdoc['services'])
        assert 'foglamp-service-notification' == jdoc['services'][0]
        assert 'link' in jdoc

    def test_install_service_package(self, foglamp_url):
        conn = http.client.HTTPConnection(foglamp_url)
        data = {"format": "repository", "name": "foglamp-service-notification"}
        conn.request("POST", '/foglamp/service?action=install', json.dumps(data))
        r = conn.getresponse()
        assert 200 == r.status
        r = r.read().decode()
        jdoc = json.loads(r)
        assert 'message' in jdoc
        assert 'link' in jdoc
        assert '{} is successfully installed'.format(data['name']) == jdoc['message']

        # verify service installed
        conn.request("GET", '/foglamp/service/installed')
        r = conn.getresponse()
        assert 200 == r.status
        r = r.read().decode()
        jdoc = json.loads(r)
        assert len(jdoc), "No data found"
        assert 3 == len(jdoc['services'])
        assert 'notification' in jdoc['services']

    def test_install_plugin_package(self, foglamp_url, package_build_source_list, package_build_list):
        # FIXME: FOGL-3276 (Remove below once we have dedicated Rpi with sensehat device attached; otherwise its discovery fails)
        if 'foglamp-south-sensehat' in available_pkg:
            available_pkg.remove('foglamp-south-sensehat')

        # When "package_build_source_list" is true then it will install all available packages
        # Otherwise install from list as we defined in JSON file
        if package_build_source_list.lower() == 'true':
            for pkg_name in available_pkg:
                self._verify_and_install_package(foglamp_url, pkg_name)
        else:
            json_data = load_data_from_json()
            # If 'all' in 'package_build_list' then it will iterate each key in JSON file
            if 'all' in package_build_list:
                package_build_list = ",".join(json_data.keys())
            my_list = package_build_list.split(",")

            for pkg_list_cat in my_list:
                for k, pkg_list_name in json_data[pkg_list_cat][0].items():
                    for pkg_name in pkg_list_name:
                        full_pkg_name = 'foglamp-{}-{}'.format(k, pkg_name)
                        if full_pkg_name in available_pkg:
                            self._verify_and_install_package(foglamp_url, full_pkg_name)
                        else:
                            # TODO: Add them into some csv (report)
                            print("{} not found in available package list".format(full_pkg_name))

    def _verify_and_install_package(self, foglamp_url, pkg_name):
        global counter
        conn = http.client.HTTPConnection(foglamp_url)
        data = {"format": "repository", "name": pkg_name}
        conn.request("POST", '/foglamp/plugins', json.dumps(data))
        r = conn.getresponse()
        assert 200 == r.status
        r = r.read().decode()
        jdoc = json.loads(r)
        assert '{} is successfully installed'.format(pkg_name) == jdoc['message']
        assert 'link' in jdoc
        # Special case: On flirax8 package installation this installs modbus package too as it depends upon
        # available package list always in alphabetically sorted order
        if pkg_name == 'foglamp-south-flirax8':
            available_pkg.remove('foglamp-south-modbus')
            counter += 1
        counter += 1
        conn.request("GET", '/foglamp/plugins/installed')
        r = conn.getresponse()
        assert 200 == r.status
        r = r.read().decode()
        jdoc = json.loads(r)
        assert len(jdoc), "No data found"
        assert counter == len(jdoc['plugins'])
<|endoftext|>


<commit_before>
'''
这个文件的主要作用是记录所有ILP的方法， 如果记不下就另来一个文件
'''

from .summarizer import abstract_summarizer
from ..query_expansion.entity_expansioner import RankRelateFilterExpansioner as RFE
from ..query_expansion.entity_finder import NgramEntityFinder as ngram
from ..read_conf import config
from ..util import NLP

from pulp import *
import sys

nlp = NLP()

import math

conf = config('/home/lavi/project/insummer/conf/question.conf')

def sent_len(sent):
    words = nlp.word_tokenize(sent)
    count = 0
    for word in words:
        if len(word) >= 2:
            count += 1

    return count

#这个方法叫tranditional_ilp好了，就是经典没有任何改动的ILP方法
#ep         , 实体扩展类
#q          , 问题
#word_limit , 字数限制，一般是250
#answer_total_entities                 , 答案中全部的实体
#answer_entities_list                  , 答案中句子和实体组成的list[(e1,w1),(e2,w2)...]
#hit_entities                          , 命中实体的权重
#hit_entities_freq                     , 命中实体的频率
#unhit_entities                        , 未命中实体，权重全部为-1

#candidate_sentence_entities_dict      , 候选答案句子和实体组成的dict[(e1:w1),(e2:w2)....]
#sent_index                            , 候选答案句子的索引
#sent_inverse_index                    , 候选答案句子的逆索引
#sent_length                           , 候选答案句子的长度
#entity_index                          , 候选实体的索引
#entity_inverse_index                  , 候选实体的逆索引

#OCC                                   , 构建出现矩阵OCC[i][j] 为实体I在句子J中出现了没
class traditional_ilp(abstract_summarizer):
    def __init__(self,q,word_limit=250):
        self.ep = RFE(q,ngram,1,1,display=False,n=40,length=500)
        self.question = q
        self.word_limit = word_limit
        print("字数限制",self.word_limit)

    def extract(self):
        #执行生成摘要前的预备工作
        print("进行生成摘要前的准备工作")
        self.init_step()

        print("命中实体数目",len(self.hit_entities_freq),len(self.hit_entities))
        print("答案实体总数",len(self.answer_total_entities))

        print("进行整数规划前的准备工作")
        self.ilp_prepare()

        print("过滤前句子大小",len(self.answer_entities_list))
        print("过滤后句子大小",len(self.candidate_sentence_entities_dict))
        print("句子索引大小",len(self.sent_inverse_index),len(self.sent_index))
        print("实体索引大小",len(self.entity_inverse_index),len(self.entity_index))

        result = self.ilp()

        return result

    #执行生成摘要前的输入工作
    #主要是得到命中实体和没有命中的实体
    #还有命中实体的频率，（未命中的就没必要了）
    #还有对实体的打分进行转换
    def init_step(self):
        ##先进行实体扩展，得到的实体是具有权重的list，[(e1,w1),(e2,w2)]
        expand_entities = self.ep.run()
        ##需要转化成字典形式，方便计算
        dict_expand_entities = dict(expand_entities)

        ##得到答案中的所有实体,加个set
        answer_total_entities = set(self.ep.get_sentence_total_entity())
        self.answer_total_entities = answer_total_entities

        #=====> 在这里定义一个子函数，方便进行分数转换
        def transform_score(s):
            return math.log(s+1)

        #记录实体，这里实体总共有两部分，一部分命中的，一部分没有，分别叫hit和unhit好了
        #这里容易产生歧义的是，unhit不是指扩展实体中没有命中的部分，指的是答案实体中没有命中的部分
        self.hit_entities      = {}
        self.unhit_entities    = {}

        #记录命中实体的频率
        self.hit_entities_freq = {}

        #====> 找到命中实体
        #对于所有这些扩展的实体
        for mentity in dict_expand_entities:
            #如果实体在 答案所有的实体里
            if mentity in answer_total_entities:
                mscore = dict_expand_entities[mentity]

                #将其加入到hit_entities中
                self.hit_entities[mentity] = mscore

            #如果不在，直接撇了
            #pass

        #对所有的命中实体进行分数的转换
        for mentity in self.hit_entities:
            mscore = self.hit_entities[mentity]
            self.hit_entities[mentity] = transform_score(mscore)

        #====> 找到没有命中的实体

        #返回所有句子和实体的列表
        answer_entities_list = self.ep.get_sentence_entity()
        self.answer_entities_list = answer_entities_list

        #没有命中的实体主要是得遍历句子中的所有实体
        for manswer_sent,mentities in answer_entities_list:
            #遍历mentities,找到没有命中的实体
            for mentity in mentities:
                if mentity not in self.hit_entities:
                    self.unhit_entities[mentity] = -1.0

                #对于是hit_entities的实体
                else:
                    self.hit_entities_freq.setdefault(mentity,0)
                    self.hit_entities_freq[mentity] += 1

    #整数规划的输入阶段
    #在这个阶段准备整数规划所需要的数据
    #主要包括: 1得到候选句子集合（过滤掉无关的句子）；2得到扩展实体的子集；3对实体和候选句子进行标号；4.构建句子和实体的OCC矩阵；
    def ilp_prepare(self):

        self.candidate_sentence_entities_dict = {}
        #给标号初始化
        self.sent_index           = {}
        self.sent_inverse_index   = {}
        self.sent_length          = {}
        self.entity_index         = {}
        self.entity_inverse_index = {}

        #两个索引现存的标号
        entity_index_number = 0
        sent_index_number = 0

        #====> step1：得到候选句子集合 , 句子标号的工作可能现在直接就做了

        hit_entities_set = set(self.hit_entities.keys())

        count = 0
        #对于答案的每个句子来说
        for manswer_sent,mentities in self.answer_entities_list:
            #先做个集合出来
            mentities_set = set(mentities)
            a1 = mentities_set

            #对于句子，进行strip，去两端
            manswer_sent  = manswer_sent.strip()

            #所有实体和临时集合取交集
            intersec_num  = len(mentities_set.intersection(hit_entities_set))
            mentities_set = mentities_set.intersection(hit_entities_set)
            a2 = mentities_set

            #如果没有交集，那么直接扔了
            if intersec_num <= 1 :
                pass
            else:
                #先进行判断，句子在不在句子索引中
                #如果在，什么都不做
                if manswer_sent in self.sent_index:
                    pass
                else:
                    #添加答案句子索引
                    self.sent_index[manswer_sent] = sent_index_number
                    self.sent_inverse_index[sent_index_number] = manswer_sent
                    sent_index_number += 1

                    self.candidate_sentence_entities_dict[manswer_sent] = mentities_set

        #====> step2：得到扩展实体子集
        #扩展实体的子集就是 hit_entities , 所以这个集合求过了

        #====> step3：给实体标号
        for mentity in self.hit_entities:
            self.entity_index[mentity] = entity_index_number
            self.entity_inverse_index[entity_index_number] = mentity
            entity_index_number += 1

        #====> step4：构建OCC矩阵
        self.OCC = [[0 for j in range(len(self.sent_index))]  for i in range(len(self.entity_index))]
        print("OCC矩阵shape",len(self.OCC),len(self.OCC[0]))

        #对于每个 candidate_sentence_entities_dict 中的句子和实体， 下面填充OCC矩阵的工作
        for manswer_sent in self.candidate_sentence_entities_dict:
            #找到其实体集
            mentities_set = self.candidate_sentence_entities_dict[manswer_sent]

            #得到句子的索引
            msent_index = self.sent_index[manswer_sent]

            #对于每个实体来说
            for mentity in mentities_set:
                #找到这个实体的索引
                mentity_index = self.entity_index[mentity]

                #现在实体  (i=menetity_index )  ,  (j=msent_index)
                self.OCC[mentity_index][msent_index] = 1

        print("OCC矩阵构建完成")

    def ilp(self):
        print("ILP开始")
        #====> 定义问题
        prob = LpProblem("ILP for summarization problem",LpMaximize)

        #变量的字典
        x_var = []
        y_var = []

        for indx in self.entity_inverse_index:
            x_var.append("x%s"%(indx))

        for indx in self.sent_inverse_index:
            y_var.append("y%s"%(indx))

        print("建立variable,类别全部设为binary")
        #====> 建立variable,类别全部设为binary
        x_lpvariable = LpVariable.dicts("entity",x_var,cat=LpBinary)
        y_lpvariable = LpVariable.dicts("sent",  y_var,cat=LpBinary)

        #获取ILP变量的权重
        def variable_weight(var):
            #首字母和尾字母
            first,last = var[0],var[1:]
            if first == "y":
                return 0

            elif first == "x":
                #得到变量实体的名字
                variable_name = self.entity_inverse_index[int(last)]

                #得到相应的权重
                w = self.hit_entities[variable_name]

                return w

            else :
                print("获取权重信息有误")
                sys.exit(1)

        print("设立优化目标")
        obj1 = [x_lpvariable[i] * variable_weight(i) for i in x_var ]
        obj2 = [y_lpvariable[i] * variable_weight(i) for i in y_var ]
        tobj = obj1#.extend(obj2)
        prob += lpSum( tobj )

        #定义一个求句子长度的函数
        def variable_length(var):
            first,last = var[0],var[1:]
            if first == "y":
                #得到句子
                msent = self.sent_inverse_index[int(last)]
                #得到句子长度
                ml = sent_len(msent)
                return ml
            else:
                print("变量有问题")
                sys.exit(1)

        print("句子长度限制")
        #满足长度限制
        prob += lpSum([y_lpvariable[i] * variable_length(i) for i in y_var]) <= self.word_limit

        print("出现次数限制")
        #满足出现次数限制
        #对每个实体而言
        for i in range(len(self.entity_index)):
            nobj = []

            #对每个句子都有
            for j in range(len(self.sent_index)):
                nobj.append(self.OCC[i][j] * y_lpvariable["y%s"%(j)])

            nobj.append(-1 * x_lpvariable["x%s"%(i)])

            prob += lpSum( nobj ) >= 0

        print("Status:", LpStatus[prob.status])
        print("开始求解")
        prob.solve()

        sent_list = []
        for v in prob.variables():
            sp = v.name.split('_')
            if sp[0] == "sent" and v.varValue > 0 :
                indx = int(sp[1][1:])
                msent = self.sent_inverse_index[indx]
                sent_list.append(msent)
            else:
                pass

        sent_length = 0
        for msent in sent_list:
            print(msent,self.candidate_sentence_entities_dict[msent])
            sent_length += sent_len(msent)

        print("摘要长度",sent_length)

        print("Total Cost of Ingredients per can = ", value(prob.objective))

        print("ILP结束")

        print("写入文件")
        wp = conf["ilp_sum"]
        wp += self.question.get_author()
        wp = wp[:-1]

        f = open(wp,"w")
        for msent in sent_list:
            f.write(msent+" ")

        f.close()
        return wp
<commit_msg>rouge2弄到8.2了，效果比贪心好了<commit_after>
'''
这个文件的主要作用是记录所有ILP的方法， 如果记不下就另来一个文件
'''

from .summarizer import abstract_summarizer
from ..query_expansion.entity_expansioner import RankRelateFilterExpansioner as RFE
from ..query_expansion.entity_finder import NgramEntityFinder as ngram
from ..read_conf import config
from ..util import NLP

from pulp import *
import sys

nlp = NLP()

import math

conf = config('/home/lavi/project/insummer/conf/question.conf')

def sent_len(sent):
    words = nlp.word_tokenize(sent)
    count = 0
    for word in words:
        if len(word) >= 2:
            count += 1

    return count

#这个方法叫tranditional_ilp好了，就是经典没有任何改动的ILP方法
#ep         , 实体扩展类
#q          , 问题
#word_limit , 字数限制，一般是250
#answer_total_entities                 , 答案中全部的实体
#answer_entities_list                  , 答案中句子和实体组成的list[(e1,w1),(e2,w2)...]
#hit_entities                          , 命中实体的权重
#hit_entities_freq                     , 命中实体的频率
#unhit_entities                        , 未命中实体，权重全部为-1

#candidate_sentence_entities_dict      , 候选答案句子和实体组成的dict[(e1:w1),(e2:w2)....]
#sent_index                            , 候选答案句子的索引
#sent_inverse_index                    , 候选答案句子的逆索引
#sent_length                           , 候选答案句子的长度
#entity_index                          , 候选实体的索引
#entity_inverse_index                  , 候选实体的逆索引

#OCC                                   , 构建出现矩阵OCC[i][j] 为实体I在句子J中出现了没
class traditional_ilp(abstract_summarizer):
    def __init__(self,q,word_limit=250):
        self.ep = RFE(q,ngram,1,1,display=False,n=40,length=4000)
        self.question = q
        self.word_limit = word_limit
        print("文章题目",self.question.get_title())

    def extract(self):
        #执行生成摘要前的预备工作
        print("进行生成摘要前的准备工作")
        self.init_step()

        print("命中实体数目",len(self.hit_entities_freq),len(self.hit_entities))
        print("答案实体总数",len(self.answer_total_entities))

        print("进行整数规划前的准备工作")
        self.ilp_prepare()

        print("过滤前句子大小",len(self.answer_entities_list))
        print("过滤后句子大小",len(self.candidate_sentence_entities_dict))
        print("句子索引大小",len(self.sent_inverse_index),len(self.sent_index))
        print("实体索引大小",len(self.entity_inverse_index),len(self.entity_index))

        result = self.ilp()

        return result

    #执行生成摘要前的输入工作
    #主要是得到命中实体和没有命中的实体
    #还有命中实体的频率，（未命中的就没必要了）
    #还有对实体的打分进行转换
    def init_step(self):
        ##先进行实体扩展，得到的实体是具有权重的list，[(e1,w1),(e2,w2)]
        expand_entities = self.ep.run()
        ##需要转化成字典形式，方便计算
        dict_expand_entities = dict(expand_entities)

        ##得到答案中的所有实体,加个set
        answer_total_entities = set(self.ep.get_sentence_total_entity())
        self.answer_total_entities = answer_total_entities

        #=====> 在这里定义一个子函数，方便进行分数转换
        def transform_score(s):
            return math.log(s+1)

        #记录实体，这里实体总共有两部分，一部分命中的，一部分没有，分别叫hit和unhit好了
        #这里容易产生歧义的是，unhit不是指扩展实体中没有命中的部分，指的是答案实体中没有命中的部分
        self.hit_entities      = {}
        self.unhit_entities    = {}

        #记录命中实体的频率
        self.hit_entities_freq = {}

        #====> 找到命中实体
        #对于所有这些扩展的实体
        for mentity in dict_expand_entities:
            #如果实体在 答案所有的实体里
            if mentity in answer_total_entities:
                mscore = dict_expand_entities[mentity]

                #将其加入到hit_entities中
                self.hit_entities[mentity] = mscore

            #如果不在，直接撇了
            #pass

        #对所有的命中实体进行分数的转换
        for mentity in self.hit_entities:
            mscore = self.hit_entities[mentity]
            self.hit_entities[mentity] = transform_score(mscore)

        #====> 找到没有命中的实体

        #返回所有句子和实体的列表
        answer_entities_list = self.ep.get_sentence_entity()
        self.answer_entities_list = answer_entities_list

        #没有命中的实体主要是得遍历句子中的所有实体
        for manswer_sent,mentities in answer_entities_list:
            #遍历mentities,找到没有命中的实体
            for mentity in mentities:
                if mentity not in self.hit_entities:
                    self.unhit_entities[mentity] = -1.0

                #对于是hit_entities的实体
                else:
                    self.hit_entities_freq.setdefault(mentity,0)
                    self.hit_entities_freq[mentity] += 1

    #整数规划的输入阶段
    #在这个阶段准备整数规划所需要的数据
    #主要包括: 1得到候选句子集合（过滤掉无关的句子）；2得到扩展实体的子集；3对实体和候选句子进行标号；4.构建句子和实体的OCC矩阵；
    def ilp_prepare(self):

        self.candidate_sentence_entities_dict = {}
        #给标号初始化
        self.sent_index           = {}
        self.sent_inverse_index   = {}
        self.sent_length          = {}
        self.entity_index         = {}
        self.entity_inverse_index = {}

        #两个索引现存的标号
        entity_index_number = 0
        sent_index_number = 0

        #====> step1：得到候选句子集合 , 句子标号的工作可能现在直接就做了

        hit_entities_set = set(self.hit_entities.keys())

        count = 0
        #对于答案的每个句子来说
        for manswer_sent,mentities in self.answer_entities_list:
            #先做个集合出来
            mentities_set = set(mentities)
            a1 = mentities_set

            #对于句子，进行strip，去两端
            manswer_sent  = manswer_sent.strip()

            #所有实体和临时集合取交集
            intersec_num  = len(mentities_set.intersection(hit_entities_set))
            mentities_set = mentities_set.intersection(hit_entities_set)
            a2 = mentities_set

            #如果没有交集，那么直接扔了
            if intersec_num <= 3 or sent_len(manswer_sent) < 6 :
                pass
            else:
                #先进行判断，句子在不在句子索引中
                #如果在，什么都不做
                if manswer_sent in self.sent_index:
                    pass
                else:
                    #添加答案句子索引
                    self.sent_index[manswer_sent] = sent_index_number
                    self.sent_inverse_index[sent_index_number] = manswer_sent
                    sent_index_number += 1

                    self.candidate_sentence_entities_dict[manswer_sent] = mentities_set

        #====> step2：得到扩展实体子集
        #扩展实体的子集就是 hit_entities , 所以这个集合求过了

        #====> step3：给实体标号
        for mentity in self.hit_entities:
            self.entity_index[mentity] = entity_index_number
            self.entity_inverse_index[entity_index_number] = mentity
            entity_index_number += 1

        #====> step4：构建OCC矩阵
        self.OCC = [[0 for j in range(len(self.sent_index))]  for i in range(len(self.entity_index))]
        print("OCC矩阵shape",len(self.OCC),len(self.OCC[0]))

        #对于每个 candidate_sentence_entities_dict 中的句子和实体， 下面填充OCC矩阵的工作
        for manswer_sent in self.candidate_sentence_entities_dict:
            #找到其实体集
            mentities_set = self.candidate_sentence_entities_dict[manswer_sent]

            #得到句子的索引
            msent_index = self.sent_index[manswer_sent]

            #对于每个实体来说
            for mentity in mentities_set:
                #找到这个实体的索引
                mentity_index = self.entity_index[mentity]

                #现在实体  (i=menetity_index )  ,  (j=msent_index)
                self.OCC[mentity_index][msent_index] = 1

        print("OCC矩阵构建完成")

    def ilp(self):
        print("ILP开始")
        #====> 定义问题
        prob = LpProblem("ILP for summarization problem",LpMaximize)

        #变量的字典
        x_var = []
        y_var = []

        for indx in self.entity_inverse_index:
            x_var.append("x%s"%(indx))

        for indx in self.sent_inverse_index:
            y_var.append("y%s"%(indx))

        print("建立variable,类别全部设为binary")
        #====> 建立variable,类别全部设为binary
        x_lpvariable = LpVariable.dicts("entity",x_var,cat=LpInteger,lowBound=0,upBound=10)
        y_lpvariable = LpVariable.dicts("sent",  y_var,cat=LpInteger,lowBound=0,upBound=1)

        #获取ILP变量的权重
        def variable_weight(var):
            #首字母和尾字母
            first,last = var[0],var[1:]
            if first == "y":
                #得到句子名字
                variable_name = self.sent_inverse_index[int(last)]

                #得到句子长度
                sl = sent_len(variable_name)

                #得到句子实体数目
                #el = len(self.candidate_sentence_entities_dict[variable_name])

                return (sl) * 4
                #return 0

            elif first == "x":
                #得到变量实体的名字
                variable_name = self.entity_inverse_index[int(last)]

                #得到相应的权重
                w = self.hit_entities[variable_name]

                return w

            else :
                print("获取权重信息有误")
                sys.exit(1)

        print("设立优化目标")
        obj1 = [x_lpvariable[i] * variable_weight(i) for i in x_var ]
        obj2 = [y_lpvariable[i] * variable_weight(i) for i in y_var ]
        obj1.extend(obj2)
        tobj = obj1
        prob += lpSum( tobj )

        #定义一个求句子长度的函数
        def variable_length(var):
            first,last = var[0],var[1:]
            if first == "y":
                #得到句子
                msent = self.sent_inverse_index[int(last)]
                #得到句子长度
                ml = sent_len(msent)
                return ml
            else:
                print("变量有问题")
                sys.exit(1)

        print("句子长度限制")
        #满足长度限制
        prob += lpSum([y_lpvariable[i] * variable_length(i) for i in y_var]) <= self.word_limit

        print("出现次数限制")
        #满足出现次数限制
        #对每个实体而言
        for i in range(len(self.entity_index)):
            nobj = []

            #对每个句子都有
            for j in range(len(self.sent_index)):
                nobj.append(self.OCC[i][j] * y_lpvariable["y%s"%(j)])

            nobj.append(-1 * x_lpvariable["x%s"%(i)])

            prob += lpSum( nobj ) == 0

        print("开始求解")
        prob.solve()
        print("Status:", LpStatus[prob.status])

        sent_list = []
        for v in prob.variables():
            sp = v.name.split('_')
            if sp[0] == "sent" and v.varValue > 0 :
                indx = int(sp[1][1:])
                msent = self.sent_inverse_index[indx]
                sent_list.append(msent)
            else:
                pass

        sent_length = 0
        for msent in sent_list:
            print(msent,self.candidate_sentence_entities_dict[msent])
            sent_length += sent_len(msent)

        print("摘要长度",sent_length)

        print("Total Cost of Ingredients per can = ", value(prob.objective))

        print("ILP结束")

        print("写入文件")
        wp = conf["ilp_sum"]
        wp += self.question.get_author()
        wp = wp[:-1]

        f = open(wp,"w")
        for msent in sent_list:
            f.write(msent+" ")

        f.close()
        return wp
<|endoftext|>


<commit_before>from datetime import datetime
from optparse import make_option
from django.core.mail import mail_admins
from gevent import monkey;monkey.patch_all()
import gevent
from pillowtop.listener import AliasedElasticPillow
from pillowtop.management.pillowstate import get_pillow_states
from pillowtop.run_pillowtop import import_pillows
from django.core.management.base import NoArgsCommand, BaseCommand
from django.core.management import call_command
from django.conf import settings

def get_reindex_command(pillow_class_name):
    pillow_command_map = {
        'DomainPillow': 'ptop_fast_reindex_domains',
        'CasePillow': 'ptop_fast_reindex_cases',
        'FullCasePillow': 'ptop_fast_reindex_fullcases',
        'XFormPillow': 'ptop_fast_reindex_xforms',
        'FullXFormPillow': 'ptop_fast_reindex_fullxforms',
        'UserPillow': 'ptop_fast_reindex_users',
        'AppPillow': 'ptop_fast_reindex_apps',
        'ReportXFormPillow': 'ptop_fast_reindex_reportxforms',
        'ReportCasePillow': 'ptop_fast_reindex_reportcases',
    }
    reindex_command = pillow_command_map.get(pillow_class_name, None)
    return reindex_command

def do_reindex(pillow_class_name):
    print "Starting pillow preindex %s" % pillow_class_name
    reindex_command = get_reindex_command(pillow_class_name)
    if reindex_command:
        call_command(reindex_command, **{'noinput': True, 'bulk': True})
        print "Pillow preindex finished %s" % pillow_class_name

class Command(BaseCommand):
    help = "Preindex ES pillows. Only run reindexer if the index doesn't exist."
    option_list = NoArgsCommand.option_list + (
        make_option('--replace',
                    action='store_true',
                    dest='replace',
                    default=False,
                    help='Reindex existing indices even if they are already there.'),
    )

    def handle(self, *args, **options):
        runs = []
        pillow_classes = import_pillows(instantiate=False)
        aliased_classes = filter(lambda x: issubclass(x, AliasedElasticPillow), pillow_classes)
        aliasable_pillows = [p(create_index=False) for p in aliased_classes]

        reindex_all = options['replace']

        mapped_masters, unmapped_masters, stale_indices = get_pillow_states(aliasable_pillows)

        # mapped masters: ES indices as known in the current running code state that
        # correctly have the alias applied to them

        # unmapped masters: ES indices as known in the current running code state that
        # do not have the alias applied to them

        # stale indices: ES indices running on ES that are not part of the current source control.

        print "Master indices missing aliases:"
        unmapped_indices = [x[0] for x in unmapped_masters]
        print unmapped_indices

        if reindex_all:
            print "Reindexing ALL master pillows that are not aliased"
            preindexable_pillows = aliasable_pillows
        else:
            print "Reindexing master pillows that do not exist yet (ones with aliases skipped)"
            preindexable_pillows = filter(lambda x: not x.index_exists(), aliasable_pillows)

        reindex_pillows = filter(lambda x: x.es_index in unmapped_indices, preindexable_pillows)

        print "Reindexing:\n\t%s" % '\n\t'.join(x.es_index for x in reindex_pillows)

        if len(reindex_pillows) > 0:
            preindex_message = """
        Heads up!

        %sis going to start preindexing the following pillows:
        %s

        This may take a while, so don't deploy until all these have reported finishing.
            """ % (settings.EMAIL_SUBJECT_PREFIX, ', '.join([x.__class__.__name__ for x in reindex_pillows]))

            mail_admins("Pillow preindexing starting", preindex_message)

        start = datetime.utcnow()
        for pillow in reindex_pillows:
            #loop through pillows once before running greenlets to fail hard on misconfigured pillows
            pillow_class_name = pillow.__class__.__name__
            reindex_command = get_reindex_command(pillow_class_name)
            if not reindex_command:
                raise Exception("Error, pillow [%s] is not configured with its own management command reindex command - it needs one" % pillow_class_name)

        for pillow in reindex_pillows:
            print pillow.__class__.__name__
            g = gevent.spawn(do_reindex, pillow.__class__.__name__)
            runs.append(g)
        gevent.joinall(runs)
        if len(reindex_pillows) > 0:
            mail_admins("Pillow preindexing completed", "Reindexing %s took %s seconds" % (
                ', '.join([x.__class__.__name__ for x in reindex_pillows]),
                (datetime.utcnow() - start).seconds
            ))
        print "All pillowtop reindexing jobs completed"

<commit_msg>use pillowtop.get_all_pillows<commit_after>from datetime import datetime
from optparse import make_option
from django.core.mail import mail_admins
from gevent import monkey;monkey.patch_all()
import gevent
from pillowtop.listener import AliasedElasticPillow
from pillowtop.management.pillowstate import get_pillow_states
from pillowtop import get_all_pillows
from django.core.management.base import NoArgsCommand, BaseCommand
from django.core.management import call_command
from django.conf import settings

def get_reindex_command(pillow_class_name):
    pillow_command_map = {
        'DomainPillow': 'ptop_fast_reindex_domains',
        'CasePillow': 'ptop_fast_reindex_cases',
        'FullCasePillow': 'ptop_fast_reindex_fullcases',
        'XFormPillow': 'ptop_fast_reindex_xforms',
        'FullXFormPillow': 'ptop_fast_reindex_fullxforms',
        'UserPillow': 'ptop_fast_reindex_users',
        'AppPillow': 'ptop_fast_reindex_apps',
        'ReportXFormPillow': 'ptop_fast_reindex_reportxforms',
        'ReportCasePillow': 'ptop_fast_reindex_reportcases',
    }
    reindex_command = pillow_command_map.get(pillow_class_name, None)
    return reindex_command

def do_reindex(pillow_class_name):
    print "Starting pillow preindex %s" % pillow_class_name
    reindex_command = get_reindex_command(pillow_class_name)
    if reindex_command:
        call_command(reindex_command, **{'noinput': True, 'bulk': True})
        print "Pillow preindex finished %s" % pillow_class_name

class Command(BaseCommand):
    help = "Preindex ES pillows. Only run reindexer if the index doesn't exist."
    option_list = NoArgsCommand.option_list + (
        make_option('--replace',
                    action='store_true',
                    dest='replace',
                    default=False,
                    help='Reindex existing indices even if they are already there.'),
    )

    def handle(self, *args, **options):
        runs = []
        pillow_classes = get_all_pillow(instantiate=False)
        aliased_classes = filter(lambda x: issubclass(x, AliasedElasticPillow), pillow_classes)
        aliasable_pillows = [p(create_index=False) for p in aliased_classes]

        reindex_all = options['replace']

        mapped_masters, unmapped_masters, stale_indices = get_pillow_states(aliasable_pillows)

        # mapped masters: ES indices as known in the current running code state that
        # correctly have the alias applied to them

        # unmapped masters: ES indices as known in the current running code state that
        # do not have the alias applied to them

        # stale indices: ES indices running on ES that are not part of the current source control.

        print "Master indices missing aliases:"
        unmapped_indices = [x[0] for x in unmapped_masters]
        print unmapped_indices

        if reindex_all:
            print "Reindexing ALL master pillows that are not aliased"
            preindexable_pillows = aliasable_pillows
        else:
            print "Reindexing master pillows that do not exist yet (ones with aliases skipped)"
            preindexable_pillows = filter(lambda x: not x.index_exists(), aliasable_pillows)

        reindex_pillows = filter(lambda x: x.es_index in unmapped_indices, preindexable_pillows)

        print "Reindexing:\n\t%s" % '\n\t'.join(x.es_index for x in reindex_pillows)

        if len(reindex_pillows) > 0:
            preindex_message = """
        Heads up!

        %s is going to start preindexing the following pillows:
        %s

        This may take a while, so don't deploy until all these have reported finishing.
            """ % (settings.EMAIL_SUBJECT_PREFIX, ', '.join([x.__class__.__name__ for x in reindex_pillows]))

            mail_admins("Pillow preindexing starting", preindex_message)

        start = datetime.utcnow()
        for pillow in reindex_pillows:
            #loop through pillows once before running greenlets to fail hard on misconfigured pillows
            pillow_class_name = pillow.__class__.__name__
            reindex_command = get_reindex_command(pillow_class_name)
            if not reindex_command:
                raise Exception("Error, pillow [%s] is not configured with its own management command reindex command - it needs one" % pillow_class_name)

        for pillow in reindex_pillows:
            print pillow.__class__.__name__
            g = gevent.spawn(do_reindex, pillow.__class__.__name__)
            runs.append(g)
        gevent.joinall(runs)
        if len(reindex_pillows) > 0:
            mail_admins("Pillow preindexing completed", "Reindexing %s took %s seconds" % (
                ', '.join([x.__class__.__name__ for x in reindex_pillows]),
                (datetime.utcnow() - start).seconds
            ))
        print "All pillowtop reindexing jobs completed"
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import print_function

import logging
from xml.etree import ElementTree
import requests

from .base import Provider as BaseProvider

logger = logging.getLogger(__name__)

APIENDPOINTS = {
        'zonomi': 'https://zonomi.com/app',
        'rimuhosting' : 'https://rimuhosting.com'
}

# Lexicon Zonomi and Rimuhosting Provider
#
# Author: Juan Rossi, 2017
#
# Zonomi API Docs: https://zonomi.com/app/dns/dyndns.jsp
# Rimuhosting API Docs: https://rimuhosting.com/dns/dyndns.jsp
#
# Implementation notes:
# * Lots of tricks taken from the PowerDNS API
# * The Zonomi API does not assign a unique identifier to each record in the way
#   that Lexicon expects. We work around this by creating an ID based on the record
#   name, type and content, which when taken together are always unique
# * The  API has no notion of 'create a single record' or 'delete a single
#   record'. All operations are either 'replace the RRSet with this new set of records'
#   or 'delete all records for this name and type. Similarly, there is no notion of
#   'change the content of this record', because records are identified by their name,
#   type and content.

def ProviderParser(subparser):
    #subparser.add_argument("--priority", help="priority")
    subparser.add_argument("--auth-token", help="specify token used authenticate")
    subparser.add_argument("--endpoint", help="Use Zonomi or Rimuhosting API", choices=[
        'zonomi', 'rimuhosting' ])

class Provider(BaseProvider):

    def __init__(self, options, engine_overrides=None):
        super(Provider, self).__init__(options, engine_overrides)
        self.domain_id = None

        if not self.options.get('auth_token'):
            raise Exception('Error, application key is not defined')

        self.api_key = self.options.get('auth_token')

        if not self.options.get('endpoint'):
            self.api_endpoint = self.engine_overrides.get('api_endpoint', APIENDPOINTS.get('zonomi'))
        else:
            self.api_endpoint = self.engine_overrides.get('api_endpoint', APIENDPOINTS.get(self.options.get('endpoint')))

    def authenticate(self):
        payload = self._get('/dns/dyndns.jsp', {
            'action' : 'QUERY',
            'name': "**." + self.options['domain']
        })

        if payload.find('is_ok').text != 'OK:':
            raise Exception('Error with api {0}'.format(payload.find('is_ok').text))

        self.domain_id = self.options['domain']

    def _make_identifier(self, type, name, content):
        return "{}/{}={}".format(type, self._full_name(name), content)

    def _parse_identifier(self, identifier):
        parts = identifier.split('/')
        type = parts[0]
        parts = parts[1].split('=')
        name = parts[0]
        content = "=".join(parts[1:])
        return type, name, content

    def create_record(self, type, name, content):
        request = {
                'action': 'SET',
                'type': type,
                'name': self.options['domain'],
                'value': content
        }

        if name is not None:
            request['name'] = self._full_name(name)

        if self.options.get('ttl'):
            request['ttl'] = self.options.get('ttl')

        if self.options.get('priority'):
            request['prio'] = self.options.get('priority')

        payload = self._get('/dns/dyndns.jsp', request)

        if payload.find('is_ok').text != 'OK:':
            raise Exception('An error occurred: {0}'.format(payload.find('is_ok').text))

        logger.debug('create_record: %s', True)
        return True

    def list_records(self, type=None, name=None, content=None):
        records = []

        request = {
            'action' : 'QUERY',
            'name': "**." + self.options['domain']
        }

        if type is not None:
            request['type'] = type
        if name is not None:
            request['name'] = self._full_name(name)
        if content is not None:
            request['value'] = content

        payload = self._get('/dns/dyndns.jsp', request)
        for rxml in payload.iter('record'):
            processed_record = {
                'type': rxml.attrib['type'],
                'name': rxml.attrib['name'],
                'content': rxml.attrib['content'],
                'id': self._make_identifier(rxml.attrib['type'],rxml.attrib['name'],rxml.attrib['content'])
            }
            records.append(processed_record)
        logger.debug('list_records: %s', records)
        return records

    def delete_record(self, identifier=None, type=None, name=None, content=None):
        if identifier is not None:
            type, name, content = self._parse_identifier(identifier)
        request = {
            'action' : 'DELETE',
            'type' : type,
            'name' : self._full_name(name),
            'value': content
        }

        payload = self._get('/dns/dyndns.jsp', request)

        if payload.find('is_ok').text != 'OK:':
            raise Exception('An error occurred: {0}'.format(payload.find('is_ok').text))

        logger.debug('delete_record: %s', True)
        return True

    def update_record(self, identifier, type=None, name=None, content=None):
        self.delete_record(identifier)
        ttype, tname, tcontent = self._parse_identifier(identifier)

        request = {
                'action': 'SET',
                'type': ttype,
                'name': self._full_name(tname),
                'value': tcontent
        }

        if name:
            request['name'] = self._full_name(name)
        if content:
            request['value'] = content
        if self.options.get('ttl'):
            request['ttl'] = self.options.get('ttl')
        if self.options.get('priority'):
            request['prio'] = self.options.get('priority')

        payload = self._get('/dns/dyndns.jsp', request)

        if payload.find('is_ok').text != 'OK:':
            raise Exception('An error occurred: {0}'.format(payload.find('is_ok').text))

        logger.debug('update_record: %s', True)
        return True

    def _request(self, action='GET', url='/', data=None, query_params=None):
        if data is None:
            data = {}
        if query_params is None:
            query_params = {}
        else:
            query_params['api_key'] = self.options.get('auth_token')

        r = requests.request(action, self.api_endpoint + url, params=query_params)
        tree = ElementTree.ElementTree(ElementTree.fromstring(r.content))
        root = tree.getroot()
        if root.tag == 'error':
            raise Exception('An error occurred: {0}'.format(root.text))
        else:
            r.raise_for_status()
        return root
<commit_msg>correct handling of name and content on delete_record<commit_after>from __future__ import absolute_import
from __future__ import print_function

import logging
from xml.etree import ElementTree
import requests

from .base import Provider as BaseProvider

logger = logging.getLogger(__name__)

APIENDPOINTS = {
        'zonomi': 'https://zonomi.com/app',
        'rimuhosting' : 'https://rimuhosting.com'
}

# Lexicon Zonomi and Rimuhosting Provider
#
# Author: Juan Rossi, 2017
#
# Zonomi API Docs: https://zonomi.com/app/dns/dyndns.jsp
# Rimuhosting API Docs: https://rimuhosting.com/dns/dyndns.jsp
#
# Implementation notes:
# * Lots of tricks taken from the PowerDNS API
# * The Zonomi API does not assign a unique identifier to each record in the way
#   that Lexicon expects. We work around this by creating an ID based on the record
#   name, type and content, which when taken together are always unique
# * The  API has no notion of 'create a single record' or 'delete a single
#   record'. All operations are either 'replace the RRSet with this new set of records'
#   or 'delete all records for this name and type. Similarly, there is no notion of
#   'change the content of this record', because records are identified by their name,
#   type and content.

def ProviderParser(subparser):
    #subparser.add_argument("--priority", help="priority")
    subparser.add_argument("--auth-token", help="specify token used authenticate")
    subparser.add_argument("--endpoint", help="Use Zonomi or Rimuhosting API", choices=[
        'zonomi', 'rimuhosting' ])

class Provider(BaseProvider):

    def __init__(self, options, engine_overrides=None):
        super(Provider, self).__init__(options, engine_overrides)
        self.domain_id = None

        if not self.options.get('auth_token'):
            raise Exception('Error, application key is not defined')

        self.api_key = self.options.get('auth_token')

        if not self.options.get('endpoint'):
            self.api_endpoint = self.engine_overrides.get('api_endpoint', APIENDPOINTS.get('zonomi'))
        else:
            self.api_endpoint = self.engine_overrides.get('api_endpoint', APIENDPOINTS.get(self.options.get('endpoint')))

    def authenticate(self):
        payload = self._get('/dns/dyndns.jsp', {
            'action' : 'QUERY',
            'name': "**." + self.options['domain']
        })

        if payload.find('is_ok').text != 'OK:':
            raise Exception('Error with api {0}'.format(payload.find('is_ok').text))

        self.domain_id = self.options['domain']

    def _make_identifier(self, type, name, content):
        return "{}/{}={}".format(type, self._full_name(name), content)

    def _parse_identifier(self, identifier):
        parts = identifier.split('/')
        type = parts[0]
        parts = parts[1].split('=')
        name = parts[0]
        content = "=".join(parts[1:])
        return type, name, content

    def create_record(self, type, name, content):
        request = {
                'action': 'SET',
                'type': type,
                'name': self.options['domain'],
                'value': content
        }

        if name is not None:
            request['name'] = self._full_name(name)

        if self.options.get('ttl'):
            request['ttl'] = self.options.get('ttl')

        if self.options.get('priority'):
            request['prio'] = self.options.get('priority')

        payload = self._get('/dns/dyndns.jsp', request)

        if payload.find('is_ok').text != 'OK:':
            raise Exception('An error occurred: {0}'.format(payload.find('is_ok').text))

        logger.debug('create_record: %s', True)
        return True

    def list_records(self, type=None, name=None, content=None):
        records = []

        request = {
            'action' : 'QUERY',
            'name': "**." + self.options['domain']
        }

        if type is not None:
            request['type'] = type
        if name is not None:
            request['name'] = self._full_name(name)
        if content is not None:
            request['value'] = content

        payload = self._get('/dns/dyndns.jsp', request)
        for rxml in payload.iter('record'):
            processed_record = {
                'type': rxml.attrib['type'],
                'name': rxml.attrib['name'],
                'content': rxml.attrib['content'],
                'id': self._make_identifier(rxml.attrib['type'],rxml.attrib['name'],rxml.attrib['content']),
                'ttl': rxml.attrib['ttl'].split()[0]
            }
            records.append(processed_record)
        logger.debug('list_records: %s', records)
        return records

    def delete_record(self, identifier=None, type=None, name=None, content=None):
        if identifier is not None:
            type, name, content = self._parse_identifier(identifier)

        request = {
            'action' : 'DELETE',
            'type' : type,
            'name': self.options['domain']
        }

        if name:
            request['name'] = self._full_name(name)
        if content:
            request['value'] = content

        payload = self._get('/dns/dyndns.jsp', request)

        if payload.find('is_ok').text != 'OK:':
            raise Exception('An error occurred: {0}'.format(payload.find('is_ok').text))

        logger.debug('delete_record: %s', True)
        return True

    def update_record(self, identifier, type=None, name=None, content=None):
        self.delete_record(identifier)
        ttype, tname, tcontent = self._parse_identifier(identifier)
        request = {
                'action': 'SET',
                'type': ttype,
                'name': self._full_name(tname),
                'value': tcontent
        }

        if name:
            request['name'] = self._full_name(name)
        if content:
            request['value'] = content
        if self.options.get('ttl'):
            request['ttl'] = self.options.get('ttl')
        if self.options.get('priority'):
            request['prio'] = self.options.get('priority')

        payload = self._get('/dns/dyndns.jsp', request)

        if payload.find('is_ok').text != 'OK:':
            raise Exception('An error occurred: {0}'.format(payload.find('is_ok').text))

        logger.debug('update_record: %s', True)
        return True

    def _request(self, action='GET', url='/', data=None, query_params=None):
        if data is None:
            data = {}
        if query_params is None:
            query_params = {}
        else:
            query_params['api_key'] = self.options.get('auth_token')

        r = requests.request(action, self.api_endpoint + url, params=query_params)
        tree = ElementTree.ElementTree(ElementTree.fromstring(r.content))
        root = tree.getroot()
        if root.tag == 'error':
            raise Exception('An error occurred: {0}'.format(root.text))
        else:
            r.raise_for_status()
        return root
<|endoftext|>


<commit_before># Copyright (C) 2012 - TODAY  Renato Lima - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

import logging

from odoo import models, fields, api, _
from odoo.exceptions import ValidationError

_logger = logging.getLogger(__name__)

try:
    from erpbrasil.base.fiscal import cnpj_cpf, ie
    from erpbrasil.base import misc
except ImportError:
    _logger.error("Biblioteca erpbrasil.base não instalada")

class Lead(models.Model):
    """ CRM Lead Case """
    _inherit = "crm.lead"

    legal_name = fields.Char(
        string='Legal Name',
        size=128,
        help="nome utilizado em documentos fiscais")

    cnpj = fields.Char(
        string='CNPJ',
        size=18)

    inscr_est = fields.Char(
        string='Inscr. Estadual/RG',
        size=16)

    inscr_mun = fields.Char(
        string='Inscr. Municipal',
        size=18)

    suframa = fields.Char(
        string='Suframa',
        size=18)

    city_id = fields.Many2one(
        comodel_name='res.city',
        string='City ID',
        domain="[('state_id', '=', state_id)]")

    country_id = fields.Many2one(
        default=lambda self: self.env.ref('base.br'))

    district = fields.Char(
        string='District',
        size=32)

    street_number = fields.Char(
        string='Número',
        size=10)

    name_surname = fields.Char(
        string='Nome e sobrenome',
        size=128,
        help="Nome utilizado em documentos fiscais")

    cpf = fields.Char(
        string='CPF',
        size=18)

    rg = fields.Char(
        string='RG',
        size=16)

    @api.multi
    @api.constrains('cnpj')
    def _check_cnpj(self):
        for record in self:
            country_code = record.country_id.code or ''
            if record.cnpj and country_code.upper() == 'BR':
                cnpj = misc.punctuation_rm(record.cnpj)
                if not cnpj_cpf.validar(cnpj):
                    raise ValidationError(_('CNPJ inválido!'))
            return True

    @api.multi
    @api.constrains('cpf')
    def _check_cpf(self):
        for record in self:
            country_code = record.country_id.code or ''
            if record.cpf and country_code.upper() == 'BR':
                cpf = misc.punctuation_rm(record.cpf)
                if not cnpj_cpf.validar(cpf):
                    raise ValidationError(_('CPF inválido!'))
            return True

    @api.multi
    @api.constrains('inscr_est')
    def _check_ie(self):
        """Checks if company register number in field insc_est is valid,
        this method call others methods because this validation is State wise

        :Return: True or False.
        """
        for record in self:
            result = True
            if record.inscr_est and record.cnpj and record.state_id:
                state_code = record.state_id.code or ''
                uf = state_code.lower()
                result = ie.validar(uf, record.inscr_est)
            if not result:
                raise ValidationError(_("Inscrição Estadual Invalida!"))

    @api.onchange('cnpj', 'country_id')
    def _onchange_cnpj(self):
        country_code = self.country_id.code or ''
        self.cnpj = cnpj_cpf.formata(self.cnpj,
                                           country_code,
                                           True)

    @api.onchange('cpf', 'country_id')
    def _onchange_mask_cpf(self):
        country_code = self.country_id.code or ''
        self.cpf = cnpj_cpf.formata(self.cpf,
                                          country_code,
                                          False)

    @api.onchange('city_id')
    def _onchange_city_id(self):
        """ Ao alterar o campo l10n_br_city_id que é um campo relacional
        com o l10n_br_base.city que são os municípios do IBGE, copia o nome
        do município para o campo city que é o campo nativo do módulo base
        para manter a compatibilidade entre os demais módulos que usam o
        campo city.
        param int l10n_br_city_id: id do l10n_br_city_id digitado.
        return: dicionário com o nome e id do município.
        """
        if self.city_id:
            self.city = self.city_id.name

    @api.onchange('zip')
    def _onchange_zip(self):
        self.zip = misc.format_zipcode(self.zip,
                                       self.country_id.code)

    @api.onchange('partner_id')
    def _onchange_partner_id(self):
        result = super(Lead, self)._onchange_partner_id_values(
            self.partner_id.id if self.partner_id else False)

        if self.partner_id:
            result['street_number'] = self.partner_id.street_number
            result['district'] = self.partner_id.district
            result['city_id'] = self.partner_id.city_id.id
            if self.partner_id.is_company:
                result['legal_name'] = self.partner_id.legal_name
                result['cnpj'] = self.partner_id.cnpj_cpf
                result['inscr_est'] = self.partner_id.inscr_est
                result['suframa'] = self.partner_id.suframa
            else:
                result['cpf'] = self.partner_id.cnpj_cpf
                result['name_surname'] = self.partner_id.legal_name
        self.update(result)

    @api.multi
    def _create_lead_partner_data(self, name, is_company, parent_id=False):
        """ extract data from lead to create a partner
            :param name : furtur name of the partner
            :param is_company : True if the partner is a company
            :param parent_id : id of the parent partner (False if no parent)
            :returns res.partner record
        """
        values = super(Lead, self)._create_lead_partner_data(
            name, is_company, parent_id)
        values.update({
            'street_number': self.street_number,
            'district': self.district,
            'city_id': self.city_id.id
        })
        if is_company:
            values.update({
                'legal_name': self.legal_name,
                'cnpj_cpf': self.cnpj,
                'inscr_est': self.inscr_est,
                'inscr_mun': self.inscr_mun,
                'suframa': self.suframa,
            })
        else:
            values.update({
                'legal_name': self.name_surname,
                'cnpj_cpf': self.cpf,
                'inscr_est': self.rg,
            })
        return values
<commit_msg>Correct parameter numbers in functions crm_lead.py<commit_after># Copyright (C) 2012 - TODAY  Renato Lima - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

import logging

from odoo import models, fields, api, _
from odoo.exceptions import ValidationError

_logger = logging.getLogger(__name__)

try:
    from erpbrasil.base.fiscal import cnpj_cpf, ie
    from erpbrasil.base import misc
except ImportError:
    _logger.error("Biblioteca erpbrasil.base não instalada")

class Lead(models.Model):
    """ CRM Lead Case """
    _inherit = "crm.lead"

    legal_name = fields.Char(
        string='Legal Name',
        size=128,
        help="nome utilizado em documentos fiscais")

    cnpj = fields.Char(
        string='CNPJ',
        size=18)

    inscr_est = fields.Char(
        string='Inscr. Estadual/RG',
        size=16)

    inscr_mun = fields.Char(
        string='Inscr. Municipal',
        size=18)

    suframa = fields.Char(
        string='Suframa',
        size=18)

    city_id = fields.Many2one(
        comodel_name='res.city',
        string='City ID',
        domain="[('state_id', '=', state_id)]")

    country_id = fields.Many2one(
        default=lambda self: self.env.ref('base.br'))

    district = fields.Char(
        string='District',
        size=32)

    street_number = fields.Char(
        string='Número',
        size=10)

    name_surname = fields.Char(
        string='Nome e sobrenome',
        size=128,
        help="Nome utilizado em documentos fiscais")

    cpf = fields.Char(
        string='CPF',
        size=18)

    rg = fields.Char(
        string='RG',
        size=16)

    @api.multi
    @api.constrains('cnpj')
    def _check_cnpj(self):
        for record in self:
            country_code = record.country_id.code or ''
            if record.cnpj and country_code.upper() == 'BR':
                cnpj = misc.punctuation_rm(record.cnpj)
                if not cnpj_cpf.validar(cnpj):
                    raise ValidationError(_('CNPJ inválido!'))
            return True

    @api.multi
    @api.constrains('cpf')
    def _check_cpf(self):
        for record in self:
            country_code = record.country_id.code or ''
            if record.cpf and country_code.upper() == 'BR':
                cpf = misc.punctuation_rm(record.cpf)
                if not cnpj_cpf.validar(cpf):
                    raise ValidationError(_('CPF inválido!'))
            return True

    @api.multi
    @api.constrains('inscr_est')
    def _check_ie(self):
        """Checks if company register number in field insc_est is valid,
        this method call others methods because this validation is State wise

        :Return: True or False.
        """
        for record in self:
            result = True
            if record.inscr_est and record.cnpj and record.state_id:
                state_code = record.state_id.code or ''
                uf = state_code.lower()
                result = ie.validar(uf, record.inscr_est)
            if not result:
                raise ValidationError(_("Inscrição Estadual Invalida!"))

    @api.onchange('cnpj', 'country_id')
    def _onchange_cnpj(self):
        self.cnpj = cnpj_cpf.formata(self.cnpj)

    @api.onchange('cpf', 'country_id')
    def _onchange_mask_cpf(self):
        self.cpf = cnpj_cpf.formata(self.cpf)

    @api.onchange('city_id')
    def _onchange_city_id(self):
        """ Ao alterar o campo l10n_br_city_id que é um campo relacional
        com o l10n_br_base.city que são os municípios do IBGE, copia o nome
        do município para o campo city que é o campo nativo do módulo base
        para manter a compatibilidade entre os demais módulos que usam o
        campo city.
        param int l10n_br_city_id: id do l10n_br_city_id digitado.
        return: dicionário com o nome e id do município.
        """
        if self.city_id:
            self.city = self.city_id.name

    @api.onchange('zip')
    def _onchange_zip(self):
        self.zip = misc.format_zipcode(self.zip,
                                       self.country_id.code)

    @api.onchange('partner_id')
    def _onchange_partner_id(self):
        result = super(Lead, self)._onchange_partner_id_values(
            self.partner_id.id if self.partner_id else False)

        if self.partner_id:
            result['street_number'] = self.partner_id.street_number
            result['district'] = self.partner_id.district
            result['city_id'] = self.partner_id.city_id.id
            if self.partner_id.is_company:
                result['legal_name'] = self.partner_id.legal_name
                result['cnpj'] = self.partner_id.cnpj_cpf
                result['inscr_est'] = self.partner_id.inscr_est
                result['suframa'] = self.partner_id.suframa
            else:
                result['cpf'] = self.partner_id.cnpj_cpf
                result['name_surname'] = self.partner_id.legal_name
        self.update(result)

    @api.multi
    def _create_lead_partner_data(self, name, is_company, parent_id=False):
        """ extract data from lead to create a partner
            :param name : furtur name of the partner
            :param is_company : True if the partner is a company
            :param parent_id : id of the parent partner (False if no parent)
            :returns res.partner record
        """
        values = super(Lead, self)._create_lead_partner_data(
            name, is_company, parent_id)
        values.update({
            'street_number': self.street_number,
            'district': self.district,
            'city_id': self.city_id.id
        })
        if is_company:
            values.update({
                'legal_name': self.legal_name,
                'cnpj_cpf': self.cnpj,
                'inscr_est': self.inscr_est,
                'inscr_mun': self.inscr_mun,
                'suframa': self.suframa,
            })
        else:
            values.update({
                'legal_name': self.name_surname,
                'cnpj_cpf': self.cpf,
                'inscr_est': self.rg,
            })
        return values
<|endoftext|>