<commit_before># coding=utf-8
#
# Copyright 2015-2016 F5 Networks Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import time

from distutils.version import LooseVersion
from f5.bigip.resource import AsmResource
from f5.bigip.resource import Collection
from icontrol.exceptions import iControlUnexpectedHTTPError

from . methods import Methods_s
# from . filetypes import Filetypes_s
# from . cookies import Cookies_s
# from . host_names import Host_Names_s
# from . urls import Urls_s
from . parameters import Parameters_s
# from . data_guard import Data_Guard
from . blocking_settings import Blocking_Settings
# from . whitelist_ips import Whitelist_Ips_s
# from . gwt_profiles import Gwt_Profiles_s
# from . json_profiles import Json_Profiles_s
# from . xml_profiles import Xml_Profiles_s
# from . signatures import Signatures_s
# from . signature_sets import Signature_Sets_s
# from . headers import Headers_s
# from . response_pages import Response_Pages_s
# from . policy_builder import Policy_Builder
# from . history_revisions import History_Revisions_s
from . vulnerability_assessment import Vulnerability_Assessment
# from . geolocation_enforcement import Geolocation_Enforcement
# from . session_tracking import Session_Tracking
# from . session_tracking_status import Session_Tracking_Statuses_s
# from . login_pages import Login_Pages_s
# from . ip_intelligence import Ip_Intelligence
# from . csrf_protection import Csrf_Protection
# from . redirection_protection import Redirection_Protection
# from . login_enforcement import Login_Enforcement
# from . sensitive_parameters import Sensitive_Parameters_s
# from . brute_force import Brute_Force_Attack_Preventions_s
# from . xml_validation import Xml_Validation_Files_s
# from . extractions import Extractions_s
# from . vulnerabilities import Vulnerabilities_s
# from . navigation_parameters import Navigation_Parameters_s
# from . character_sets import Character_Sets_s
# from . web_scraping import Web_Scraping
# from . audit_logs import Audit_Logs_s
# from . suggestions import Suggestions_s
# from . plain_text_profiles import Plain_Text_Profiles_s
# from . websocket_urls import Websocket_Urls_s
from . general import General

class Policies_s(Collection):
    """BIG-IP® ASM Policies collection."""
    def __init__(self, asm):
        super(Policies_s, self).__init__(asm)
        self._meta_data['object_has_stats'] = False
        self._meta_data['allowed_lazy_attributes'] = [Policy]
        self._meta_data['attribute_registry'] = {
            'tm:asm:policies:policystate': Policy
        }

class Policy(AsmResource):
    """BIG-IP® ASM Policies resource."""
    def __init__(self, policies_s):
        super(Policy, self).__init__(policies_s)
        self._meta_data['required_json_kind'] = 'tm:asm:policies:policystate'
        self._meta_data['attribute_registry'] = {
            'tm:asm:policies:methods:methodcollectionstate': Methods_s,
            #            'tm:asm:policies:filetypes:filetypecollectionstate': Filetypes_s,
            #            'tm:asm:policies:cookies:cookiecollectionstate': Cookies_s,
            #            'tm:asm:policies:host-names:host-namecollectionstate': Host_Names_s,
            #            'tm:asm:policies:urls:urlcollectionstate': Urls_s,
            'tm:asm:policies:parameters:parametercollectionstate': Parameters_s,
            #            'tm:asm:policies:whitelist-ips:whitelist-ipcollectionstate': Whitelist_Ips_s,
            #            'tm:asm:policies:gwt-profiles:gwt-profilecollectionstate': Gwt_Profiles_s,
            #            'tm:asm:policies:json-profiles:json-profilecollectionstate': Json_Profiles_s,
            #            'tm:asm:policies:xml-profiles:xml-profilecollectionstate': Xml_Profiles_s,
            #            'tm:asm:policies:signatures:signaturecollectionstate': Signatures_s,
            #            'tm:asm:policies:signature-sets:signature-setcollectionstate': Signature_Sets_s,
            #            'tm:asm:policies:headers:headercollectionstate': Headers_s,
            #            'tm:asm:policies:response-pages:response-pagecollectionstate': Response_Pages_s,
            #            'tm:asm:policies:history-revisions:history-revisioncollectionstate': History_Revisions_s,
            'tm:asm:policies:vulnerability-assessment:vulnerability-assessmentstate': Vulnerability_Assessment,
            #            'tm:asm:policies:data-guard:data-guardstate': Data_Guard,
            #            'tm:asm:policies:geolocation-enforcement:geolocation-enforcementstate': Geolocation_Enforcement,
            #            'tm:asm:policies:session-tracking:session-awareness-settingsstate': Session_Tracking,
            #            'tm:asm:policies:session-tracking-statuses:session-tracking-statuscollectionstate': Session_Tracking_Statuses_s,
            #            'tm:asm:policies:login-pages:login-pagecollectionstate': Login_Pages_s,
            #            'tm:asm:policies:ip-intelligence:ip-intelligencestate': Ip_Intelligence,
            #            'tm:asm:policies:csrf-protection:csrf-protectionstate': Csrf_Protection,
            #            'tm:asm:policies:redirection-protection:redirection-protectionstate': Redirection_Protection,
            #            'tm:asm:policies:login-enforcement:login-enforcementstate': Login_Enforcement,
            #            'tm:asm:policies:sensitive-parameters:sensitive-parametercollectionstate': Sensitive_Parameters_s,
            #            'tm:asm:policies:brute-force-attack-preventions:brute-force-attack-preventioncollectionstate': Brute_Force_Attack_Preventions_s,
            #            'tm:asm:policies:xml-validation-files:xml-validation-filecollectionstate': Xml_Validation_Files_s,
            #            'tm:asm:policies:extractions:extractioncollectionstate': Extractions_s,
            #            'tm:asm:policies:vulnerabilities:vulnerabilitycollectionstate': Vulnerabilities_s,
            #            'tm:asm:policies:navigation-parameters:navigation-parametercollectionstate': Navigation_Parameters_s,
            #            'tm:asm:policies:character-sets:character-setcollectionstate': Character_Sets_s,
            #            'tm:asm:policies:web-scraping:web-scrapingstate': Web_Scraping,
            #            'tm:asm:policies:audit-logs:audit-logcollectionstate': Audit_Logs_s,
            #            'tm:asm:policies:suggestions:suggestioncollectionstate': Suggestions_s,
            #            'tm:asm:policies:plain-text-profiles:plain-text-profilecollectionstate': Plain_Text_Profiles_s,
            #            'tm:asm:policies:websocket-urls:websocket-urlcollectionstate': Websocket_Urls_s
        }

        tmos_v = self._meta_data['bigip']._meta_data['tmos_version']
        if LooseVersion(tmos_v) >= LooseVersion('13.0.0'):
            self._meta_data['attribute_registry'].update({
                'tm:asm:policies:general:generalstate': General
            })

        self._set_attr_reg()

    def _set_attr_reg(self):
        """Helper method.

        Appends correct attribute registry, depending on TMOS version

        """
        tmos_v = self._meta_data['bigip']._meta_data['tmos_version']
        attributes = self._meta_data['attribute_registry']
        v12kind = 'tm:asm:policies:blocking-settings:blocking-settingcollectionstate'
        v11kind = 'tm:asm:policies:blocking-settings'
        builderv11 = 'tm:asm:policies:policy-builder:pbconfigstate'
        builderv12 = 'tm:asm:policies:policy-builder:policy-builderstate'
        if LooseVersion(tmos_v) < LooseVersion('12.0.0'):
            attributes[v11kind] = Blocking_Settings
            # attributes[builderv11] = Policy_Builder
        else:
            attributes[v12kind] = Blocking_Settings
            # attributes[builderv12] = Policy_Builder

    def create(self, **kwargs):
        """Custom creation logic to handle edge cases

        This shouldn't be needed, but ASM has a tendency to raise various errors that
        are painful to handle from a customer point-of-view

        The error itself are described in their exception handler

        To address these failure, we try a number of exception handling cases to catch
        and reliably deal with the error.

        :param kwargs:
        :return:
        """
        for x in range(0, 30):
            try:
                return self._create(**kwargs)
            except iControlUnexpectedHTTPError as ex:
                if self._check_exception(ex):
                    continue
                else:
                    raise

    def delete(self, **kwargs):
        """Custom deletion logic to handle edge cases

        This shouldn't be needed, but ASM has a tendency to raise various errors that
        are painful to handle from a customer point-of-view

        The error itself are described in their exception handler

        To address these failure, we try a number of exception handling cases to catch
        and reliably deal with the error.

        :param kwargs:
        :return:
        """
        for x in range(0, 30):
            try:
                return self._delete(**kwargs)
            except iControlUnexpectedHTTPError as ex:
                if self._check_exception(ex):
                    continue
                else:
                    raise

    def _check_exception(self, ex):
        retryable = [
            # iControlUnexpectedHTTPError: 400 Unexpected Error: Bad Request for uri: ...
            # {
            #     "code":400,
            #     "message": "remoteSender:10.0.2.2,method:DELETE ",
            #     "referer":"10.0.2.2",
            #     "restOperationId":103685,
            #     "errorStack":[
            #         "java.util.concurrent.TimeoutException: remoteSender:10.0.2.2, method:DELETE ",
            #         "at com.f5.rest.common.RestWorker.logAndFailExpiredOperation(RestWorker.java:3035)",
            #         "at com.f5.reent.TimeoutException: remoteSender:10.0.2.2, method:DELETE ",
            #         "at com.f5.rest.common.RestWorker.logAndFailExpiredOperation(RestWorker.java:3035)",
            #         "at com.f5.rest.common.RestWorker.checkForExpiredOperations(RestWorker.java:3024)",
            #         "at com.f5.rest.common.RestServer.checkAndExpirePendingWorkerOperations(RestServer.java:1416)",
            #         "at com.f5.rest.common.RestServer.access$400(RestServer.java:45)",
            #         "at com.f5.rest.common.RestServer$4.run(RestServer.java:1389)",
            #         "at com.f5.rest.common.ScheduleTaskManager$2$1.run(ScheduleTaskManager.java:123)",
            #         "at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)",
            #         "at java.util.concurrent.FutureTask.run(FutureTask.java:262)",
            #         "at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)",
            #         "at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)",
            #         "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)",
            #         "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)",
            #         "at java.lang.Thread.run(Thread.java:745)\\n"
            #     ],
            #     "kind":":resterrorresponse"
            # }'
            'TimeoutException',

            # {
            #   "code":400,
            #   "message":"Failed on insert to DCC.FILE_REFERENCES_FOR_DELETION
            #              (DBD::mysql::db do failed: Duplicate entry
            #              \'/ts/var/xml/defense_config/xml_profile_2312\' for key \'filename\')",
            #   "referer":"10.0.2.2",
            #   "restOperationId":142433,
            #   "kind":":resterrorresponse"
            # }
            'Failed on insert to DCC.FILE_REFERENCES_FOR_DELETION',

            # iControlUnexpectedHTTPError: 500 Unexpected Error: Internal Server Error ...
            # {
            #   "code": 500,
            #   "message": "Could not add_signature the Attack Signature.  "
            #              "Failed on insert to PLC.NEGSIG_SET_SIGNATURES "
            #              "(DBD::mysql::db do failed: Lock wait timeout exceeded; "
            #              "try restarting transaction)
            #
            'Lock wait timeout exceeded',
        ]
        if any(x in str(ex) for x in retryable):
            time.sleep(3)
            return True
        elif 'errorStack' in ex:
            stack = ' '.join(ex['errorStack'])
            if any(x in stack for x in retryable):
                time.sleep(3)
                return True
            else:
                return False
        else:
            return False
<commit_msg>Re-adds all imports to ASM<commit_after># coding=utf-8
#
# Copyright 2015-2016 F5 Networks Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import time

from distutils.version import LooseVersion
from f5.bigip.resource import AsmResource
from f5.bigip.resource import Collection
from icontrol.exceptions import iControlUnexpectedHTTPError

from . methods import Methods_s
from . filetypes import Filetypes_s
from . cookies import Cookies_s
from . host_names import Host_Names_s
from . urls import Urls_s
from . parameters import Parameters_s
from . data_guard import Data_Guard
from . blocking_settings import Blocking_Settings
from . whitelist_ips import Whitelist_Ips_s
from . gwt_profiles import Gwt_Profiles_s
from . json_profiles import Json_Profiles_s
from . xml_profiles import Xml_Profiles_s
from . signatures import Signatures_s
from . signature_sets import Signature_Sets_s
from . headers import Headers_s
from . response_pages import Response_Pages_s
from . policy_builder import Policy_Builder
from . history_revisions import History_Revisions_s
from . vulnerability_assessment import Vulnerability_Assessment
from . geolocation_enforcement import Geolocation_Enforcement
from . session_tracking import Session_Tracking
from . session_tracking_status import Session_Tracking_Statuses_s
from . login_pages import Login_Pages_s
from . ip_intelligence import Ip_Intelligence
from . csrf_protection import Csrf_Protection
from . redirection_protection import Redirection_Protection
from . login_enforcement import Login_Enforcement
from . sensitive_parameters import Sensitive_Parameters_s
from . brute_force import Brute_Force_Attack_Preventions_s
from . xml_validation import Xml_Validation_Files_s
from . extractions import Extractions_s
from . vulnerabilities import Vulnerabilities_s
from . navigation_parameters import Navigation_Parameters_s
from . character_sets import Character_Sets_s
from . web_scraping import Web_Scraping
from . audit_logs import Audit_Logs_s
from . suggestions import Suggestions_s
from . plain_text_profiles import Plain_Text_Profiles_s
from . websocket_urls import Websocket_Urls_s
from . general import General

class Policies_s(Collection):
    """BIG-IP® ASM Policies collection."""
    def __init__(self, asm):
        super(Policies_s, self).__init__(asm)
        self._meta_data['object_has_stats'] = False
        self._meta_data['allowed_lazy_attributes'] = [Policy]
        self._meta_data['attribute_registry'] = {
            'tm:asm:policies:policystate': Policy
        }

class Policy(AsmResource):
    """BIG-IP® ASM Policies resource."""
    def __init__(self, policies_s):
        super(Policy, self).__init__(policies_s)
        self._meta_data['required_json_kind'] = 'tm:asm:policies:policystate'
        self._meta_data['attribute_registry'] = {
            'tm:asm:policies:methods:methodcollectionstate': Methods_s,
            'tm:asm:policies:filetypes:filetypecollectionstate': Filetypes_s,
            'tm:asm:policies:cookies:cookiecollectionstate': Cookies_s,
            'tm:asm:policies:host-names:host-namecollectionstate': Host_Names_s,
            'tm:asm:policies:urls:urlcollectionstate': Urls_s,
            'tm:asm:policies:parameters:parametercollectionstate': Parameters_s,
            'tm:asm:policies:whitelist-ips:whitelist-ipcollectionstate': Whitelist_Ips_s,
            'tm:asm:policies:gwt-profiles:gwt-profilecollectionstate': Gwt_Profiles_s,
            'tm:asm:policies:json-profiles:json-profilecollectionstate': Json_Profiles_s,
            'tm:asm:policies:xml-profiles:xml-profilecollectionstate': Xml_Profiles_s,
            'tm:asm:policies:signatures:signaturecollectionstate': Signatures_s,
            'tm:asm:policies:signature-sets:signature-setcollectionstate': Signature_Sets_s,
            'tm:asm:policies:headers:headercollectionstate': Headers_s,
            'tm:asm:policies:response-pages:response-pagecollectionstate': Response_Pages_s,
            'tm:asm:policies:history-revisions:history-revisioncollectionstate': History_Revisions_s,
            'tm:asm:policies:vulnerability-assessment:vulnerability-assessmentstate': Vulnerability_Assessment,
            'tm:asm:policies:data-guard:data-guardstate': Data_Guard,
            'tm:asm:policies:geolocation-enforcement:geolocation-enforcementstate': Geolocation_Enforcement,
            'tm:asm:policies:session-tracking:session-awareness-settingsstate': Session_Tracking,
            'tm:asm:policies:session-tracking-statuses:session-tracking-statuscollectionstate': Session_Tracking_Statuses_s,
            'tm:asm:policies:login-pages:login-pagecollectionstate': Login_Pages_s,
            'tm:asm:policies:ip-intelligence:ip-intelligencestate': Ip_Intelligence,
            'tm:asm:policies:csrf-protection:csrf-protectionstate': Csrf_Protection,
            'tm:asm:policies:redirection-protection:redirection-protectionstate': Redirection_Protection,
            'tm:asm:policies:login-enforcement:login-enforcementstate': Login_Enforcement,
            'tm:asm:policies:sensitive-parameters:sensitive-parametercollectionstate': Sensitive_Parameters_s,
            'tm:asm:policies:brute-force-attack-preventions:brute-force-attack-preventioncollectionstate': Brute_Force_Attack_Preventions_s,
            'tm:asm:policies:xml-validation-files:xml-validation-filecollectionstate': Xml_Validation_Files_s,
            'tm:asm:policies:extractions:extractioncollectionstate': Extractions_s,
            'tm:asm:policies:vulnerabilities:vulnerabilitycollectionstate': Vulnerabilities_s,
            'tm:asm:policies:navigation-parameters:navigation-parametercollectionstate': Navigation_Parameters_s,
            'tm:asm:policies:character-sets:character-setcollectionstate': Character_Sets_s,
            'tm:asm:policies:web-scraping:web-scrapingstate': Web_Scraping,
            'tm:asm:policies:audit-logs:audit-logcollectionstate': Audit_Logs_s,
            'tm:asm:policies:suggestions:suggestioncollectionstate': Suggestions_s,
            'tm:asm:policies:plain-text-profiles:plain-text-profilecollectionstate': Plain_Text_Profiles_s,
            'tm:asm:policies:websocket-urls:websocket-urlcollectionstate': Websocket_Urls_s
        }

        tmos_v = self._meta_data['bigip']._meta_data['tmos_version']
        if LooseVersion(tmos_v) >= LooseVersion('13.0.0'):
            self._meta_data['attribute_registry'].update({
                'tm:asm:policies:general:generalstate': General
            })

        self._set_attr_reg()

    def _set_attr_reg(self):
        """Helper method.

        Appends correct attribute registry, depending on TMOS version

        """
        tmos_v = self._meta_data['bigip']._meta_data['tmos_version']
        attributes = self._meta_data['attribute_registry']
        v12kind = 'tm:asm:policies:blocking-settings:blocking-settingcollectionstate'
        v11kind = 'tm:asm:policies:blocking-settings'
        builderv11 = 'tm:asm:policies:policy-builder:pbconfigstate'
        builderv12 = 'tm:asm:policies:policy-builder:policy-builderstate'
        if LooseVersion(tmos_v) < LooseVersion('12.0.0'):
            attributes[v11kind] = Blocking_Settings
            attributes[builderv11] = Policy_Builder
        else:
            attributes[v12kind] = Blocking_Settings
            attributes[builderv12] = Policy_Builder

    def create(self, **kwargs):
        """Custom creation logic to handle edge cases

        This shouldn't be needed, but ASM has a tendency to raise various errors that
        are painful to handle from a customer point-of-view

        The error itself are described in their exception handler

        To address these failure, we try a number of exception handling cases to catch
        and reliably deal with the error.

        :param kwargs:
        :return:
        """
        for x in range(0, 30):
            try:
                return self._create(**kwargs)
            except iControlUnexpectedHTTPError as ex:
                if self._check_exception(ex):
                    continue
                else:
                    raise

    def delete(self, **kwargs):
        """Custom deletion logic to handle edge cases

        This shouldn't be needed, but ASM has a tendency to raise various errors that
        are painful to handle from a customer point-of-view

        The error itself are described in their exception handler

        To address these failure, we try a number of exception handling cases to catch
        and reliably deal with the error.

        :param kwargs:
        :return:
        """
        for x in range(0, 30):
            try:
                return self._delete(**kwargs)
            except iControlUnexpectedHTTPError as ex:
                if self._check_exception(ex):
                    continue
                else:
                    raise

    def _check_exception(self, ex):
        retryable = [
            # iControlUnexpectedHTTPError: 400 Unexpected Error: Bad Request for uri: ...
            # {
            #     "code":400,
            #     "message": "remoteSender:10.0.2.2,method:DELETE ",
            #     "referer":"10.0.2.2",
            #     "restOperationId":103685,
            #     "errorStack":[
            #         "java.util.concurrent.TimeoutException: remoteSender:10.0.2.2, method:DELETE ",
            #         "at com.f5.rest.common.RestWorker.logAndFailExpiredOperation(RestWorker.java:3035)",
            #         "at com.f5.reent.TimeoutException: remoteSender:10.0.2.2, method:DELETE ",
            #         "at com.f5.rest.common.RestWorker.logAndFailExpiredOperation(RestWorker.java:3035)",
            #         "at com.f5.rest.common.RestWorker.checkForExpiredOperations(RestWorker.java:3024)",
            #         "at com.f5.rest.common.RestServer.checkAndExpirePendingWorkerOperations(RestServer.java:1416)",
            #         "at com.f5.rest.common.RestServer.access$400(RestServer.java:45)",
            #         "at com.f5.rest.common.RestServer$4.run(RestServer.java:1389)",
            #         "at com.f5.rest.common.ScheduleTaskManager$2$1.run(ScheduleTaskManager.java:123)",
            #         "at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)",
            #         "at java.util.concurrent.FutureTask.run(FutureTask.java:262)",
            #         "at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)",
            #         "at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)",
            #         "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)",
            #         "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)",
            #         "at java.lang.Thread.run(Thread.java:745)\\n"
            #     ],
            #     "kind":":resterrorresponse"
            # }'
            'TimeoutException',

            # {
            #   "code":400,
            #   "message":"Failed on insert to DCC.FILE_REFERENCES_FOR_DELETION
            #              (DBD::mysql::db do failed: Duplicate entry
            #              \'/ts/var/xml/defense_config/xml_profile_2312\' for key \'filename\')",
            #   "referer":"10.0.2.2",
            #   "restOperationId":142433,
            #   "kind":":resterrorresponse"
            # }
            'Failed on insert to DCC.FILE_REFERENCES_FOR_DELETION',

            # iControlUnexpectedHTTPError: 500 Unexpected Error: Internal Server Error ...
            # {
            #   "code": 500,
            #   "message": "Could not add_signature the Attack Signature.  "
            #              "Failed on insert to PLC.NEGSIG_SET_SIGNATURES "
            #              "(DBD::mysql::db do failed: Lock wait timeout exceeded; "
            #              "try restarting transaction)
            #
            'Lock wait timeout exceeded',
        ]
        if any(x in str(ex) for x in retryable):
            time.sleep(3)
            return True
        elif 'errorStack' in ex:
            stack = ' '.join(ex['errorStack'])
            if any(x in stack for x in retryable):
                time.sleep(3)
                return True
            else:
                return False
        else:
            return False
<|endoftext|>


<commit_before>__author__ = 'arkilic'

from mongoengine import (Document, FloatField, DateTimeField, StringField, DictField, ReferenceField, \
                        IntField, BooleanField, DENY)
from fileStore.database.file_base import FileBase
import time
from datetime import datetime

class FileAttributes(Document):
    """

    Parameters
    ----------

    """
    file_base = ReferenceField(FileBase, reverse_delete_rule=DENY, required=True,
                               db_field='file_base_id')
    shape = StringField(unique=False, required=True)
    dtype = StringField(unique=False, required=True)
    total_bytes = IntField(min_value=0, required=False, default=0)
    hashed_data = StringField(required=False)
    last_access = FloatField(required=False, default=time.time())
    datetime_last_access = DateTimeField(required=False, default=datetime.fromtimestamp(last_access))
    in_use = BooleanField(required=False, default=0)
<commit_msg>ENH: Additional field to file_attribute collection for custom attribute appending<commit_after>__author__ = 'arkilic'

from mongoengine import (Document, FloatField, DateTimeField, StringField, DictField, ReferenceField, \
                        IntField, BooleanField, DENY)
from fileStore.database.file_base import FileBase
import time
from datetime import datetime

class FileAttributes(Document):
    """

    Parameters
    ----------

    """
    file_base = ReferenceField(FileBase, reverse_delete_rule=DENY, required=True,
                               db_field='file_base_id')
    shape = StringField(unique=False, required=True)
    dtype = StringField(unique=False, required=True)
    total_bytes = IntField(min_value=0, required=False, default=0)
    hashed_data = StringField(required=False)
    last_access = FloatField(required=False, default=time.time())
    datetime_last_access = DateTimeField(required=False, default=datetime.fromtimestamp(last_access))
    in_use = BooleanField(required=False, default=0)
    custom_attributes = DictField(required=False)<|endoftext|>


<commit_before>import tensorflow as tf
import tensorflow.contrib.slim as slim

from tensorflow.contrib.layers.python.layers import initializers, utils
from tensorflow.contrib.framework.python.ops import variables
from tensorflow.python.ops import init_ops

from luminoth.models.base import BaseNetwork

VALID_ARCHITECTURES = set([
    'vgg_16',
])

class SSDFeatureExtractor(BaseNetwork):

    def __init__(self, config, parent_name=None, name='ssd_feature_extractor',
                 **kwargs):
        super(SSDFeatureExtractor, self).__init__(config, name=name, **kwargs)
        if config.get('architecture') not in VALID_ARCHITECTURES:
            raise ValueError('Invalid architecture "{}"'.format(
                config.get('architecture')
            ))

        self._architecture = config.get('architecture')
        self._config = config
        self.parent_name = parent_name
        self._dropout_keep_prob = config.dropout_keep_prob

    def _build(self, inputs, is_training=True):
        """
        Args:
            inputs: A Tensor of shape `(batch_size, height, width, channels)`.

        Returns:
            A dict of feature maps to be consumed by an SSD network
        """
        # TODO: Is there a better way to manage scoping in these cases?
        scope = self.module_name
        if self.parent_name:
            scope = self.parent_name + '/' + scope

        inputs = self.preprocess(inputs)

        base_net_endpoints = super(SSDFeatureExtractor, self)._build(
            inputs, is_training=is_training)['end_points']

        # The original SSD paper uses a modified version of the vgg16 network,
        # which we'll build here
        if self.vgg_type:
            base_network_truncation_endpoint = base_net_endpoints[
                scope + '/vgg_16/conv5/conv5_3']

            # We'll add the feature maps to a collection. In the paper they use
            # one of vgg16's layers as a feature map, so we start by adding it.
            vgg_conv4_3_name = scope + '/vgg_16/conv4/conv4_3'
            vgg_conv4_3 = base_net_endpoints[vgg_conv4_3_name]

            # As it is pointed out in SSD and ParseNet papers, `conv4_3` has a
            # different features scale compared to other layers, to adjust it
            # we need to add a spatial normalization before adding the
            # predictors.
            with tf.variable_scope(vgg_conv4_3_name + '_norm'):
                inputs_shape = vgg_conv4_3.shape
                inputs_rank = inputs_shape.ndims
                dtype = vgg_conv4_3.dtype.base_dtype

                norm_dim = tf.range(inputs_rank - 1, inputs_rank)
                params_shape = inputs_shape[-1:]

                vgg_conv4_3_norm = tf.nn.l2_normalize(
                    vgg_conv4_3, norm_dim, epsilon=1e-12
                )

                # Post scaling.
                scale = variables.model_variable(
                    'gamma', shape=params_shape, dtype=dtype,
                    initializer=init_ops.ones_initializer()
                )

                vgg_conv4_3_norm = tf.multiply(vgg_conv4_3_norm, scale)

            tf.add_to_collection('FEATURE_MAPS', vgg_conv4_3_norm)

            # TODO: check that the usage of `padding='VALID'` is correct
            # TODO: check that the 1x1 convs actually use relu
            # Modifications to vgg16
            with tf.variable_scope('extra_feature_layers'):
                net = slim.max_pool2d(base_network_truncation_endpoint, [3, 3],
                                      padding='SAME', stride=1, scope='pool5')
                # TODO: or is it rate=12?
                net = slim.conv2d(net, 1024, [3, 3], rate=6, scope='conv6')
                net = slim.dropout(net, self._dropout_keep_prob)
                net = slim.conv2d(net, 1024, [1, 1], scope='conv7',
                                  outputs_collections='FEATURE_MAPS')
                net = slim.dropout(net, self._dropout_keep_prob)
                net = slim.conv2d(net, 256, [1, 1], scope='conv8_1')
                net = slim.conv2d(net, 512, [3, 3], stride=2, scope='conv8_2',
                                  outputs_collections='FEATURE_MAPS')
                net = slim.conv2d(net, 128, [1, 1], scope='conv9_1')
                net = slim.conv2d(net, 256, [3, 3], stride=2, scope='conv9_2',
                                  outputs_collections='FEATURE_MAPS')
                net = slim.conv2d(net, 128, [1, 1], scope='conv10_1')
                net = slim.conv2d(net, 256, [3, 3], scope='conv10_2',
                                  padding='VALID',
                                  outputs_collections='FEATURE_MAPS')
                net = slim.conv2d(net, 128, [1, 1], scope='conv11_1')
                net = slim.conv2d(net, 256, [3, 3], scope='conv11_2',
                                  padding='VALID',
                                  outputs_collections='FEATURE_MAPS')

            # This parameter determines onto which variables we try to load the
            # pretrained weights
            self.pretrained_weights_scope = 'ssd_feature_extractor/vgg_16'

        # Its actually an ordered dict
        return utils.convert_collection_to_dict('FEATURE_MAPS')
<commit_msg>SSD: Fix substracting the dataset's mean from input images twice<commit_after>import tensorflow as tf
import tensorflow.contrib.slim as slim

from tensorflow.contrib.layers.python.layers import initializers, utils
from tensorflow.contrib.framework.python.ops import variables
from tensorflow.python.ops import init_ops

from luminoth.models.base import BaseNetwork

VALID_ARCHITECTURES = set([
    'vgg_16',
])

class SSDFeatureExtractor(BaseNetwork):

    def __init__(self, config, parent_name=None, name='ssd_feature_extractor',
                 **kwargs):
        super(SSDFeatureExtractor, self).__init__(config, name=name, **kwargs)
        if config.get('architecture') not in VALID_ARCHITECTURES:
            raise ValueError('Invalid architecture "{}"'.format(
                config.get('architecture')
            ))

        self._architecture = config.get('architecture')
        self._config = config
        self.parent_name = parent_name
        self._dropout_keep_prob = config.dropout_keep_prob

    def _build(self, inputs, is_training=True):
        """
        Args:
            inputs: A Tensor of shape `(batch_size, height, width, channels)`.

        Returns:
            A dict of feature maps to be consumed by an SSD network
        """
        # TODO: Is there a better way to manage scoping in these cases?
        scope = self.module_name
        if self.parent_name:
            scope = self.parent_name + '/' + scope

        base_net_endpoints = super(SSDFeatureExtractor, self)._build(
            inputs, is_training=is_training)['end_points']

        # The original SSD paper uses a modified version of the vgg16 network,
        # which we'll build here
        if self.vgg_type:
            base_network_truncation_endpoint = base_net_endpoints[
                scope + '/vgg_16/conv5/conv5_3']

            # We'll add the feature maps to a collection. In the paper they use
            # one of vgg16's layers as a feature map, so we start by adding it.
            vgg_conv4_3_name = scope + '/vgg_16/conv4/conv4_3'
            vgg_conv4_3 = base_net_endpoints[vgg_conv4_3_name]

            # As it is pointed out in SSD and ParseNet papers, `conv4_3` has a
            # different features scale compared to other layers, to adjust it
            # we need to add a spatial normalization before adding the
            # predictors.
            with tf.variable_scope(vgg_conv4_3_name + '_norm'):
                inputs_shape = vgg_conv4_3.shape
                inputs_rank = inputs_shape.ndims
                dtype = vgg_conv4_3.dtype.base_dtype

                norm_dim = tf.range(inputs_rank - 1, inputs_rank)
                params_shape = inputs_shape[-1:]

                vgg_conv4_3_norm = tf.nn.l2_normalize(
                    vgg_conv4_3, norm_dim, epsilon=1e-12
                )

                # Post scaling.
                scale = variables.model_variable(
                    'gamma', shape=params_shape, dtype=dtype,
                    initializer=init_ops.ones_initializer()
                )

                vgg_conv4_3_norm = tf.multiply(vgg_conv4_3_norm, scale)

            tf.add_to_collection('FEATURE_MAPS', vgg_conv4_3_norm)

            # TODO: check that the usage of `padding='VALID'` is correct
            # TODO: check that the 1x1 convs actually use relu
            # Modifications to vgg16
            with tf.variable_scope('extra_feature_layers'):
                net = slim.max_pool2d(base_network_truncation_endpoint, [3, 3],
                                      padding='SAME', stride=1, scope='pool5')
                # TODO: or is it rate=12?
                net = slim.conv2d(net, 1024, [3, 3], rate=6, scope='conv6')
                net = slim.dropout(net, self._dropout_keep_prob)
                net = slim.conv2d(net, 1024, [1, 1], scope='conv7',
                                  outputs_collections='FEATURE_MAPS')
                net = slim.dropout(net, self._dropout_keep_prob)
                net = slim.conv2d(net, 256, [1, 1], scope='conv8_1')
                net = slim.conv2d(net, 512, [3, 3], stride=2, scope='conv8_2',
                                  outputs_collections='FEATURE_MAPS')
                net = slim.conv2d(net, 128, [1, 1], scope='conv9_1')
                net = slim.conv2d(net, 256, [3, 3], stride=2, scope='conv9_2',
                                  outputs_collections='FEATURE_MAPS')
                net = slim.conv2d(net, 128, [1, 1], scope='conv10_1')
                net = slim.conv2d(net, 256, [3, 3], scope='conv10_2',
                                  padding='VALID',
                                  outputs_collections='FEATURE_MAPS')
                net = slim.conv2d(net, 128, [1, 1], scope='conv11_1')
                net = slim.conv2d(net, 256, [3, 3], scope='conv11_2',
                                  padding='VALID',
                                  outputs_collections='FEATURE_MAPS')

            # This parameter determines onto which variables we try to load the
            # pretrained weights
            self.pretrained_weights_scope = 'ssd_feature_extractor/vgg_16'

        # Its actually an ordered dict
        return utils.convert_collection_to_dict('FEATURE_MAPS')
<|endoftext|>


<commit_before>import json

from django.db import models, transaction
from django.contrib.auth.models import User
from django.urls.base import reverse
from django.utils.html import escape

from world.models import Tile, Character

class Organization(models.Model):
    DEMOCRATIC = 'democratic'  # decisions are voted among members
    DISTRIBUTED = 'distributed'  # decisions can be taken by each member
    DECISION_TAKING_CHOICES = (
        (DEMOCRATIC, DEMOCRATIC),
        (DISTRIBUTED, DISTRIBUTED),
    )

    CHARACTER = 'character'
    ORGANIZATION = 'organization'
    MEMBERSHIP_TYPE_CHOICES = (
        (CHARACTER, CHARACTER),
        (ORGANIZATION, ORGANIZATION),
    )

    INHERITED = 'inherited'
    ELECTED = 'elected'
    POSITION_TYPE_CHOICES = (
        (INHERITED, INHERITED),
        (ELECTED, ELECTED),
    )

    world = models.ForeignKey('world.World')
    name = models.CharField(max_length=100)
    color = models.CharField(max_length=6, default="FFFFFF", help_text="Format: RRGGBB (hex)")
    description = models.TextField()
    is_position = models.BooleanField()
    position_type = models.CharField(max_length=15, choices=POSITION_TYPE_CHOICES, blank=True, default='')
    owner = models.ForeignKey('Organization', null=True, blank=True, related_name='owned_organizations')
    leader = models.ForeignKey('Organization', null=True, blank=True, related_name='leaded_organizations')
    owner_and_leader_locked = models.BooleanField(
        help_text="If set, this organization will have always the same leader as it's owner."
    )
    violence_monopoly = models.BooleanField(default=False)
    decision_taking = models.CharField(max_length=15, choices=DECISION_TAKING_CHOICES)
    membership_type = models.CharField(max_length=15, choices=MEMBERSHIP_TYPE_CHOICES)
    character_members = models.ManyToManyField('world.Character', blank=True)
    organization_members = models.ManyToManyField('Organization', blank=True)
    election_period_months = models.IntegerField(default=0)
    current_election = models.ForeignKey('PositionElection', blank=True, null=True, related_name='+')
    last_election = models.ForeignKey('PositionElection', blank=True, null=True, related_name='+')
    heir_first = models.ForeignKey(Character, blank=True, null=True, related_name='first_heir_to')
    heir_second = models.ForeignKey(Character, blank=True, null=True, related_name='second_heir_to')

    def get_all_descendants(self, including_self=False):
        descendants = list(self.owned_organizations.all())
        if including_self:
            descendants.append(self)
        for child in descendants:
            descendants += child.get_all_descendants()
        return descendants

    def get_membership_including_descendants(self):
        members = list(self.character_members.all())
        for child in self.owned_organizations.all():
            members += child.get_membership_including_descendants()
        return members

    def character_can_use_capabilities(self, character):
        if character in self.character_members.all():
            return True
        for member_organization in self.organization_members.all():
            if (
                            member_organization.leader and
                            member_organization.leader.is_position and
                            character in member_organization.organization_members.all()
            ):
                return True
            if member_organization.is_position and character in member_organization.organization_members.all():
                return True
        return False

    def organizations_character_can_apply_capabilities_to_this_with(self, character, capability_type):
        result = []
        capabilities = Capability.objects.filter(applying_to=self, type=capability_type)
        for capability in capabilities:
            if capability.organization.character_is_member(character):
                result.append(capability.organization)
        return result

    def character_is_member(self, character):
        if character in self.character_members.all():
            return True
        for member_organization in self.organization_members.all():
            if character in member_organization.member_set.all():
                return True
        return False

    def is_part_of_violence_monopoly(self):
        return (
            True if self.violence_monopoly
            else False if not self.owner
            else self.owner.is_part_of_violence_monopoly()
        )

    def get_open_proposals(self):
        return CapabilityProposal.objects.filter(capability__applying_to=self, closed=False)

    def get_all_controlled_tiles(self):
        return Tile.objects.filter(controlled_by__in=self.get_all_descendants(including_self=True)).all()

    def external_capabilities_to_this(self):
        return self.capabilities_to_this.exclude(organization=self)

    def get_position_ocuppier(self):
        if not self.is_position or not self.character_members.exists():
            return None
        return list(self.character_members.all())[0]

    @transaction.atomic
    def convoke_elections(self, months_to_election=6):
        if not self.is_position:
            raise Exception("Elections only work for positions")
        election = PositionElection.objects.create(
            position=self,
            turn=self.world.current_turn + months_to_election
        )
        self.current_election = election
        self.save()
        if self.get_position_ocuppier() is not None:
            PositionCandidate.objects.create(
                election=election,
                candidate=self.get_position_ocuppier(),
                description="Auto-generated candidacy for incumbent character."
            )

    def __str__(self):
        return self.name

    def get_absolute_url(self):
        return reverse('organization:view', kwargs={'organization_id': self.id})

    def get_html_name(self):
        if self.violence_monopoly:
            template = '◼{name}◼'
        elif self.leaded_organizations.filter(violence_monopoly=True).exists():
            template = '◢{name}◣'
        elif self.is_part_of_violence_monopoly():
            template = '▼{name}▼'
        elif self.leaded_organizations.exists():
            template = '◮{name}◭'
        elif not self.owner:
            template = '◨{name}◧'
        else:
            template = '◞{name}◟'

        return template.format(
            name=escape(self.name),
            color=escape(self.color)
        )

    def get_html_link(self):
        return '{}'.format(self.get_absolute_url(), self.get_html_name())

class PositionElection(models.Model):
    position = models.ForeignKey(Organization)
    turn = models.IntegerField()
    closed = models.BooleanField(default=False)
    winner = models.ForeignKey('PositionCandidate', blank=True, null=True)

class PositionCandidate(models.Model):
    class Meta:
        unique_together = (
            ("election", "candidate"),
        )
    election = models.ForeignKey(PositionElection)
    candidate = models.ForeignKey(Character)
    description = models.TextField()

class PositionElectionVote(models.Model):
    class Meta:
        unique_together = (
            ("election", "voter"),
        )
    election = models.ForeignKey(PositionElection)
    voter = models.ForeignKey(Character)
    candidate = models.ForeignKey(PositionCandidate, blank=True, null=True)

class Capability(models.Model):
    BAN = 'ban'
    POLICY_DOCUMENT = 'policy'
    CONSCRIPT = 'conscript'
    DIPLOMACY = 'diplomacy'
    DISSOLVE = 'dissolve'
    MANAGE_SUBORGANIZATIONS = 'suborganizations'
    SECEDE = 'secede'
    MEMBERSHIPS = 'memberships'
    HEIR = 'heir'
    ELECT = 'elect'
    CANDIDACY = 'candidacy'
    CONVOKE_ELECTIONS = 'convoke elections'

    TYPE_CHOICES = (
        (BAN, 'ban'),
        (POLICY_DOCUMENT, 'write policy and law'),
        (DIPLOMACY, 'diplomacy'),
        (CONSCRIPT, 'conscript troops'),
        (DISSOLVE, 'dissolve'),
        (MANAGE_SUBORGANIZATIONS, 'manage subordinate organizations'),
        (SECEDE, 'secede'),
        (MEMBERSHIPS, 'memberships'),
        (HEIR, 'set heir'),
        (ELECT, 'elect'),
        (CANDIDACY, 'present candidacy'),
        (CONVOKE_ELECTIONS, 'convoke elections'),
    )

    organization = models.ForeignKey(Organization)
    type = models.CharField(max_length=20, choices=TYPE_CHOICES)
    applying_to = models.ForeignKey(Organization, related_name='capabilities_to_this')
    stemming_from = models.ForeignKey('Capability', null=True, blank=True, related_name='transfers')

    def get_absolute_url(self):
        return reverse('organization:capability', kwargs={'capability_id': self.id})

    def create_proposal(self, character, proposal_dict):
        proposal = CapabilityProposal.objects.create(
            proposing_character=character,
            capability=self,
            proposal_json=json.dumps(proposal_dict),
            vote_end_turn=self.organization.world.current_turn + 2,
        )
        if self.organization.is_position or self.organization.decision_taking == Organization.DISTRIBUTED:
            proposal.execute()
        else:
            proposal.issue_vote(character, CapabilityVote.YEA)

    def is_passive(self):
        return self.type in (self.CONSCRIPT, )

class CapabilityProposal(models.Model):
    proposing_character = models.ForeignKey(Character)
    capability = models.ForeignKey(Capability)
    proposal_json = models.TextField()
    vote_end_turn = models.IntegerField()
    executed = models.BooleanField(default=False)
    closed = models.BooleanField(default=False)

    def execute(self):
        proposal = json.loads(self.proposal_json)
        if self.capability.type == Capability.POLICY_DOCUMENT:
            if proposal['new']:
                document = PolicyDocument(organization=self.capability.applying_to)
            else:
                document = PolicyDocument.objects.get(id=proposal['document_id'])

            if proposal['delete']:
                document.delete()
            else:
                document.title = proposal['title']
                document.body = proposal['body']
                document.public = 'public' in proposal.keys()
                document.last_modified_turn = self.capability.organization.world.current_turn
                document.save()

        elif self.capability.type == Capability.BAN:
            character_to_ban = Character.objects.get(id=proposal['character_id'])
            self.capability.applying_to.character_members.remove(character_to_ban)

            leader_organization = self.capability.applying_to.leader
            if leader_organization and character_to_ban in leader_organization.character_members.all():
                leader_organization.character_members.remove(character_to_ban)

        elif self.capability.type == Capability.CONVOKE_ELECTIONS:
            if self.capability.applying_to.current_election is None:
                months_to_election = proposal['months_to_election']
                self.capability.applying_to.convoke_elections(months_to_election)

        else:
            raise Exception("Executing unknown capability type '{}'".format(self.capability.type))

        self.executed = True
        self.closed = True
        self.save()

    def issue_vote(self, character, vote):
        CapabilityVote.objects.create(
            proposal=self,
            voter=character,
            vote=vote
        )
        self.execute_if_enough_votes()

    def delete_disallowed_votes(self):
        for vote in self.capabilityvote_set.all():
            if not self.capability.organization.character_is_member(vote.voter):
                vote.delete()

    def execute_if_enough_votes(self):
        self.delete_disallowed_votes()
        possible_votes = self.votes_possible()
        if self.votes_yea().count() > possible_votes / 2:
            self.execute()

    def votes_possible(self):
        return self.capability.organization.character_members.count()

    def execute_if_majority(self):
        self.delete_disallowed_votes()
        if self.votes_yea().count() > self.votes_nay().count():
            self.execute()

    def votes_yea(self):
        return self.capabilityvote_set.filter(vote=CapabilityVote.YEA)

    def votes_nay(self):
        return self.capabilityvote_set.filter(vote=CapabilityVote.NAY)

    def votes_invalid(self):
        return self.capabilityvote_set.filter(vote=CapabilityVote.INVALID)

    def get_absolute_url(self):
        return reverse('organization:proposal', kwargs={'proposal_id': self.id})

class CapabilityVote(models.Model):
    YEA = 'yea'
    NAY = 'nay'
    INVALID = 'invalid'
    VOTE_CHOICES = (
        (YEA, YEA),
        (NAY, NAY),
        (INVALID, INVALID),
    )

    proposal = models.ForeignKey(CapabilityProposal)
    voter = models.ForeignKey(Character)
    vote = models.CharField(max_length=10, choices=VOTE_CHOICES)

class PolicyDocument(models.Model):
    organization = models.ForeignKey(Organization)
    parent = models.ForeignKey('PolicyDocument', related_name='children', null=True, blank=True)
    public = models.BooleanField(default=False)
    title = models.TextField(max_length=100)
    body = models.TextField()
    last_modified_turn = models.IntegerField()

    def __str__(self):
        return self.title

    def get_absolute_url(self):
        return reverse('organization:document', kwargs={'document_id': self.id})
<commit_msg>Use glyphicons instead of UTF-8 for organization HTML names + refactor<commit_after>import json

from django.db import models, transaction
from django.contrib.auth.models import User
from django.urls.base import reverse
from django.utils.html import escape

from world.models import Tile, Character

class Organization(models.Model):
    DEMOCRATIC = 'democratic'  # decisions are voted among members
    DISTRIBUTED = 'distributed'  # decisions can be taken by each member
    DECISION_TAKING_CHOICES = (
        (DEMOCRATIC, DEMOCRATIC),
        (DISTRIBUTED, DISTRIBUTED),
    )

    CHARACTER = 'character'
    ORGANIZATION = 'organization'
    MEMBERSHIP_TYPE_CHOICES = (
        (CHARACTER, CHARACTER),
        (ORGANIZATION, ORGANIZATION),
    )

    INHERITED = 'inherited'
    ELECTED = 'elected'
    POSITION_TYPE_CHOICES = (
        (INHERITED, INHERITED),
        (ELECTED, ELECTED),
    )

    world = models.ForeignKey('world.World')
    name = models.CharField(max_length=100)
    color = models.CharField(max_length=6, default="FFFFFF", help_text="Format: RRGGBB (hex)")
    description = models.TextField()
    is_position = models.BooleanField()
    position_type = models.CharField(max_length=15, choices=POSITION_TYPE_CHOICES, blank=True, default='')
    owner = models.ForeignKey('Organization', null=True, blank=True, related_name='owned_organizations')
    leader = models.ForeignKey('Organization', null=True, blank=True, related_name='leaded_organizations')
    owner_and_leader_locked = models.BooleanField(
        help_text="If set, this organization will have always the same leader as it's owner."
    )
    violence_monopoly = models.BooleanField(default=False)
    decision_taking = models.CharField(max_length=15, choices=DECISION_TAKING_CHOICES)
    membership_type = models.CharField(max_length=15, choices=MEMBERSHIP_TYPE_CHOICES)
    character_members = models.ManyToManyField('world.Character', blank=True)
    organization_members = models.ManyToManyField('Organization', blank=True)
    election_period_months = models.IntegerField(default=0)
    current_election = models.ForeignKey('PositionElection', blank=True, null=True, related_name='+')
    last_election = models.ForeignKey('PositionElection', blank=True, null=True, related_name='+')
    heir_first = models.ForeignKey(Character, blank=True, null=True, related_name='first_heir_to')
    heir_second = models.ForeignKey(Character, blank=True, null=True, related_name='second_heir_to')

    def get_all_descendants(self, including_self=False):
        descendants = list(self.owned_organizations.all())
        if including_self:
            descendants.append(self)
        for child in descendants:
            descendants += child.get_all_descendants()
        return descendants

    def get_membership_including_descendants(self):
        members = list(self.character_members.all())
        for child in self.owned_organizations.all():
            members += child.get_membership_including_descendants()
        return members

    def character_can_use_capabilities(self, character):
        if character in self.character_members.all():
            return True
        for member_organization in self.organization_members.all():
            if (
                            member_organization.leader and
                            member_organization.leader.is_position and
                            character in member_organization.organization_members.all()
            ):
                return True
            if member_organization.is_position and character in member_organization.organization_members.all():
                return True
        return False

    def organizations_character_can_apply_capabilities_to_this_with(self, character, capability_type):
        result = []
        capabilities = Capability.objects.filter(applying_to=self, type=capability_type)
        for capability in capabilities:
            if capability.organization.character_is_member(character):
                result.append(capability.organization)
        return result

    def character_is_member(self, character):
        if character in self.character_members.all():
            return True
        for member_organization in self.organization_members.all():
            if character in member_organization.member_set.all():
                return True
        return False

    def is_part_of_violence_monopoly(self):
        return (
            True if self.violence_monopoly
            else False if not self.owner
            else self.owner.is_part_of_violence_monopoly()
        )

    def get_open_proposals(self):
        return CapabilityProposal.objects.filter(capability__applying_to=self, closed=False)

    def get_all_controlled_tiles(self):
        return Tile.objects.filter(controlled_by__in=self.get_all_descendants(including_self=True)).all()

    def external_capabilities_to_this(self):
        return self.capabilities_to_this.exclude(organization=self)

    def get_position_ocuppier(self):
        if not self.is_position or not self.character_members.exists():
            return None
        return list(self.character_members.all())[0]

    @transaction.atomic
    def convoke_elections(self, months_to_election=6):
        if not self.is_position:
            raise Exception("Elections only work for positions")
        election = PositionElection.objects.create(
            position=self,
            turn=self.world.current_turn + months_to_election
        )
        self.current_election = election
        self.save()
        if self.get_position_ocuppier() is not None:
            PositionCandidate.objects.create(
                election=election,
                candidate=self.get_position_ocuppier(),
                description="Auto-generated candidacy for incumbent character."
            )

    def __str__(self):
        return self.name

    def get_absolute_url(self):
        return reverse('organization:view', kwargs={'organization_id': self.id})

    def get_html_name(self):
        template = '{name}'
        if self.violence_monopoly:
            icon = "tower"
        elif self.leaded_organizations.filter(violence_monopoly=True).exists():
            icon = "king"
        elif self.is_part_of_violence_monopoly():
            icon = "knight"
        elif self.leaded_organizations.exists():
            icon = "menu-up"
        elif not self.owner:
            icon = "triangle-top"
        else:
            icon = "option-vertical"

        return template.format(
            name=escape(self.name),
            icon=icon,
            color=escape(self.color)
        )

    def get_html_link(self):
        return '{}'.format(self.get_absolute_url(), self.get_html_name())

class PositionElection(models.Model):
    position = models.ForeignKey(Organization)
    turn = models.IntegerField()
    closed = models.BooleanField(default=False)
    winner = models.ForeignKey('PositionCandidate', blank=True, null=True)

class PositionCandidate(models.Model):
    class Meta:
        unique_together = (
            ("election", "candidate"),
        )
    election = models.ForeignKey(PositionElection)
    candidate = models.ForeignKey(Character)
    description = models.TextField()

class PositionElectionVote(models.Model):
    class Meta:
        unique_together = (
            ("election", "voter"),
        )
    election = models.ForeignKey(PositionElection)
    voter = models.ForeignKey(Character)
    candidate = models.ForeignKey(PositionCandidate, blank=True, null=True)

class Capability(models.Model):
    BAN = 'ban'
    POLICY_DOCUMENT = 'policy'
    CONSCRIPT = 'conscript'
    DIPLOMACY = 'diplomacy'
    DISSOLVE = 'dissolve'
    MANAGE_SUBORGANIZATIONS = 'suborganizations'
    SECEDE = 'secede'
    MEMBERSHIPS = 'memberships'
    HEIR = 'heir'
    ELECT = 'elect'
    CANDIDACY = 'candidacy'
    CONVOKE_ELECTIONS = 'convoke elections'

    TYPE_CHOICES = (
        (BAN, 'ban'),
        (POLICY_DOCUMENT, 'write policy and law'),
        (DIPLOMACY, 'diplomacy'),
        (CONSCRIPT, 'conscript troops'),
        (DISSOLVE, 'dissolve'),
        (MANAGE_SUBORGANIZATIONS, 'manage subordinate organizations'),
        (SECEDE, 'secede'),
        (MEMBERSHIPS, 'memberships'),
        (HEIR, 'set heir'),
        (ELECT, 'elect'),
        (CANDIDACY, 'present candidacy'),
        (CONVOKE_ELECTIONS, 'convoke elections'),
    )

    organization = models.ForeignKey(Organization)
    type = models.CharField(max_length=20, choices=TYPE_CHOICES)
    applying_to = models.ForeignKey(Organization, related_name='capabilities_to_this')
    stemming_from = models.ForeignKey('Capability', null=True, blank=True, related_name='transfers')

    def get_absolute_url(self):
        return reverse('organization:capability', kwargs={'capability_id': self.id})

    def create_proposal(self, character, proposal_dict):
        proposal = CapabilityProposal.objects.create(
            proposing_character=character,
            capability=self,
            proposal_json=json.dumps(proposal_dict),
            vote_end_turn=self.organization.world.current_turn + 2,
        )
        if self.organization.is_position or self.organization.decision_taking == Organization.DISTRIBUTED:
            proposal.execute()
        else:
            proposal.issue_vote(character, CapabilityVote.YEA)

    def is_passive(self):
        return self.type in (self.CONSCRIPT, )

class CapabilityProposal(models.Model):
    proposing_character = models.ForeignKey(Character)
    capability = models.ForeignKey(Capability)
    proposal_json = models.TextField()
    vote_end_turn = models.IntegerField()
    executed = models.BooleanField(default=False)
    closed = models.BooleanField(default=False)

    def execute(self):
        proposal = json.loads(self.proposal_json)
        if self.capability.type == Capability.POLICY_DOCUMENT:
            if proposal['new']:
                document = PolicyDocument(organization=self.capability.applying_to)
            else:
                document = PolicyDocument.objects.get(id=proposal['document_id'])

            if proposal['delete']:
                document.delete()
            else:
                document.title = proposal['title']
                document.body = proposal['body']
                document.public = 'public' in proposal.keys()
                document.last_modified_turn = self.capability.organization.world.current_turn
                document.save()

        elif self.capability.type == Capability.BAN:
            character_to_ban = Character.objects.get(id=proposal['character_id'])
            self.capability.applying_to.character_members.remove(character_to_ban)

            leader_organization = self.capability.applying_to.leader
            if leader_organization and character_to_ban in leader_organization.character_members.all():
                leader_organization.character_members.remove(character_to_ban)

        elif self.capability.type == Capability.CONVOKE_ELECTIONS:
            if self.capability.applying_to.current_election is None:
                months_to_election = proposal['months_to_election']
                self.capability.applying_to.convoke_elections(months_to_election)

        else:
            raise Exception("Executing unknown capability type '{}'".format(self.capability.type))

        self.executed = True
        self.closed = True
        self.save()

    def issue_vote(self, character, vote):
        CapabilityVote.objects.create(
            proposal=self,
            voter=character,
            vote=vote
        )
        self.execute_if_enough_votes()

    def delete_disallowed_votes(self):
        for vote in self.capabilityvote_set.all():
            if not self.capability.organization.character_is_member(vote.voter):
                vote.delete()

    def execute_if_enough_votes(self):
        self.delete_disallowed_votes()
        possible_votes = self.votes_possible()
        if self.votes_yea().count() > possible_votes / 2:
            self.execute()

    def votes_possible(self):
        return self.capability.organization.character_members.count()

    def execute_if_majority(self):
        self.delete_disallowed_votes()
        if self.votes_yea().count() > self.votes_nay().count():
            self.execute()

    def votes_yea(self):
        return self.capabilityvote_set.filter(vote=CapabilityVote.YEA)

    def votes_nay(self):
        return self.capabilityvote_set.filter(vote=CapabilityVote.NAY)

    def votes_invalid(self):
        return self.capabilityvote_set.filter(vote=CapabilityVote.INVALID)

    def get_absolute_url(self):
        return reverse('organization:proposal', kwargs={'proposal_id': self.id})

class CapabilityVote(models.Model):
    YEA = 'yea'
    NAY = 'nay'
    INVALID = 'invalid'
    VOTE_CHOICES = (
        (YEA, YEA),
        (NAY, NAY),
        (INVALID, INVALID),
    )

    proposal = models.ForeignKey(CapabilityProposal)
    voter = models.ForeignKey(Character)
    vote = models.CharField(max_length=10, choices=VOTE_CHOICES)

class PolicyDocument(models.Model):
    organization = models.ForeignKey(Organization)
    parent = models.ForeignKey('PolicyDocument', related_name='children', null=True, blank=True)
    public = models.BooleanField(default=False)
    title = models.TextField(max_length=100)
    body = models.TextField()
    last_modified_turn = models.IntegerField()

    def __str__(self):
        return self.title

    def get_absolute_url(self):
        return reverse('organization:document', kwargs={'document_id': self.id})
<|endoftext|>


<commit_before># coding: utf-8
import pandas as pd

def dimension_levels(dimension_key, dimension):
    if 'display_field' in dimension:
        return [dimension_key, dimension['display_field']]
    return [dimension_key]

def correct_dimension_level_order(dataframe, display_schema):
    if isinstance(dataframe.index, pd.MultiIndex):
        dimension_orders = [order
                            for key, dimension in display_schema['dimensions'].items()
                            for order in dimension_levels(key, dimension)]

        dataframe = dataframe.reorder_levels(dataframe.index.names.index(level)
                                             for level in dimension_orders)

    return dataframe[list(display_schema['metrics'].keys())]
<commit_msg>Fixed reordering the columns when references are used<commit_after># coding: utf-8
import itertools

import pandas as pd

def dimension_levels(dimension_key, dimension):
    if 'display_field' in dimension:
        return [dimension_key, dimension['display_field']]
    return [dimension_key]

def correct_dimension_level_order(dataframe, display_schema):
    if isinstance(dataframe.index, pd.MultiIndex):
        dimension_orders = [order
                            for key, dimension in display_schema['dimensions'].items()
                            for order in dimension_levels(key, dimension)]

        dataframe = dataframe.reorder_levels(dataframe.index.names.index(level)
                                             for level in dimension_orders)

    metrics = list(display_schema['metrics'])
    if display_schema.get('references'):
        references = [''] + list(display_schema['references'])
        return dataframe[list(itertools.product(references, metrics))]

    return dataframe[metrics]
<|endoftext|>


<commit_before># encoding: utf-8
# pylint: disable=missing-docstring
import logging

from six import itervalues

from flask_marshmallow import Schema, base_fields
from marshmallow import validate, validates_schema, ValidationError

log = logging.getLogger(__name__)  # pylint: disable=invalid-name

class Parameters(Schema):

    def __init__(self, **kwargs):
        super(Parameters, self).__init__(strict=True, **kwargs)

    def __contains__(self, field):
        return field in self.fields

    def make_instance(self, data):
        # pylint: disable=unused-argument
        """
        This is a no-op function which shadows ``ModelSchema.make_instance``
        method (when inherited classes inherit from ``ModelSchema``). Thus, we
        avoid a new instance creation because it is undesirable behaviour for
        parameters (they can be used not only for saving new instances).
        """
        return

class PostFormParameters(Parameters):

    def __init__(self, *args, **kwargs):
        super(PostFormParameters, self).__init__(*args, **kwargs)
        for field in itervalues(self.fields):
            if field.dump_only:
                continue
            field.metadata['location'] = 'form'

class PatchJSONParameters(Parameters):
    """
    Base parameters class for handling PATCH arguments according to RFC 6902.
    """

    # All operations described in RFC 6902
    OP_ADD = 'add'
    OP_REMOVE = 'remove'
    OP_REPLACE = 'replace'
    OP_MOVE = 'move'
    OP_COPY = 'copy'
    OP_TEST = 'test'

    # However, we use only those which make sense in RESTful API
    OPERATION_CHOICES = (
        OP_TEST,
        OP_ADD,
        OP_REMOVE,
        OP_REPLACE,
    )
    op = base_fields.String(required=True)  # pylint: disable=invalid-name

    PATH_CHOICES = None

    path = base_fields.String(required=True)

    NO_VALUE_OPERATIONS = (OP_REMOVE,)

    value = base_fields.Raw(required=False)

    def __init__(self, *args, **kwargs):
        super(PatchJSONParameters, self).__init__(*args, many=True, **kwargs)
        if not self.PATH_CHOICES:
            raise ValueError("%s.PATH_CHOICES has to be set" % self.__class__.__name__)
        # Make a copy of `validators` as otherwise we will modify the behaviour
        # of all `marshmallow.Schema`-based classes
        self.fields['op'].validators = \
            self.fields['op'].validators + [validate.OneOf(self.OPERATION_CHOICES)]
        self.fields['path'].validators = \
            self.fields['path'].validators + [validate.OneOf(self.PATH_CHOICES)]

    @validates_schema
    def validate_patch_structure(self, data):
        """
        Common validation of PATCH structure

        Provide check that 'value' present in all operations expect it.

        Provide check if 'path' is present. 'path' can be absent if provided
        without '/' at the start. Supposed that if 'path' is present than it
        is prepended with '/'.
        Removing '/' in the beginning to simplify usage in resource.
        """
        if data['op'] not in self.NO_VALUE_OPERATIONS and 'value' not in data:
            raise ValidationError('value is required')

        if 'path' not in data:
            raise ValidationError('Path is required and must always begin with /')
        else:
            data['field_name'] = data['path'][1:]

    @classmethod
    def perform_patch(cls, operations, obj, state=None):
        """
        Performs all necessary operations by calling class methods with corresponding names
        """
        for operation in operations:
            if not cls._process_patch_operation(operation, obj=obj, state=state):
                log.info("%s patching has stopped because of unknown operation %s", (obj.__name__, operation))
                raise ValidationError("Failed to update %s details." % obj.__name__)
        return True

    @classmethod
    def _process_patch_operation(cls, operation, obj, state=None):
        """
        Args:
            operation (dict) - one patch operation in RFC 6902 format.
            obj (Team) - team instance which is needed to be patched.
            state (dict) - inter-operations state storage

        Returns:
            processing_status (bool) - True if operation was handled, otherwise False.
        """
        field_operaion = operation['op']

        if field_operaion == cls.OP_REPLACE:
            return cls.replace(obj, operation['field_name'], operation['value'], state=state)

        elif field_operaion == cls.OP_TEST:
            return cls.test(obj, operation['field_name'], operation['value'], state=state)

        elif field_operaion == cls.OP_ADD:
            return cls.add(obj, operation['field_name'], operation['value'], state=state)

        elif field_operaion == cls.OP_MOVE:
            return cls.move(obj, operation['field_name'], operation['value'], state=state)

        elif field_operaion == cls.OP_COPY:
            return cls.copy(obj, operation['field_name'], operation['value'], state=state)

        elif field_operaion == cls.OP_REMOVE:
            return cls.remove(obj, operation['field_name'], state=state)

        return False

    @classmethod
    def replace(cls, obj, field, value, state=None):
        """
        This is method for replace operation. It is separated to have a possibility to easily
        override it in your Parameters

        Args:
            obj: instance to change
            field: (str) name
            value: (str) value
            state: (dict) inter-operations state storage

        Returns:
            processing_status: (bool) - True
        """
        setattr(obj, field, value)
        return True

    @classmethod
    def test(cls, obj, field, value, state=None):
        """
        This is method for test operation. It is separated to have a possibility to easily
        override it in your Parameters

        Args:
            obj: instance to change
            field: (str) name
            value: (str) value
            state: (dict) inter-operations state storage

        Returns:
            processing_status: (bool) - True
        """
        if getattr(obj, field) == value:
            return True
        else:
            return False

    @classmethod
    def add(cls, obj, field, value, state=None):
        # pylint: disable=abstract-method
        raise NotImplementedError()

    @classmethod
    def remove(cls, obj, field, state=None):
        # pylint: disable=abstract-method
        raise NotImplementedError()

    @classmethod
    def move(cls, obj, field, value, state=None):
        # pylint: disable=abstract-method
        raise NotImplementedError()

    @classmethod
    def copy(cls, obj, field, value, state=None):
        # pylint: disable=abstract-method
        raise NotImplementedError()
<commit_msg>Fixed PostFormParameters implementation to avoid unnecessary location override<commit_after># encoding: utf-8
# pylint: disable=missing-docstring
import logging

from six import itervalues

from flask_marshmallow import Schema, base_fields
from marshmallow import validate, validates_schema, ValidationError

log = logging.getLogger(__name__)  # pylint: disable=invalid-name

class Parameters(Schema):

    def __init__(self, **kwargs):
        super(Parameters, self).__init__(strict=True, **kwargs)

    def __contains__(self, field):
        return field in self.fields

    def make_instance(self, data):
        # pylint: disable=unused-argument
        """
        This is a no-op function which shadows ``ModelSchema.make_instance``
        method (when inherited classes inherit from ``ModelSchema``). Thus, we
        avoid a new instance creation because it is undesirable behaviour for
        parameters (they can be used not only for saving new instances).
        """
        return

class PostFormParameters(Parameters):

    def __init__(self, *args, **kwargs):
        super(PostFormParameters, self).__init__(*args, **kwargs)
        for field in itervalues(self.fields):
            if field.dump_only:
                continue
            if not field.metadata.get('location'):
                field.metadata['location'] = 'form'

class PatchJSONParameters(Parameters):
    """
    Base parameters class for handling PATCH arguments according to RFC 6902.
    """

    # All operations described in RFC 6902
    OP_ADD = 'add'
    OP_REMOVE = 'remove'
    OP_REPLACE = 'replace'
    OP_MOVE = 'move'
    OP_COPY = 'copy'
    OP_TEST = 'test'

    # However, we use only those which make sense in RESTful API
    OPERATION_CHOICES = (
        OP_TEST,
        OP_ADD,
        OP_REMOVE,
        OP_REPLACE,
    )
    op = base_fields.String(required=True)  # pylint: disable=invalid-name

    PATH_CHOICES = None

    path = base_fields.String(required=True)

    NO_VALUE_OPERATIONS = (OP_REMOVE,)

    value = base_fields.Raw(required=False)

    def __init__(self, *args, **kwargs):
        super(PatchJSONParameters, self).__init__(*args, many=True, **kwargs)
        if not self.PATH_CHOICES:
            raise ValueError("%s.PATH_CHOICES has to be set" % self.__class__.__name__)
        # Make a copy of `validators` as otherwise we will modify the behaviour
        # of all `marshmallow.Schema`-based classes
        self.fields['op'].validators = \
            self.fields['op'].validators + [validate.OneOf(self.OPERATION_CHOICES)]
        self.fields['path'].validators = \
            self.fields['path'].validators + [validate.OneOf(self.PATH_CHOICES)]

    @validates_schema
    def validate_patch_structure(self, data):
        """
        Common validation of PATCH structure

        Provide check that 'value' present in all operations expect it.

        Provide check if 'path' is present. 'path' can be absent if provided
        without '/' at the start. Supposed that if 'path' is present than it
        is prepended with '/'.
        Removing '/' in the beginning to simplify usage in resource.
        """
        if data['op'] not in self.NO_VALUE_OPERATIONS and 'value' not in data:
            raise ValidationError('value is required')

        if 'path' not in data:
            raise ValidationError('Path is required and must always begin with /')
        else:
            data['field_name'] = data['path'][1:]

    @classmethod
    def perform_patch(cls, operations, obj, state=None):
        """
        Performs all necessary operations by calling class methods with corresponding names
        """
        for operation in operations:
            if not cls._process_patch_operation(operation, obj=obj, state=state):
                log.info("%s patching has stopped because of unknown operation %s", (obj.__name__, operation))
                raise ValidationError("Failed to update %s details." % obj.__name__)
        return True

    @classmethod
    def _process_patch_operation(cls, operation, obj, state=None):
        """
        Args:
            operation (dict) - one patch operation in RFC 6902 format.
            obj (Team) - team instance which is needed to be patched.
            state (dict) - inter-operations state storage

        Returns:
            processing_status (bool) - True if operation was handled, otherwise False.
        """
        field_operaion = operation['op']

        if field_operaion == cls.OP_REPLACE:
            return cls.replace(obj, operation['field_name'], operation['value'], state=state)

        elif field_operaion == cls.OP_TEST:
            return cls.test(obj, operation['field_name'], operation['value'], state=state)

        elif field_operaion == cls.OP_ADD:
            return cls.add(obj, operation['field_name'], operation['value'], state=state)

        elif field_operaion == cls.OP_MOVE:
            return cls.move(obj, operation['field_name'], operation['value'], state=state)

        elif field_operaion == cls.OP_COPY:
            return cls.copy(obj, operation['field_name'], operation['value'], state=state)

        elif field_operaion == cls.OP_REMOVE:
            return cls.remove(obj, operation['field_name'], state=state)

        return False

    @classmethod
    def replace(cls, obj, field, value, state=None):
        """
        This is method for replace operation. It is separated to have a possibility to easily
        override it in your Parameters

        Args:
            obj: instance to change
            field: (str) name
            value: (str) value
            state: (dict) inter-operations state storage

        Returns:
            processing_status: (bool) - True
        """
        setattr(obj, field, value)
        return True

    @classmethod
    def test(cls, obj, field, value, state=None):
        """
        This is method for test operation. It is separated to have a possibility to easily
        override it in your Parameters

        Args:
            obj: instance to change
            field: (str) name
            value: (str) value
            state: (dict) inter-operations state storage

        Returns:
            processing_status: (bool) - True
        """
        if getattr(obj, field) == value:
            return True
        else:
            return False

    @classmethod
    def add(cls, obj, field, value, state=None):
        # pylint: disable=abstract-method
        raise NotImplementedError()

    @classmethod
    def remove(cls, obj, field, state=None):
        # pylint: disable=abstract-method
        raise NotImplementedError()

    @classmethod
    def move(cls, obj, field, value, state=None):
        # pylint: disable=abstract-method
        raise NotImplementedError()

    @classmethod
    def copy(cls, obj, field, value, state=None):
        # pylint: disable=abstract-method
        raise NotImplementedError()
<|endoftext|>