<commit_before>#!/usr/bin/python -tt
# vim:set ts=4 sw=4 expandtab:
#
# $Id$
# $URL$
#
# NodeManager plugin for installing SFA GID's in slivers
#

import os
import sys
sys.path.append('/usr/share/NodeManager')
import logger
import traceback
try:
    from sfa.util.namespace import *
    from sfa.util.config import Config
    import sfa.util.xmlrpcprotocol as xmlrpcprotocol
    from sfa.trust.certificate import Keypair, Certificate
    from sfa.trust.credential import Credential
    from sfa.trust.gid import GID
    from sfa.trust.hierarchy import Hierarchy
    from sfa.plc.api import ComponentAPI
    sfa = True
except:
    sfa = None

def start():
    logger.log("sfagid: plugin starting up ...")
    if not sfa:
        return
    keyfile, certfile = get_keypair(None)
    api = ComponentAPI(key_file=keyfile, cert_file=certfile)
    api.get_node_key()

def GetSlivers(data, config=None, plc=None):
    if not sfa:
        return

    keyfile, certfile = get_keypair(config)
    api = ComponentAPI(key_file=keyfile, cert_file=certfile)
    slivers = [sliver['name'] for sliver in data['slivers']]
    install_gids(api, slivers)
    install_trusted_certs(api)

def install_gids(api, slivers):
    # install node gid
    node_gid_file = api.config.config_path + os.sep + "node.gid"
    node_gid = GID(filename=node_gid_file)
    node_gid_str = node_gid.save_to_string(save_parents=True)
    node_hrn = node_gid.get_hrn()

    # get currently installed slice and node gids
    interface_hrn = api.config.SFA_INTERFACE_HRN
    slice_gids = {}
    node_gids = {}
    for slicename in slivers:
        slice_gid_filename = "/vservers/%s/etc/slice.gid" % slicename
        node_gid_filename = "/vservers/%s/etc/node.gid" % slicename
        if os.path.isfile(slice_gid_filename):
            gid_file = open(slice_gid_filename, 'r')
            slice_gids[sliver] = gid_file.read()
            gid_file.close()
        if os.path.isfile(node_gid_filename):
            gid_file = open(node_gid_filename, 'r')
            node_gids[sliver] = gid_file.read()
            gid_file.close()

    # convert slicenames to hrns
    hrns = [slicename_to_hrn(interface_hrn, slicename) \
            for slicename in slivers]

    # get current gids from registry
    cred = api.getCredential()
    registry = api.get_registry()
    #records = registry.GetGids(cred, hrns)
    records = registry.get_gids(cred, hrns)
    for record in records:
        # skip if this isnt a slice record
        if not record['type'] == 'slice':
            continue
        vserver_path = "/vservers/%(slicename)s" % locals()
        # skip if the slice isnt instantiated
        if not os.path.exists(vserver_path):
            continue

        # install slice gid if it doesnt already exist or has changed
        slice_gid_str = record['gid']
        slicename = hrn_to_pl_slicename(record['hrn'])
        if slicename not in slice_gids or slice_gids[slicename] != slice_gid_str:
            gid_filename = os.sep.join([vserver_path, "etc", "slice.gid"])
            GID(string=slice_gid_str).save_to_file(gid_filename, save_parents=True)

        # install slice gid if it doesnt already exist or has changed
        if slicename not in node_gids or node_gids[slicename] != node_gid_str:
            gid_filename = os.sep.join([vserver_path, "etc", "node.gid"])
            GID(string=node_gid_str).save_to_file(gid_filename, save_parents=True)

def install_trusted_certs(api):
    cred = api.getCredential()
    registry = api.get_registry()
    trusted_certs = registry.get_trusted_certs(cred)
    trusted_gid_names = []
    for gid_str in trusted_certs:
        gid = GID(string=gid_str)
        gid.decode()
        relative_filename = gid.get_hrn() + ".gid"
        trusted_gid_names.append(relative_filename)
        gid_filename = trusted_certs_dir + os.sep + relative_filename
        if verbose:
            print "Writing GID for %s as %s" % (gid.get_hrn(), gid_filename)
        gid.save_to_file(gid_filename, save_parents=True)

    # remove old certs
    all_gids_names = os.listdir(trusted_certs_dir)
    for gid_name in all_gids_names:
        if gid_name not in trusted_gid_names:
            if verbose:
                print "Removing old gid ", gid_name
            os.unlink(trusted_certs_dir + os.sep + gid_name)

def get_keypair(config = None):
    if not config:
        config = Config()
    hierarchy = Hierarchy()
    key_dir= hierarchy.basedir
    data_dir = config.data_path
    keyfile =data_dir + os.sep + "server.key"
    certfile = data_dir + os.sep + "server.cert"

    # check if files already exist
    if os.path.exists(keyfile) and os.path.exists(certfile):
        return (keyfile, certfile)

    # create server key and certificate
    key = Keypair(filename=node_pkey_file)
    cert = Certificate(subject=hrn)
    cert.set_issuer(key=key, subject=hrn)
    cert.set_pubkey(key)
    cert.sign()
    cert.save_to_file(certfile, save_parents=True)
    return (keyfile, certfile)

if __name__ == '__main__':
    test_slivers = {'slivers': [
        {'name': 'tmacktestslice', 'attributes': []}
        ]}
    start()
    GetSlivers(test_slivers)

<commit_msg>fix NameError in get_keypair<commit_after>#!/usr/bin/python -tt
# vim:set ts=4 sw=4 expandtab:
#
# $Id$
# $URL$
#
# NodeManager plugin for installing SFA GID's in slivers
#

import os
import sys
sys.path.append('/usr/share/NodeManager')
import logger
import traceback
try:
    from sfa.util.namespace import *
    from sfa.util.config import Config
    import sfa.util.xmlrpcprotocol as xmlrpcprotocol
    from sfa.trust.certificate import Keypair, Certificate
    from sfa.trust.credential import Credential
    from sfa.trust.gid import GID
    from sfa.trust.hierarchy import Hierarchy
    from sfa.plc.api import ComponentAPI
    sfa = True
except:
    sfa = None

def start():
    logger.log("sfagid: plugin starting up ...")
    if not sfa:
        return
    keyfile, certfile = get_keypair(None)
    api = ComponentAPI(key_file=keyfile, cert_file=certfile)
    api.get_node_key()

def GetSlivers(data, config=None, plc=None):
    if not sfa:
        return

    keyfile, certfile = get_keypair(config)
    api = ComponentAPI(key_file=keyfile, cert_file=certfile)
    slivers = [sliver['name'] for sliver in data['slivers']]
    install_gids(api, slivers)
    install_trusted_certs(api)

def install_gids(api, slivers):
    # install node gid
    node_gid_file = api.config.config_path + os.sep + "node.gid"
    node_gid = GID(filename=node_gid_file)
    node_gid_str = node_gid.save_to_string(save_parents=True)
    node_hrn = node_gid.get_hrn()

    # get currently installed slice and node gids
    interface_hrn = api.config.SFA_INTERFACE_HRN
    slice_gids = {}
    node_gids = {}
    for slicename in slivers:
        slice_gid_filename = "/vservers/%s/etc/slice.gid" % slicename
        node_gid_filename = "/vservers/%s/etc/node.gid" % slicename
        if os.path.isfile(slice_gid_filename):
            gid_file = open(slice_gid_filename, 'r')
            slice_gids[sliver] = gid_file.read()
            gid_file.close()
        if os.path.isfile(node_gid_filename):
            gid_file = open(node_gid_filename, 'r')
            node_gids[sliver] = gid_file.read()
            gid_file.close()

    # convert slicenames to hrns
    hrns = [slicename_to_hrn(interface_hrn, slicename) \
            for slicename in slivers]

    # get current gids from registry
    cred = api.getCredential()
    registry = api.get_registry()
    #records = registry.GetGids(cred, hrns)
    records = registry.get_gids(cred, hrns)
    for record in records:
        # skip if this isnt a slice record
        if not record['type'] == 'slice':
            continue
        vserver_path = "/vservers/%(slicename)s" % locals()
        # skip if the slice isnt instantiated
        if not os.path.exists(vserver_path):
            continue

        # install slice gid if it doesnt already exist or has changed
        slice_gid_str = record['gid']
        slicename = hrn_to_pl_slicename(record['hrn'])
        if slicename not in slice_gids or slice_gids[slicename] != slice_gid_str:
            gid_filename = os.sep.join([vserver_path, "etc", "slice.gid"])
            GID(string=slice_gid_str).save_to_file(gid_filename, save_parents=True)

        # install slice gid if it doesnt already exist or has changed
        if slicename not in node_gids or node_gids[slicename] != node_gid_str:
            gid_filename = os.sep.join([vserver_path, "etc", "node.gid"])
            GID(string=node_gid_str).save_to_file(gid_filename, save_parents=True)

def install_trusted_certs(api):
    cred = api.getCredential()
    registry = api.get_registry()
    trusted_certs = registry.get_trusted_certs(cred)
    trusted_gid_names = []
    for gid_str in trusted_certs:
        gid = GID(string=gid_str)
        gid.decode()
        relative_filename = gid.get_hrn() + ".gid"
        trusted_gid_names.append(relative_filename)
        gid_filename = trusted_certs_dir + os.sep + relative_filename
        if verbose:
            print "Writing GID for %s as %s" % (gid.get_hrn(), gid_filename)
        gid.save_to_file(gid_filename, save_parents=True)

    # remove old certs
    all_gids_names = os.listdir(trusted_certs_dir)
    for gid_name in all_gids_names:
        if gid_name not in trusted_gid_names:
            if verbose:
                print "Removing old gid ", gid_name
            os.unlink(trusted_certs_dir + os.sep + gid_name)

def get_keypair(config = None):
    if not config:
        config = Config()
    hierarchy = Hierarchy()
    key_dir= hierarchy.basedir
    data_dir = config.data_path
    keyfile =data_dir + os.sep + "server.key"
    certfile = data_dir + os.sep + "server.cert"

    # check if files already exist
    if os.path.exists(keyfile) and os.path.exists(certfile):
        return (keyfile, certfile)

    # create server key and certificate
    key = Keypair(filename=keyfile)
    cert = Certificate(subject=hrn)
    cert.set_issuer(key=key, subject=hrn)
    cert.set_pubkey(key)
    cert.sign()
    cert.save_to_file(certfile, save_parents=True)
    return (keyfile, certfile)

if __name__ == '__main__':
    test_slivers = {'slivers': [
        {'name': 'tmacktestslice', 'attributes': []}
        ]}
    start()
    GetSlivers(test_slivers)

<|endoftext|>


<commit_before># Copyright (c) 2013 Paul Tagliamonte <paultag@debian.org>
# Copyright (c) 2013 Julien Danjou <julien@danjou.info>
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import contextlib
import sys

if sys.version_info[0] >= 3:
    str_type = str
else:
    str_type = unicode

@contextlib.contextmanager
def temporary_attribute_value(obj, attribute, value):
    """Temporarily switch an object attribute value to another value."""
    original_value = getattr(obj, attribute)
    setattr(obj, attribute, value)

    try:
        yield
    except Exception:
        pass

    setattr(obj, attribute, original_value)

def flatten_literal_list(entry):
    for e in entry:
        if type(e) == list:
            for x in flatten_literal_list(e):
                yield x  # needs more yield-from
        else:
            yield e
<commit_msg>Revert wrong changes on temporary_attribute_value<commit_after># Copyright (c) 2013 Paul Tagliamonte <paultag@debian.org>
# Copyright (c) 2013 Julien Danjou <julien@danjou.info>
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import contextlib
import sys

if sys.version_info[0] >= 3:
    str_type = str
else:
    str_type = unicode

@contextlib.contextmanager
def temporary_attribute_value(obj, attribute, value):
    """Temporarily switch an object attribute value to another value."""
    original_value = getattr(obj, attribute)
    setattr(obj, attribute, value)
    yield
    setattr(obj, attribute, original_value)

def flatten_literal_list(entry):
    for e in entry:
        if type(e) == list:
            for x in flatten_literal_list(e):
                yield x  # needs more yield-from
        else:
            yield e
<|endoftext|>


<commit_before>"""
Django settings for running tests for Resolwe package.

"""
from __future__ import absolute_import, division, print_function, unicode_literals

import os
from distutils.util import strtobool  # pylint: disable=import-error,no-name-in-module

PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))

SECRET_KEY = 'secret'

# TODO: Support local Resolwe server in tests
RESOLWE_HOST_URL = 'https://qa2.genialis.com'

DEBUG = True

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.staticfiles',

    'channels',
    'rest_framework',
    'guardian',
    'mathfilters',
    'versionfield',

    'resolwe',
    'resolwe.permissions',
    'resolwe.flow',
    'resolwe.elastic',
    'resolwe.toolkit',
    'resolwe.test_helpers',

    'resolwe_bio',
    'resolwe_bio.kb',
)

ROOT_URLCONF = 'tests.urls'

TEST_RUNNER = 'resolwe.test_helpers.test_runner.ResolweRunner'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
            ],
        },
    },
]

AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
    'guardian.backends.ObjectPermissionBackend',
)

ANONYMOUS_USER_ID = -1

# Check if PostgreSQL settings are set via environment variables
pgname = os.environ.get('RESOLWE_POSTGRESQL_NAME', 'resolwe-bio')
pguser = os.environ.get('RESOLWE_POSTGRESQL_USER', 'resolwe')
pghost = os.environ.get('RESOLWE_POSTGRESQL_HOST', 'localhost')
pgport = int(os.environ.get('RESOLWE_POSTGRESQL_PORT', 55433))

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': pgname,
        'USER': pguser,
        'HOST': pghost,
        'PORT': pgport,
    }
}

STATIC_URL = '/static/'

REDIS_CONNECTION = {
    'host': 'localhost',
    'port': int(os.environ.get('RESOLWE_REDIS_PORT', 56380)),
    'db': int(os.environ.get('RESOLWE_REDIS_DATABASE', 0)),
}

FLOW_EXECUTOR = {
    'NAME': 'resolwe.flow.executors.docker',
    # XXX: Change to a stable resolwe image when it will include all the required tools
    'CONTAINER_IMAGE': 'resolwe/bio-linux8-resolwe-preview',
    'CONTAINER_NAME_PREFIX': 'resolwebio',
    'REDIS_CONNECTION': REDIS_CONNECTION,
    'DATA_DIR': os.path.join(PROJECT_ROOT, 'test_data'),
    'UPLOAD_DIR': os.path.join(PROJECT_ROOT, 'test_upload'),
    'RUNTIME_DIR': os.path.join(PROJECT_ROOT, 'test_runtime'),
}
# Set custom executor command if set via environment variable
if 'RESOLWE_DOCKER_COMMAND' in os.environ:
    FLOW_DOCKER_COMMAND = os.environ['RESOLWE_DOCKER_COMMAND']
FLOW_API = {
    'PERMISSIONS': 'resolwe.permissions.permissions',
}
FLOW_EXPRESSION_ENGINES = [
    {
        'ENGINE': 'resolwe.flow.expression_engines.jinja',
        'CUSTOM_FILTERS': [
            'resolwe_bio.expression_filters.sample',
        ]
    },
]
FLOW_EXECUTION_ENGINES = [
    'resolwe.flow.execution_engines.bash',
    'resolwe.flow.execution_engines.workflow',
]

# Check if any Manager settings are set via environment variables
manager_prefix = os.environ.get('RESOLWE_MANAGER_REDIS_PREFIX', 'resolwe-bio.manager')
FLOW_MANAGER = {
    'NAME': 'resolwe.flow.managers.workload_connectors.local',
    'REDIS_PREFIX': manager_prefix,
    'REDIS_CONNECTION': REDIS_CONNECTION,
}

FLOW_DOCKER_VOLUME_EXTRA_OPTIONS = {
    'data': 'Z',
    'data_all': 'z',
    'upload': 'z',
    'secrets': 'Z',
    'users': 'Z',
    'tools': 'z',
}

# Don't pull Docker images if set via the environment variable.
FLOW_DOCKER_DONT_PULL = strtobool(os.environ.get('RESOLWE_DOCKER_DONT_PULL', '0'))

# Ensure all container images follow a specific format.
FLOW_CONTAINER_VALIDATE_IMAGE = r'.+:(?!latest)'

REST_FRAMEWORK = {
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'rest_framework.authentication.SessionAuthentication',
    ),
    'DEFAULT_FILTER_BACKENDS': (
        'resolwe.permissions.filters.ResolwePermissionsFilter',
        'rest_framework_filters.backends.DjangoFilterBackend',
    ),
}

FLOW_PROCESSES_FINDERS = (
    'resolwe.flow.finders.FileSystemProcessesFinder',
    'resolwe.flow.finders.AppDirectoriesFinder',
)

FLOW_PROCESSES_DIRS = (os.path.join(PROJECT_ROOT, '../resolwe_bio/tests/'),)

# Do not skip tests that fail on Docker executor if this is set via environment
# variable
if os.environ.get('RESOLWEBIO_TESTS_SKIP_DOCKER_FAILURES', '').lower() in ["no", "false"]:
    TESTS_SKIP_DOCKER_FAILURES = False

# Elastic Search.

ELASTICSEARCH_HOST = os.environ.get('RESOLWE_ES_HOST', 'localhost')
ELASTICSEARCH_PORT = int(os.environ.get('RESOLWE_ES_PORT', '59201'))

# Testing.

TEST_RUNNER = 'resolwe.test_helpers.test_runner.ResolweRunner'
TEST_PROCESS_REQUIRE_TAGS = True
# Don't profile unless set via the environment variable.
TEST_PROCESS_PROFILE = strtobool(os.environ.get('RESOLWE_TEST_PROCESS_PROFILE', '0'))

# Channels.

CHANNEL_LAYERS = {
    'default': {
        'BACKEND': 'asgi_redis.RedisChannelLayer',
        'ROUTING': 'tests.routing.channel_routing',
        'CONFIG': {
            'hosts': [(REDIS_CONNECTION['host'], REDIS_CONNECTION['port'])],
            'expiry': 3600,
        },
    },
}
<commit_msg>Set FLOW_PROCESS_MAX_CORES for test project<commit_after>"""
Django settings for running tests for Resolwe package.

"""
from __future__ import absolute_import, division, print_function, unicode_literals

import os
from distutils.util import strtobool  # pylint: disable=import-error,no-name-in-module

PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))

SECRET_KEY = 'secret'

# TODO: Support local Resolwe server in tests
RESOLWE_HOST_URL = 'https://qa2.genialis.com'

DEBUG = True

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.staticfiles',

    'channels',
    'rest_framework',
    'guardian',
    'mathfilters',
    'versionfield',

    'resolwe',
    'resolwe.permissions',
    'resolwe.flow',
    'resolwe.elastic',
    'resolwe.toolkit',
    'resolwe.test_helpers',

    'resolwe_bio',
    'resolwe_bio.kb',
)

ROOT_URLCONF = 'tests.urls'

TEST_RUNNER = 'resolwe.test_helpers.test_runner.ResolweRunner'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
            ],
        },
    },
]

AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
    'guardian.backends.ObjectPermissionBackend',
)

ANONYMOUS_USER_ID = -1

# Check if PostgreSQL settings are set via environment variables
pgname = os.environ.get('RESOLWE_POSTGRESQL_NAME', 'resolwe-bio')
pguser = os.environ.get('RESOLWE_POSTGRESQL_USER', 'resolwe')
pghost = os.environ.get('RESOLWE_POSTGRESQL_HOST', 'localhost')
pgport = int(os.environ.get('RESOLWE_POSTGRESQL_PORT', 55433))

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': pgname,
        'USER': pguser,
        'HOST': pghost,
        'PORT': pgport,
    }
}

STATIC_URL = '/static/'

REDIS_CONNECTION = {
    'host': 'localhost',
    'port': int(os.environ.get('RESOLWE_REDIS_PORT', 56380)),
    'db': int(os.environ.get('RESOLWE_REDIS_DATABASE', 0)),
}

FLOW_EXECUTOR = {
    'NAME': 'resolwe.flow.executors.docker',
    # XXX: Change to a stable resolwe image when it will include all the required tools
    'CONTAINER_IMAGE': 'resolwe/bio-linux8-resolwe-preview',
    'CONTAINER_NAME_PREFIX': 'resolwebio',
    'REDIS_CONNECTION': REDIS_CONNECTION,
    'DATA_DIR': os.path.join(PROJECT_ROOT, 'test_data'),
    'UPLOAD_DIR': os.path.join(PROJECT_ROOT, 'test_upload'),
    'RUNTIME_DIR': os.path.join(PROJECT_ROOT, 'test_runtime'),
}
# Set custom executor command if set via environment variable
if 'RESOLWE_DOCKER_COMMAND' in os.environ:
    FLOW_DOCKER_COMMAND = os.environ['RESOLWE_DOCKER_COMMAND']
FLOW_API = {
    'PERMISSIONS': 'resolwe.permissions.permissions',
}
FLOW_EXPRESSION_ENGINES = [
    {
        'ENGINE': 'resolwe.flow.expression_engines.jinja',
        'CUSTOM_FILTERS': [
            'resolwe_bio.expression_filters.sample',
        ]
    },
]
FLOW_EXECUTION_ENGINES = [
    'resolwe.flow.execution_engines.bash',
    'resolwe.flow.execution_engines.workflow',
]

# Check if any Manager settings are set via environment variables
manager_prefix = os.environ.get('RESOLWE_MANAGER_REDIS_PREFIX', 'resolwe-bio.manager')
FLOW_MANAGER = {
    'NAME': 'resolwe.flow.managers.workload_connectors.local',
    'REDIS_PREFIX': manager_prefix,
    'REDIS_CONNECTION': REDIS_CONNECTION,
}

FLOW_DOCKER_VOLUME_EXTRA_OPTIONS = {
    'data': 'Z',
    'data_all': 'z',
    'upload': 'z',
    'secrets': 'Z',
    'users': 'Z',
    'tools': 'z',
}

FLOW_PROCESS_MAX_CORES = 1

# Don't pull Docker images if set via the environment variable.
FLOW_DOCKER_DONT_PULL = strtobool(os.environ.get('RESOLWE_DOCKER_DONT_PULL', '0'))

# Ensure all container images follow a specific format.
FLOW_CONTAINER_VALIDATE_IMAGE = r'.+:(?!latest)'

REST_FRAMEWORK = {
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'rest_framework.authentication.SessionAuthentication',
    ),
    'DEFAULT_FILTER_BACKENDS': (
        'resolwe.permissions.filters.ResolwePermissionsFilter',
        'rest_framework_filters.backends.DjangoFilterBackend',
    ),
}

FLOW_PROCESSES_FINDERS = (
    'resolwe.flow.finders.FileSystemProcessesFinder',
    'resolwe.flow.finders.AppDirectoriesFinder',
)

FLOW_PROCESSES_DIRS = (os.path.join(PROJECT_ROOT, '../resolwe_bio/tests/'),)

# Do not skip tests that fail on Docker executor if this is set via environment
# variable
if os.environ.get('RESOLWEBIO_TESTS_SKIP_DOCKER_FAILURES', '').lower() in ["no", "false"]:
    TESTS_SKIP_DOCKER_FAILURES = False

# Elastic Search.

ELASTICSEARCH_HOST = os.environ.get('RESOLWE_ES_HOST', 'localhost')
ELASTICSEARCH_PORT = int(os.environ.get('RESOLWE_ES_PORT', '59201'))

# Testing.

TEST_RUNNER = 'resolwe.test_helpers.test_runner.ResolweRunner'
TEST_PROCESS_REQUIRE_TAGS = True
# Don't profile unless set via the environment variable.
TEST_PROCESS_PROFILE = strtobool(os.environ.get('RESOLWE_TEST_PROCESS_PROFILE', '0'))

# Channels.

CHANNEL_LAYERS = {
    'default': {
        'BACKEND': 'asgi_redis.RedisChannelLayer',
        'ROUTING': 'tests.routing.channel_routing',
        'CONFIG': {
            'hosts': [(REDIS_CONNECTION['host'], REDIS_CONNECTION['port'])],
            'expiry': 3600,
        },
    },
}
<|endoftext|>


<commit_before># Copyright (C) 2012  Renato Lima (Akretion)                                  #
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

import logging

from odoo import models, fields, api, _
from odoo.exceptions import UserError

from odoo.addons.l10n_br_base.tools import misc

_logger = logging.getLogger(__name__)

try:
    import pycep_correios
except ImportError:
    _logger.warning("Library PyCEP-Correios not installed !")

class L10nBrZip(models.Model):
    """ Este objeto persiste todos os códigos postais que podem ser
    utilizados para pesquisar e auxiliar o preenchimento dos endereços.
    """
    _name = 'l10n_br.zip'
    _description = 'CEP'
    _rec_name = 'zip_code'

    zip_code = fields.Char(
        string='CEP',
        size=8,
        required=True)

    street_type = fields.Char(
        string='Street Type',
        size=26)

    zip_complement = fields.Char(
        string='Range',
        size=200)

    street = fields.Char(
        string='Logradouro',
        size=72)

    district = fields.Char(
        string='District',
        size=72)

    country_id = fields.Many2one(
        comodel_name='res.country',
        string='Country')

    state_id = fields.Many2one(
        comodel_name='res.country.state',
        string='State',
        domain="[('country_id','=',country_id)]")

    city_id = fields.Many2one(
        comodel_name='res.city',
        string='City',
        required=True,
        domain="[('state_id','=',state_id)]")

    def _set_domain(self, country_id=False, state_id=False,
                    city_id=False, district=False,
                    street=False, zip_code=False):
        domain = []
        if zip_code:
            new_zip = misc.punctuation_rm(zip_code or '')
            domain.append(('zip_code', '=', new_zip))
        else:
            if not state_id or not city_id or len(street or '') == 0:
                raise UserError(
                    _('Necessário informar Estado, município e logradouro'))

            if country_id:
                domain.append(('country_id', '=', country_id))
            if state_id:
                domain.append(('state_id', '=', state_id))
            if city_id:
                domain.append(('city_id', '=', city_id))
            if district:
                domain.append(('district', 'ilike', district))
            if street:
                domain.append(('street', 'ilike', street))

        return domain

    def set_result(self, zip_obj=None):
        if zip_obj:
            if type(zip_obj).__name__ == 'l10n_br.zip.result':
                zip_code = zip_obj.zip
            else:
                zip_code = zip_obj.zip_code

            if len(zip_code) == 8:
                zip_code = '%s-%s' % (zip_code[0:5], zip_code[5:8])
            result = {
                'country_id': zip_obj.country_id.id,
                'state_id': zip_obj.state_id.id,
                'city_id': zip_obj.city_id.id,
                'city': zip_obj.city_id.name,
                'district': zip_obj.district,
                'street': ((zip_obj.street_type or '') +
                           ' ' + (zip_obj.street or '')) if
                zip_obj.street_type else (zip_obj.street or ''),
                'zip': zip_code,
            }
        else:
            result = {}
        return result

    @api.model
    def zip_search(self, obj):

        try:
            domain = self._set_domain(
                country_id=obj.country_id.id,
                state_id=obj.state_id.id,
                city_id=obj.city_id.id,
                district=obj.district,
                street=obj.street,
                zip_code=obj.zip)
        except AttributeError as e:
            raise UserError(
                _('Erro a Carregar Atributo: ') + str(e))

        zip_ids = self.search(domain)

        # One ZIP was found
        if len(zip_ids) == 1:
            result = self.set_result(zip_ids[0])
            obj.write(result)
            return True

        # More than one ZIP was found
        elif len(zip_ids) > 1:

            return self.create_wizard(obj, zip_ids)

        # Address not found in local DB, search by PyCEP-Correios
        elif not zip_ids and obj.zip:

            zip_str = misc.punctuation_rm(obj.zip)
            try:
                result = pycep_correios.consultar_cep(zip_str)
            except Exception as e:
                raise UserError(
                    _('Erro no PyCEP-Correios : ') + str(e))

            if result:
                # Search Brazil id
                country = self.env['res.country'].search(
                    [('code', '=', 'BR')], limit=1)

                # Search state with state_code and country id
                state = self.env['res.country.state'].search([
                    ('code', '=', result['uf']),
                    ('country_id', '=', country.id)], limit=1)

                # search city with name and state
                city = self.env['res.city'].search([
                    ('name', '=', result['cidade']),
                    ('state_id.id', '=', state.id)], limit=1)

                values = {
                    'zip_code': zip_str,
                    'street': result['end'],
                    'zip_complement': result['complemento2'],
                    'district': result['bairro'],
                    'city_id': city.id or False,
                    'state_id': state.id or False,
                    'country_id': country.id or False,
                }

                # Create zip object
                self.env['l10n_br.zip'].create(values)

                zip_ids = self.search(domain)
                result = self.set_result(zip_ids[0])
                obj.write(result)
                return True

    def create_wizard(self, obj, zips):
        wizard = self.env['l10n_br.zip.search'].create({
            'zip': obj.zip,
            'street': obj.street,
            'district': obj.district,
            'country_id': obj.country_id.id,
            'state_id': obj.state_id.id,
            'city_id': obj.city_id.id,
            'zip_ids': [[6, 0, [zip.id for zip in zips]]],
            'address_id': obj.id,
            'object_name': obj._name
        })

        return {
            'name': 'Zip Search',
            'view_type': 'form',
            'view_mode': 'form',
            'res_model': 'l10n_br.zip.search',
            'view_id': False,
            'type': 'ir.actions.act_window',
            'target': 'new',
            'nodestroy': True,
            'res_id': wizard.id,
        }

    @api.one
    def zip_select(self):
        data = self
        address_id = self._context.get('address_id')
        object_name = self._context.get('object_name')
        if address_id and object_name:
            obj = self.env[object_name].browse(address_id)
            obj_zip = self.env['l10n_br.zip']
            result = obj_zip.set_result(data)
            obj.write(result)
        return True
<commit_msg>[REF] Clean up code.<commit_after># Copyright (C) 2012  Renato Lima (Akretion)                                  #
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

import logging

from odoo import models, fields, api, _
from odoo.exceptions import UserError

from odoo.addons.l10n_br_base.tools import misc

_logger = logging.getLogger(__name__)

try:
    import pycep_correios
except ImportError:
    _logger.warning("Library PyCEP-Correios not installed !")

class L10nBrZip(models.Model):
    """ Este objeto persiste todos os códigos postais que podem ser
    utilizados para pesquisar e auxiliar o preenchimento dos endereços.
    """
    _name = 'l10n_br.zip'
    _description = 'CEP'
    _rec_name = 'zip_code'

    zip_code = fields.Char(
        string='CEP',
        size=8,
        required=True)

    street_type = fields.Char(
        string='Street Type',
        size=26)

    zip_complement = fields.Char(
        string='Range',
        size=200)

    street = fields.Char(
        string='Logradouro',
        size=72)

    district = fields.Char(
        string='District',
        size=72)

    country_id = fields.Many2one(
        comodel_name='res.country',
        string='Country')

    state_id = fields.Many2one(
        comodel_name='res.country.state',
        string='State',
        domain="[('country_id','=',country_id)]")

    city_id = fields.Many2one(
        comodel_name='res.city',
        string='City',
        required=True,
        domain="[('state_id','=',state_id)]")

    def _set_domain(self, country_id=False, state_id=False,
                    city_id=False, district=False,
                    street=False, zip_code=False):
        domain = []
        if zip_code:
            new_zip = misc.punctuation_rm(zip_code or '')
            domain.append(('zip_code', '=', new_zip))
        else:
            if not state_id or not city_id or len(street or '') == 0:
                raise UserError(
                    _('Necessário informar Estado, município e logradouro'))

            if country_id:
                domain.append(('country_id', '=', country_id))
            if state_id:
                domain.append(('state_id', '=', state_id))
            if city_id:
                domain.append(('city_id', '=', city_id))
            if district:
                domain.append(('district', 'ilike', district))
            if street:
                domain.append(('street', 'ilike', street))

        return domain

    @api.multi
    def set_result(self):
        self.ensure_one()
        return {
            'country_id': self.country_id.id,
            'state_id': self.state_id.id,
            'city_id': self.city_id.id,
            'city': self.city_id.name,
            'district': self.district,
            'street': ((self.street_type or '') +
                       ' ' + (self.street or '')) if
            self.street_type else (self.street or ''),
            'zip': misc.format_zipcode(
                self.zip_code, self.country_id.code)}

    @api.model
    def zip_search(self, obj):

        try:
            domain = self._set_domain(
                country_id=obj.country_id.id,
                state_id=obj.state_id.id,
                city_id=obj.city_id.id,
                district=obj.district,
                street=obj.street,
                zip_code=obj.zip)
        except AttributeError as e:
            raise UserError(
                _('Erro a Carregar Atributo: ') + str(e))

        zips = self.search(domain)

        # One ZIP was found
        if len(zips) == 1:
            obj.write(zips[0].set_result())
            return True

        # More than one ZIP was found
        elif len(zips) > 1:

            return self.create_wizard(obj, zips)

        # Address not found in local DB, search by PyCEP-Correios
        elif not zips and obj.zip:

            zip_str = misc.punctuation_rm(obj.zip)
            try:
                result = pycep_correios.consultar_cep(zip_str)
            except Exception as e:
                raise UserError(
                    _('Erro no PyCEP-Correios : ') + str(e))

            if result:
                # Search Brazil id
                country = self.env['res.country'].search(
                    [('code', '=', 'BR')], limit=1)

                # Search state with state_code and country id
                state = self.env['res.country.state'].search([
                    ('code', '=', result['uf']),
                    ('country_id', '=', country.id)], limit=1)

                # search city with name and state
                city = self.env['res.city'].search([
                    ('name', '=', result['cidade']),
                    ('state_id.id', '=', state.id)], limit=1)

                values = {
                    'zip_code': zip_str,
                    'street': result['end'],
                    'zip_complement': result['complemento2'],
                    'district': result['bairro'],
                    'city_id': city.id or False,
                    'state_id': state.id or False,
                    'country_id': country.id or False,
                }

                # Create zip object
                zip = self.create(values)
                obj.write(zip.set_result())
                return True

    def create_wizard(self, obj, zips):

        context = dict(self.env.context)
        context.update({
            'address_id': obj.id,
            'object_name': obj._name,
        })

        wizard = self.env['l10n_br.zip.search'].create({
            'zip': obj.zip,
            'street': obj.street,
            'district': obj.district,
            'country_id': obj.country_id.id,
            'state_id': obj.state_id.id,
            'city_id': obj.city_id.id,
            'zip_ids': [[6, 0, [zip.id for zip in zips]]],
            'address_id': obj.id,
            'object_name': obj._name
        })

        return {
            'name': 'Zip Search',
            'view_type': 'form',
            'view_mode': 'form',
            'res_model': 'l10n_br.zip.search',
            'view_id': False,
            'type': 'ir.actions.act_window',
            'target': 'new',
            'nodestroy': True,
            'res_id': wizard.id,
            'context': context,
        }

    @api.multi
    def zip_select(self):
        self.ensure_one()
        address_id = self._context.get('address_id')
        object_name = self._context.get('object_name')
        if address_id and object_name:
            obj = self.env[object_name].browse(address_id)
            obj.write(self.set_result())
        return True
<|endoftext|>


<commit_before>#!/usr/bin/python
import sys, getopt
import os.path
import math
import numpy
import csv

HEADERS = ['seqNum', 'seqRNA', 'areaDiff', 'normDiff']

# def parse_float(string):
#     try:
#         return float(string)
#     except Exception:
#         throw TypeError

def try_parse_float(string, fail=None):
    try:
        return float(string)
    except Exception:
        return fail;

def usage():
    print 'outlyers.py -i <inputfile> [-o <outputfile>]'

def average_percentage(l, out, percentage, percentage_outlier):
    ordered = sorted(l, reverse=True)
    outliers = [x for x in ordered if x > out]
    out_percentage_length = int(math.ceil((len(outliers) * percentage_outlier) / 100.0))
    wout = ordered[out_percentage_length:]
    percentage_length = int(math.ceil(len(l) * percentage / 100.0))
    return average(wout[:percentage_length])

def average(l):
    length = len(l)
    if length > 0:
        return sum(l)/len(l)
    return 0

def average_out(l, out, percentage):
    ordered = sorted(l, reverse=True)
    length = int(math.ceil(len(l) * percentage / 100.0))
    return average(ordered[:length])

def average_wout(l, out, percentage):
    ordered = sorted(l, reverse=True)
    outliers = [x for x in ordered if x > out]
    out_length = len(outliers)
    wout = ordered[out_length:]
    length = int(math.ceil(len(l) * percentage / 100.0))
    return average(wout[:length])

def out(itr):
    return itr + (itr * 1.5)

def itr(q_1, q_3):
    return q_3 - q_1

def q1(l):
    positive_nums = [x for x in l if x >= 0]
    sorted_nums = sorted(positive_nums)
    low_mid = int(round( (len(sorted_nums) + 1 )  / 4.0 ) - 1 )
    lq = sorted_nums[low_mid]
    return lq

# First quartile of positive data
def q3(l):
    uq = 0;
    l = [x for x in l if x >= 0]
    positiveNums = sorted(l) #< Sort the list in ascending order
    try:
        high_mid = ( len( positiveNums ) - 1 ) * 0.75
        uq = positiveNums[ high_mid ]
    except TypeError:   #<  There were an even amount of values
        # Make sure to type results of math.floor/ceil to int for use in list indices
        ceil = int( math.ceil( high_mid ) )
        floor = int( math.floor( high_mid ) )
        uq = ( positiveNums[ ceil ] + positiveNums[ floor ] ) / 2

    return uq

def bound_range(v, ll, ul):
    if v < ll:
        return ll
    if v > ul:
        return ul
    return v

def readInput(filename):
    header = []
    data  = []
    with open(filename) as inputfile:
        lines = inputfile.read().splitlines()
        original_header = lines[0].split()
        original_data = lines[1:]
        rows_to_process = [k for k in range(len(original_header)) if original_header[k] in HEADERS]

        header = [original_header[x] for x in rows_to_process]
        for cols in header:
            data.append([])

        for line in original_data:
            row = line.split()
            i = 0
            for col in rows_to_process:
                data[i].append(try_parse_float(row[col], row[col]))
                i += 1
    return header, data

def writeOutput(filename, header, table):
    csv.writer(sys.stdout)
    outputfile = sys.stdout
    if filename != '':
        outputfile =  open(filename, 'w')

    csv_writer = csv.writer(outputfile)
    csv_writer.writerow(header)
    for row in table:
        csv_writer.writerow(row)

def processData(header, table):
    percentage = 10
    processed_header = header
    processed_table = table
    area_diff_list = table[header.index('areaDiff')];
    param = dict()
    param_list = dict()

    # Calculate parameters
    param['q1'] = q1(area_diff_list)
    param['q3'] = q3(area_diff_list)
    param['itr'] = itr(param['q1'], param['q3'])
    param['out'] = out(param['itr'])
    param_list['average_out'] = average_out(area_diff_list, param['out'], percentage)
    param_list['average_wout'] = average_wout(area_diff_list, param['out'], percentage)
    param_list['average_percentage_50'] = average_percentage(area_diff_list, param['out'], percentage, 50)
    param_list['average_percentage_20'] = average_percentage(area_diff_list, param['out'], percentage, 20)
    param_list['average_percentage_70'] = average_percentage(area_diff_list, param['out'], percentage, 70)

    # Calculate new columns
    for (k, v) in param_list.items():
        processed_header.append(k)
        new_col = [bound_range(x/v, 0, 3) for x in area_diff_list]
        processed_table.append(new_col)
    return processed_header, processed_table, param, param_list

def main(argv):
    data = ''
    inputfile = ''
    outputfile = ''
    try:
        opts, args = getopt.getopt(argv,"hi:w:s:o:",["help","inputfile=","outputfile=","window=","step="])
    except getopt.GetoptError:
        usage()
    for opt, arg in opts:
        if opt == '-h':
            usage()
        elif opt in ("-i", "--inputfile"):
            inputfile = arg
        elif opt in ("-o", "--outputfile"):
            outputfile = arg
    if os.path.isfile(inputfile):
        header, table = readInput(inputfile)
    else:
        print "Invalid input file '%s'." % inputfile
        usage()
        sys.exit(2)

    header, outputTable, param, param_list = processData(header, table)
    data_to_write = zip(*outputTable)
    writeOutput(outputfile, header, data_to_write)

if __name__ == "__main__":
    main(sys.argv[1:])
<commit_msg>Added calculated parameters to output file<commit_after>#!/usr/bin/python
import sys, getopt
import os.path
import math
import numpy
import csv

HEADERS = ['seqNum', 'seqRNA', 'areaDiff', 'normDiff']

# def parse_float(string):
#     try:
#         return float(string)
#     except Exception:
#         throw TypeError

def try_parse_float(string, fail=None):
    try:
        return float(string)
    except Exception:
        return fail;

def usage():
    print 'outlyers.py -i <inputfile> [-o <outputfile>]'

def average_percentage(l, out, percentage, percentage_outlier):
    ordered = sorted(l, reverse=True)
    outliers = [x for x in ordered if x > out]
    out_percentage_length = int(math.ceil((len(outliers) * percentage_outlier) / 100.0))
    wout = ordered[out_percentage_length:]
    percentage_length = int(math.ceil(len(l) * percentage / 100.0))
    return average(wout[:percentage_length])

def average(l):
    length = len(l)
    if length > 0:
        return sum(l)/len(l)
    return 0

def average_out(l, out, percentage):
    ordered = sorted(l, reverse=True)
    length = int(math.ceil(len(l) * percentage / 100.0))
    return average(ordered[:length])

def average_wout(l, out, percentage):
    ordered = sorted(l, reverse=True)
    outliers = [x for x in ordered if x > out]
    out_length = len(outliers)
    wout = ordered[out_length:]
    length = int(math.ceil(len(l) * percentage / 100.0))
    return average(wout[:length])

def out(itr):
    return itr + (itr * 1.5)

def itr(q_1, q_3):
    return q_3 - q_1

def q1(l):
    positive_nums = [x for x in l if x >= 0]
    sorted_nums = sorted(positive_nums)
    low_mid = int(round( (len(sorted_nums) + 1 )  / 4.0 ) - 1 )
    lq = sorted_nums[low_mid]
    return lq

# First quartile of positive data
def q3(l):
    uq = 0;
    l = [x for x in l if x >= 0]
    positiveNums = sorted(l) #< Sort the list in ascending order
    try:
        high_mid = ( len( positiveNums ) - 1 ) * 0.75
        uq = positiveNums[ high_mid ]
    except TypeError:   #<  There were an even amount of values
        # Make sure to type results of math.floor/ceil to int for use in list indices
        ceil = int( math.ceil( high_mid ) )
        floor = int( math.floor( high_mid ) )
        uq = ( positiveNums[ ceil ] + positiveNums[ floor ] ) / 2

    return uq

def bound_range(v, ll, ul):
    if v < ll:
        return ll
    if v > ul:
        return ul
    return v

def readInput(filename):
    header = []
    data  = []
    with open(filename) as inputfile:
        lines = inputfile.read().splitlines()
        original_header = lines[0].split()
        original_data = lines[1:]
        rows_to_process = [k for k in range(len(original_header)) if original_header[k] in HEADERS]

        header = [original_header[x] for x in rows_to_process]
        for cols in header:
            data.append([])

        for line in original_data:
            row = line.split()
            i = 0
            for col in rows_to_process:
                data[i].append(try_parse_float(row[col], row[col]))
                i += 1
    return header, data

def writeOutput(filename, header, table):
    csv.writer(sys.stdout)
    outputfile = sys.stdout
    if filename != '':
        outputfile =  open(filename, 'w')

    csv_writer = csv.writer(outputfile)
    csv_writer.writerow(header)
    for row in table:
        csv_writer.writerow(row)

def processData(header, table):
    percentage = 10
    processed_header = header
    processed_table = table
    area_diff_list = table[header.index('areaDiff')];
    param = dict()
    param_list = dict()

    # Calculate parameters
    param['q1'] = q1(area_diff_list)
    param['q3'] = q3(area_diff_list)
    param['itr'] = itr(param['q1'], param['q3'])
    param['out'] = out(param['itr'])
    param_list['average_out'] = average_out(area_diff_list, param['out'], percentage)
    param_list['average_wout'] = average_wout(area_diff_list, param['out'], percentage)
    param_list['average_percentage_50'] = average_percentage(area_diff_list, param['out'], percentage, 50)
    param_list['average_percentage_20'] = average_percentage(area_diff_list, param['out'], percentage, 20)
    param_list['average_percentage_70'] = average_percentage(area_diff_list, param['out'], percentage, 70)

    # Calculate new columns
    for (k, v) in param_list.items():
        processed_header.append(k)
        new_col = [bound_range(x/v, 0, 3) for x in area_diff_list]
        processed_table.append(new_col)
    return processed_header, processed_table, param, param_list

def main(argv):
    data = ''
    inputfile = ''
    outputfile = ''
    try:
        opts, args = getopt.getopt(argv,"hi:w:s:o:",["help","inputfile=","outputfile=","window=","step="])
    except getopt.GetoptError:
        usage()
    for opt, arg in opts:
        if opt == '-h':
            usage()
        elif opt in ("-i", "--inputfile"):
            inputfile = arg
        elif opt in ("-o", "--outputfile"):
            outputfile = arg
    if os.path.isfile(inputfile):
        header, table = readInput(inputfile)
    else:
        print "Invalid input file '%s'." % inputfile
        usage()
        sys.exit(2)

    header, outputTable, param, param_list = processData(header, table)
    data_to_write = zip(*outputTable)
    header = (header + param.keys())
    data_to_write[0] = (data_to_write[0] + tuple(param.values()))
    header = (header + param_list.keys())
    data_to_write[0] = (data_to_write[0] + tuple(param_list.values()))

    writeOutput(outputfile, header, data_to_write)

if __name__ == "__main__":
    main(sys.argv[1:])
<|endoftext|>


<commit_before># i10chic.py
# Animated simulation of chicane magnets

# Import libraries

import dls_packages
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation

# Define matrices to modify the electron beam vector:
# drift, kicker magnets, insertion devices.

class Drifting:

    def __init__(self):
        pass # Do I want to have a value of step in here? Apparently yes but couldn't get code to work when I tried to put it in so leaving it for now
        #self.step = None

    def set_length(self, step):
        self.step = step

    def increment(self, e):
        drift = np.array([[1,self.step],
                          [0,1]])
        return np.dot(drift,e)

    def get_type(self):
        return 'drift'

class Kicker:

    def __init__(self):
        pass
        #self.k = None

    def set_strength(self, k):
        self.k = k

    def increment(self, e):
        kick = np.array([0, self.k])
        return e + kick

    def get_type(self):
        return 'kicker'

class InsertionDevice:

    def __init__(self):
        pass

    def increment(self, e):
        return e

    def get_type(self):
        return 'id'

k3 = 1

# Define positions of devices in system
length = [2,2,4,4,4,4,2,20] # lengths to drift between kickers and IDs
pos = [0]
pos.extend(np.cumsum(length))

# Positions of kickers, IDs, detector and 'on axis' photon beam points for plotting purposes.
kicker_pos = [pos[1],pos[2],pos[4],pos[6],pos[7]]
id_pos = [pos[3],pos[5]]
detector_pos = 40
p_pos = [[pos[3], detector_pos],[pos[5],detector_pos]]

# Define magnet strength factors (dependent on relative positions and time).
len1 = kicker_pos[1] - kicker_pos[0]
len2 = kicker_pos[2] - kicker_pos[1]
d12 = float(len1)/float(len2)
len3 = kicker_pos[3] - kicker_pos[2]
len4 = kicker_pos[4] - kicker_pos[3]
d34 = float(len3)/float(len4)
max_kick = np.array([1, 1 + d12, 2*d12, d12*(1+d34), d12*d34])

# Define time-varying strengths of kicker magnets.
def calculate_strengths(t):

    kick = 0.5*max_kick*np.array([
        np.sin(t*np.pi/100) + 1, -(np.sin(t*np.pi/100) + 1),
        k3, np.sin(t*np.pi/100) - 1,
        -np.sin(t*np.pi/100) + 1
        ]) # Factor 0.5 so that maximum kick applied = 1.

    return kick

# Define path through system.
path = [
    Drifting(),Kicker(),
    Drifting(),Kicker(),
    Drifting(),InsertionDevice(),
    Drifting(),Kicker(),
    Drifting(),InsertionDevice(),
    Drifting(),Kicker(),
    Drifting(),Kicker(),
    Drifting()
    ]

# Function that returns all objects of a particular type from path.
def get_elements(path, which):
    objects = []
    for p in path:
        if p.get_type() == which:
            objects.append(p)
    return objects

# Set drift distances (time independent).
for drift, dist in zip(get_elements(path, 'drift'), length):
    drift.set_length(dist)

# Send electron vector through chicane magnets at time t.
def timestep(t):

    # Initialise electron beam position and velocity
    e_beam = np.array([0,0])
    e_vector = [[0,0]]

    # Initialise photon beam
    p_vector = []

    # Calculate positions of electron beam and photon beam relative to main axis.
    for kicker, strength in zip(get_elements(path, 'kicker'), calculate_strengths(t)):
         kicker.set_strength(strength)
    for p in path:
         e_beam = p.increment(e_beam)
         device = p.get_type()
         if device == 'drift':  # Better way of doing this??
             e_vector.append(e_beam.tolist())  # Allow electron vector to drift and append its new location and velocity to vector collecting the data
         elif device == 'id':
            p_vector.append(e_beam.tolist())  # Electron vector passes through insertion device, photon vector created

    return e_vector, p_vector # returns positions and velocities of electrons and photons

# Extract electron beam positions for plotting.
def e_plot(t):

    e_positions = np.array(timestep(t)[0])[:,0]

    return e_positions

# Allow the two photon vectors to drift over large distance
# (ie off the graph) and add the vector for new position and
# velocity to original vector to create beam for plotting.
def p_plot(t):

    p_beam = timestep(t)[1]
    travel = [Drifting(),Drifting()]
    for i in range(2):
        travel[i].set_length(p_pos[i][1]-p_pos[i][0])
        p_beam[i].extend(travel[i].increment(p_beam[i]))

    p_positions = np.array(p_beam)[:,[0,2]]

    return p_positions

########################################################

# Set up figure, axis and plot element to be animated.
fig = plt.figure()
ax1 = fig.add_subplot(2, 1, 1)
ax1.set_xlim(0, sum(length))
ax1.set_ylim(-2, 5)
ax3 = fig.add_subplot(2, 2, 4)
ax3.set_xlim(-10, 10)
ax3.set_ylim(0, 1000)
#ax = plt.axes(xlim=(0, sum(length)), ylim=(-2, 5))
beams = [ax1.plot([], [])[0], ax1.plot([], [], 'r')[0], ax1.plot([], [], 'r')[0],
         ax3.plot([], [], 'r.')[0], ax3.plot([], [], 'r.')[0], ax3.plot([], [], 'y.')[0]]

# Initialisation function: plot the background of each frame.
def init():

    for line in beams:
        line.set_data([], [])

    return beams

import gc  # This can't stay here! This is garbage collection

flash = []
ftime = []

# Animation function
def animate(t):

    # Obtain data for plotting.
    e_data = e_plot(t)
    p_data = p_plot(t)
    d_data = p_data[:,1].tolist()
    if t < 1000:
        if d_data[0] == 0:
            flash.append(d_data[0])
            ftime.append(t)
        elif d_data[1] == 0:
            flash.append(d_data[1])
            ftime.append(t) # IS THERE A WAY OF SHIFTING THEM UP OFF THE GRAPH AND NO LONGER PLOTTING THEM??
#    elif t > 1000:
#        flash == [0]
#        ftime == [0] #this is dodgy and probably wrong
    time = [t,t]

    # Set data for electron beam.
    beams[0].set_data(pos, e_data)

    # Set data for two photon beams.
    for line, x, y in zip([beams[1],beams[2]], p_pos, p_data):
        line.set_data(x,y)

    # Set data for photon beam at detector.
    beams[3].set_data(d_data, [10,10])
    beams[4].set_data(d_data, time)
    beams[5].set_data(flash, ftime)

    # HOW TO ANIMATE A MOVING WAVE LIKE IN THE MATLAB PLOT??

    gc.collect(0)

    return beams

# Call the animator
anim = animation.FuncAnimation(fig, animate, init_func=init,
                               frames=1000, interval=20, blit=True)

# Plot positions of kickers and IDs.
for i in kicker_pos:
    ax1.axvline(x=i, color='k', linestyle='dashed')
for i in id_pos:
    ax1.axvline(x=i, color='r', linestyle='dashed')

#############

ax2 = fig.add_subplot(2, 2, 3)
ax2.set_xlim(-10, 10)
ax2.set_ylim(0, 10)

#############

plt.show()

<commit_msg>changed photon beam plot slightly<commit_after># i10chic.py
# Animated simulation of chicane magnets

# Import libraries

import dls_packages
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation

# Define matrices to modify the electron beam vector:
# drift, kicker magnets, insertion devices.

class Drifting:

    def __init__(self):
        pass # Do I want to have a value of step in here? Apparently yes but couldn't get code to work when I tried to put it in so leaving it for now
        #self.step = None

    def set_length(self, step):
        self.step = step

    def increment(self, e):
        drift = np.array([[1,self.step],
                          [0,1]])
        return np.dot(drift,e)

    def get_type(self):
        return 'drift'

class Kicker:

    def __init__(self):
        pass
        #self.k = None

    def set_strength(self, k):
        self.k = k

    def increment(self, e):
        kick = np.array([0, self.k])
        return e + kick

    def get_type(self):
        return 'kicker'

class InsertionDevice:

    def __init__(self):
        pass

    def increment(self, e):
        return e

    def get_type(self):
        return 'id'

k3 = 1

# Define positions of devices in system
length = [2,2,4,4,4,4,2,20] # lengths to drift between kickers and IDs
pos = [0]
pos.extend(np.cumsum(length))

# Positions of kickers, IDs, detector and 'on axis' photon beam points for plotting purposes.
kicker_pos = [pos[1],pos[2],pos[4],pos[6],pos[7]]
id_pos = [pos[3],pos[5]]
detector_pos = 40
p_pos = [[pos[3], detector_pos],[pos[5],detector_pos]]

# Define magnet strength factors (dependent on relative positions and time).
len1 = kicker_pos[1] - kicker_pos[0]
len2 = kicker_pos[2] - kicker_pos[1]
d12 = float(len1)/float(len2)
len3 = kicker_pos[3] - kicker_pos[2]
len4 = kicker_pos[4] - kicker_pos[3]
d34 = float(len3)/float(len4)
max_kick = np.array([1, 1 + d12, 2*d12, d12*(1+d34), d12*d34])

# Define time-varying strengths of kicker magnets.
def calculate_strengths(t):

    kick = 0.5*max_kick*np.array([
        np.sin(t*np.pi/100) + 1, -(np.sin(t*np.pi/100) + 1),
        k3, np.sin(t*np.pi/100) - 1,
        -np.sin(t*np.pi/100) + 1
        ]) # Factor 0.5 so that maximum kick applied = 1.

    return kick

# Define path through system.
path = [
    Drifting(),Kicker(),
    Drifting(),Kicker(),
    Drifting(),InsertionDevice(),
    Drifting(),Kicker(),
    Drifting(),InsertionDevice(),
    Drifting(),Kicker(),
    Drifting(),Kicker(),
    Drifting()
    ]

# Function that returns all objects of a particular type from path.
def get_elements(path, which):
    objects = []
    for p in path:
        if p.get_type() == which:
            objects.append(p)
    return objects

# Set drift distances (time independent).
for drift, dist in zip(get_elements(path, 'drift'), length):
    drift.set_length(dist)

# Send electron vector through chicane magnets at time t.
def timestep(t):

    # Initialise electron beam position and velocity
    e_beam = np.array([0,0])
    e_vector = [[0,0]]

    # Initialise photon beam
    p_vector = []

    # Calculate positions of electron beam and photon beam relative to main axis.
    for kicker, strength in zip(get_elements(path, 'kicker'), calculate_strengths(t)):
         kicker.set_strength(strength)
    for p in path:
         e_beam = p.increment(e_beam)
         device = p.get_type()
         if device == 'drift':  # Better way of doing this??
             e_vector.append(e_beam.tolist())  # Allow electron vector to drift and append its new location and velocity to vector collecting the data
         elif device == 'id':
            p_vector.append(e_beam.tolist())  # Electron vector passes through insertion device, photon vector created

    return e_vector, p_vector # returns positions and velocities of electrons and photons

# Extract electron beam positions for plotting.
def e_plot(t):

    e_positions = np.array(timestep(t)[0])[:,0]

    return e_positions

# Allow the two photon vectors to drift over large distance
# (ie off the graph) and add the vector for new position and
# velocity to original vector to create beam for plotting.
def p_plot(t):

    p_beam = timestep(t)[1]
    travel = [Drifting(),Drifting()]
    for i in range(2):
        travel[i].set_length(p_pos[i][1]-p_pos[i][0])
        p_beam[i].extend(travel[i].increment(p_beam[i]))

    p_positions = np.array(p_beam)[:,[0,2]]

    return p_positions

########################################################

# Set up figure, axis and plot element to be animated.
fig = plt.figure()
ax1 = fig.add_subplot(2, 1, 1)
ax1.set_xlim(0, sum(length))
ax1.set_ylim(-2, 5)
ax3 = fig.add_subplot(2, 2, 4)
ax3.set_xlim(-10, 10)
ax3.set_ylim(0, 1000)
#ax = plt.axes(xlim=(0, sum(length)), ylim=(-2, 5))
beams = [ax1.plot([], [])[0], ax1.plot([], [], 'r')[0], ax1.plot([], [], 'r')[0],
         ax3.plot([], [], 'r.')[0], ax3.plot([], [], 'r.')[0], ax3.plot([], [], 'y.')[0], ax3.plot([], [], 'ro')[0]]

# Initialisation function: plot the background of each frame.
def init():

    for line in beams:
        line.set_data([], [])

    return beams

import gc  # This can't stay here! This is garbage collection

flash = []
ftime = []
flash2 = []
ftime2 = []

# Animation function
def animate(t):

    # Obtain data for plotting.
    e_data = e_plot(t)
    p_data = p_plot(t)
    d_data = p_data[:,1].tolist()
    if t < 1000:
        if d_data[0] == 0:
            flash.append(d_data[0])
            ftime.append(t)
        elif d_data[1] == 0:
            flash.append(d_data[1])
            ftime.append(t) # IS THERE A WAY OF SHIFTING THEM UP OFF THE GRAPH?
    time = [t,t]

    if t < 1000 and t % 10 == 0:
        flash2.append(d_data)
        ftime2.append([t,t])

    # Set data for electron beam.
    beams[0].set_data(pos, e_data)

    # Set data for two photon beams.
    for line, x, y in zip([beams[1],beams[2]], p_pos, p_data):
        line.set_data(x,y)

    # Set data for photon beam at detector.
    beams[3].set_data(d_data, [10,10])
    beams[4].set_data(d_data, time)
    beams[5].set_data(flash2, ftime2) # Some extra plotting as a guide to the eye.
    beams[6].set_data(flash, ftime)

    # HOW TO ANIMATE A MOVING WAVE LIKE IN THE MATLAB PLOT??

    gc.collect(0)

    return beams

# Call the animator
anim = animation.FuncAnimation(fig, animate, init_func=init,
                               frames=1000, interval=20, blit=True)

# Plot positions of kickers and IDs.
for i in kicker_pos:
    ax1.axvline(x=i, color='k', linestyle='dashed')
for i in id_pos:
    ax1.axvline(x=i, color='r', linestyle='dashed')

#############

ax2 = fig.add_subplot(2, 2, 3)
ax2.set_xlim(-10, 10)
ax2.set_ylim(0, 10)

#############

plt.show()

<|endoftext|>


<commit_before># Copyright (C) 2012  Renato Lima (Akretion)                                  #
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

import logging
from odoo import models, fields, api, _
from odoo.exceptions import UserError

from odoo.addons.l10n_br_base.tools import misc

_logger = logging.getLogger(__name__)

try:
    import pycep_correios
except ImportError:
    _logger.warning("Library PyCEP-Correios not installed !")

class L10nBrZip(models.Model):
    """ Este objeto persiste todos os códigos postais que podem ser
    utilizados para pesquisar e auxiliar o preenchimento dos endereços.
    """
    _name = 'l10n_br.zip'
    _description = 'CEP'
    _rec_name = 'zip_code'

    zip_code = fields.Char(
        string='CEP',
        size=8,
        required=True)

    street_type = fields.Char(
        string='Street Type',
        size=26)

    zip_complement = fields.Char(
        string='Range',
        size=200
    )

    street = fields.Char(
        string='Logradouro',
        size=72)

    district = fields.Char(
        string='District',
        size=72)

    country_id = fields.Many2one(
        comodel_name='res.country',
        string='Country')

    state_id = fields.Many2one(
        comodel_name='res.country.state',
        string='State',
        domain="[('country_id','=',country_id)]")

    city_id = fields.Many2one(
        comodel_name='res.city',
        string='City',
        required=True,
        domain="[('state_id','=',state_id)]")

    def _set_domain(self, country_id=False, state_id=False,
                    city_id=False, district=False,
                    street=False, zip_code=False):
        domain = []
        if zip_code:
            new_zip = misc.punctuation_rm(zip_code or '')
            domain.append(('zip_code', '=', new_zip))
        else:
            if not state_id or not city_id or len(street or '') == 0:
                raise UserError(
                    _('Necessário informar Estado, município e logradouro'))

            if country_id:
                domain.append(('country_id', '=', country_id))
            if state_id:
                domain.append(('state_id', '=', state_id))
            if city_id:
                domain.append(('city_id', '=', city_id))
            if district:
                domain.append(('district', 'ilike', district))
            if street:
                domain.append(('street', 'ilike', street))

        return domain

    def set_result(self, zip_obj=None):
        if zip_obj:
            zip_code = zip_obj.zip_code
            if len(zip_code) == 8:
                zip_code = '%s-%s' % (zip_code[0:5], zip_code[5:8])
            result = {
                'country_id': zip_obj.country_id.id,
                'state_id': zip_obj.state_id.id,
                'city_id': zip_obj.city_id.id,
                'city': zip_obj.city_id.name,
                'district': zip_obj.district,
                'street': ((zip_obj.street_type or '') +
                           ' ' + (zip_obj.street or '')) if
                zip_obj.street_type else (zip_obj.street or ''),
                'zip': zip_code,
            }
        else:
            result = {}
        return result

    @api.model
    def zip_search(self, obj):

        try:
            domain = self._set_domain(
                country_id=obj.country_id.id,
                state_id=obj.state_id.id,
                city_id=obj.city_id.id,
                district=obj.district,
                street=obj.street,
                zip_code=obj.zip)
        except AttributeError as e:
            raise UserError(
                _('Erro a Carregar Atributo: ') + str(e))

        zip_ids = self.search(domain)

        if len(zip_ids) == 1:
            result = self.set_result(zip_ids[0])
            obj.write(result)
            return True

        elif len(zip_ids) > 1:

            obj_zip_result = self.env['l10n_br.zip.result']
            zip_ids = obj_zip_result.map_to_zip_result(
                zip_ids, obj._name, obj.id)

            return self.create_wizard(
                obj._name,
                obj.id,
                country_id=obj.country_id.id,
                state_id=obj.state_id.id,
                city_id=obj.city_id.id,
                district=obj.district,
                street=obj.street,
                zip_code=obj.zip,
                zip_ids=[zip.id for zip in zip_ids]
            )
        elif not zip_ids:

            zip_str = misc.punctuation_rm(obj.zip)
            try:
                res = pycep_correios.consultar_cep(zip_str)

                # Search Brazil id
                country = self.env['res.country'].search(
                    [('code', '=', 'BR')], limit=1)

                # Search state with state_code and country id
                state = self.env['res.country.state'].search([
                    ('code', '=', res['uf']),
                    ('country_id', '=', country.id)], limit=1)

                # search city with name and state
                city = self.env['res.city'].search([
                    ('name', '=', res['cidade']),
                    ('state_id.id', '=', state.id)], limit=1)

                values = {
                    'zip_code': zip_str,
                    'street': res['end'],
                    'zip_complement': res['complemento2'],
                    'district': res['bairro'],
                    'city_id': city.id or False,
                    'state_id': state.id or False,
                    'country_id': country.id or False,
                }

                # Create zip object
                self.env['l10n_br.zip'].create(values)

                zip_ids = self.search(domain)
                result = self.set_result(zip_ids[0])
                obj.write(result)
                return True

            except Exception as e:
                raise UserError(
                    _('Erro no PyCEP-Correios : ') + str(e))

    def create_wizard(self, object_name, address_id, country_id=False,
                      state_id=False, city_id=False,
                      district=False, street=False, zip_code=False,
                      zip_ids=False):
        context = dict(self.env.context)
        context.update({
            'zip': zip_code,
            'street': street,
            'district': district,
            'country_id': country_id,
            'state_id': state_id,
            'city_id': city_id,
            'zip_ids': zip_ids,
            'address_id': address_id,
            'object_name': object_name})

        result = {
            'name': 'Zip Search',
            'view_type': 'form',
            'view_mode': 'form',
            'res_model': 'l10n_br.zip.search',
            'view_id': False,
            'context': context,
            'type': 'ir.actions.act_window',
            'target': 'new',
            'nodestroy': True,
        }

        return result
<commit_msg>[REF] Reduce exception code.<commit_after># Copyright (C) 2012  Renato Lima (Akretion)                                  #
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

import logging
from odoo import models, fields, api, _
from odoo.exceptions import UserError

from odoo.addons.l10n_br_base.tools import misc

_logger = logging.getLogger(__name__)

try:
    import pycep_correios
except ImportError:
    _logger.warning("Library PyCEP-Correios not installed !")

class L10nBrZip(models.Model):
    """ Este objeto persiste todos os códigos postais que podem ser
    utilizados para pesquisar e auxiliar o preenchimento dos endereços.
    """
    _name = 'l10n_br.zip'
    _description = 'CEP'
    _rec_name = 'zip_code'

    zip_code = fields.Char(
        string='CEP',
        size=8,
        required=True)

    street_type = fields.Char(
        string='Street Type',
        size=26)

    zip_complement = fields.Char(
        string='Range',
        size=200
    )

    street = fields.Char(
        string='Logradouro',
        size=72)

    district = fields.Char(
        string='District',
        size=72)

    country_id = fields.Many2one(
        comodel_name='res.country',
        string='Country')

    state_id = fields.Many2one(
        comodel_name='res.country.state',
        string='State',
        domain="[('country_id','=',country_id)]")

    city_id = fields.Many2one(
        comodel_name='res.city',
        string='City',
        required=True,
        domain="[('state_id','=',state_id)]")

    def _set_domain(self, country_id=False, state_id=False,
                    city_id=False, district=False,
                    street=False, zip_code=False):
        domain = []
        if zip_code:
            new_zip = misc.punctuation_rm(zip_code or '')
            domain.append(('zip_code', '=', new_zip))
        else:
            if not state_id or not city_id or len(street or '') == 0:
                raise UserError(
                    _('Necessário informar Estado, município e logradouro'))

            if country_id:
                domain.append(('country_id', '=', country_id))
            if state_id:
                domain.append(('state_id', '=', state_id))
            if city_id:
                domain.append(('city_id', '=', city_id))
            if district:
                domain.append(('district', 'ilike', district))
            if street:
                domain.append(('street', 'ilike', street))

        return domain

    def set_result(self, zip_obj=None):
        if zip_obj:
            zip_code = zip_obj.zip_code
            if len(zip_code) == 8:
                zip_code = '%s-%s' % (zip_code[0:5], zip_code[5:8])
            result = {
                'country_id': zip_obj.country_id.id,
                'state_id': zip_obj.state_id.id,
                'city_id': zip_obj.city_id.id,
                'city': zip_obj.city_id.name,
                'district': zip_obj.district,
                'street': ((zip_obj.street_type or '') +
                           ' ' + (zip_obj.street or '')) if
                zip_obj.street_type else (zip_obj.street or ''),
                'zip': zip_code,
            }
        else:
            result = {}
        return result

    @api.model
    def zip_search(self, obj):

        try:
            domain = self._set_domain(
                country_id=obj.country_id.id,
                state_id=obj.state_id.id,
                city_id=obj.city_id.id,
                district=obj.district,
                street=obj.street,
                zip_code=obj.zip)
        except AttributeError as e:
            raise UserError(
                _('Erro a Carregar Atributo: ') + str(e))

        zip_ids = self.search(domain)

        if len(zip_ids) == 1:
            result = self.set_result(zip_ids[0])
            obj.write(result)
            return True

        elif len(zip_ids) > 1:

            obj_zip_result = self.env['l10n_br.zip.result']
            zip_ids = obj_zip_result.map_to_zip_result(
                zip_ids, obj._name, obj.id)

            return self.create_wizard(
                obj._name,
                obj.id,
                country_id=obj.country_id.id,
                state_id=obj.state_id.id,
                city_id=obj.city_id.id,
                district=obj.district,
                street=obj.street,
                zip_code=obj.zip,
                zip_ids=[zip.id for zip in zip_ids]
            )
        elif not zip_ids:

            zip_str = misc.punctuation_rm(obj.zip)
            try:
                res = pycep_correios.consultar_cep(zip_str)
            except Exception as e:
                raise UserError(
                    _('Erro no PyCEP-Correios : ') + str(e))

            if res:
                # Search Brazil id
                country = self.env['res.country'].search(
                    [('code', '=', 'BR')], limit=1)

                # Search state with state_code and country id
                state = self.env['res.country.state'].search([
                    ('code', '=', res['uf']),
                    ('country_id', '=', country.id)], limit=1)

                # search city with name and state
                city = self.env['res.city'].search([
                    ('name', '=', res['cidade']),
                    ('state_id.id', '=', state.id)], limit=1)

                values = {
                    'zip_code': zip_str,
                    'street': res['end'],
                    'zip_complement': res['complemento2'],
                    'district': res['bairro'],
                    'city_id': city.id or False,
                    'state_id': state.id or False,
                    'country_id': country.id or False,
                }

                # Create zip object
                self.env['l10n_br.zip'].create(values)

                zip_ids = self.search(domain)
                result = self.set_result(zip_ids[0])
                obj.write(result)
                return True

    def create_wizard(self, object_name, address_id, country_id=False,
                      state_id=False, city_id=False,
                      district=False, street=False, zip_code=False,
                      zip_ids=False):
        context = dict(self.env.context)
        context.update({
            'zip': zip_code,
            'street': street,
            'district': district,
            'country_id': country_id,
            'state_id': state_id,
            'city_id': city_id,
            'zip_ids': zip_ids,
            'address_id': address_id,
            'object_name': object_name})

        result = {
            'name': 'Zip Search',
            'view_type': 'form',
            'view_mode': 'form',
            'res_model': 'l10n_br.zip.search',
            'view_id': False,
            'context': context,
            'type': 'ir.actions.act_window',
            'target': 'new',
            'nodestroy': True,
        }

        return result
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
    Lexing error finder
    ~~~~~~~~~~~~~~~~~~~

    For the source files given on the command line, display
    the text where Error tokens are being generated, along
    with some context.

    :copyright: Copyright 2006-2011 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import sys, os

try:
    import pygments
except ImportError:
    # try parent path
    sys.path.append(os.path.join(os.path.dirname(__file__), '..'))

from pygments.lexer import RegexLexer
from pygments.lexers import get_lexer_for_filename, get_lexer_by_name
from pygments.token import Error, Text, _TokenType
from pygments.cmdline import _parse_options

class DebuggingRegexLexer(RegexLexer):
    """Make the state stack, position and current match instance attributes."""

    def get_tokens_unprocessed(self, text, stack=('root',)):
        """
        Split ``text`` into (tokentype, text) pairs.

        ``stack`` is the inital stack (default: ``['root']``)
        """
        self.pos = 0
        tokendefs = self._tokens
        self.statestack = list(stack)
        statetokens = tokendefs[self.statestack[-1]]
        while 1:
            for rexmatch, action, new_state in statetokens:
                self.m = m = rexmatch(text, self.pos)
                if m:
                    if type(action) is _TokenType:
                        yield self.pos, action, m.group()
                    else:
                        for item in action(self, m):
                            yield item
                    self.pos = m.end()
                    if new_state is not None:
                        # state transition
                        if isinstance(new_state, tuple):
                            for state in new_state:
                                if state == '#pop':
                                    self.statestack.pop()
                                elif state == '#push':
                                    self.statestack.append(self.statestack[-1])
                                else:
                                    self.statestack.append(state)
                        elif isinstance(new_state, int):
                            # pop
                            del self.statestack[new_state:]
                        elif new_state == '#push':
                            self.statestack.append(self.statestack[-1])
                        else:
                            assert False, 'wrong state def: %r' % new_state
                        statetokens = tokendefs[self.statestack[-1]]
                    break
            else:
                try:
                    if text[self.pos] == '\n':
                        # at EOL, reset state to 'root'
                        self.pos += 1
                        self.statestack = ['root']
                        statetokens = tokendefs['root']
                        yield self.pos, Text, u'\n'
                        continue
                    yield self.pos, Error, text[self.pos]
                    self.pos += 1
                except IndexError:
                    break

def main(fn, lexer=None, options={}):
    if lexer is not None:
        lx = get_lexer_by_name(lexer)
    else:
        try:
            lx = get_lexer_for_filename(os.path.basename(fn), **options)
        except ValueError:
            try:
                name, rest = fn.split('_', 1)
                lx = get_lexer_by_name(name, **options)
            except ValueError:
                raise AssertionError('no lexer found for file %r' % fn)
    debug_lexer = False
    # does not work for e.g. ExtendedRegexLexers
    if lx.__class__.__bases__ == (RegexLexer,):
        lx.__class__.__bases__ = (DebuggingRegexLexer,)
        debug_lexer = True
    lno = 1
    text = file(fn, 'U').read()
    text = text.strip('\n') + '\n'
    tokens = []
    states = []

    def show_token(tok, state):
        reprs = map(repr, tok)
        print '   ' + reprs[1] + ' ' + ' ' * (29-len(reprs[1])) + reprs[0],
        if debug_lexer:
            print ' ' + ' ' * (29-len(reprs[0])) + repr(state),
        print

    for type, val in lx.get_tokens(text):
        lno += val.count('\n')
        if type == Error:
            print 'Error parsing', fn, 'on line', lno
            print 'Previous tokens' + (debug_lexer and ' and states' or '') + ':'
            if showall:
                for tok, state in map(None, tokens, states):
                    show_token(tok, state)
            else:
                for i in range(len(tokens) - num, len(tokens)):
                    show_token(tokens[i], states[i])
            print 'Error token:'
            l = len(repr(val))
            print '   ' + repr(val),
            if debug_lexer and hasattr(lx, 'statestack'):
                print ' ' * (60-l) + repr(lx.statestack),
            print
            print
            return 1
        tokens.append((type, val))
        if debug_lexer:
            if hasattr(lx, 'statestack'):
                states.append(lx.statestack[:])
            else:
                states.append(None)
    if showall:
        for tok, state in map(None, tokens, states):
            show_token(tok, state)
    return 0

num = 10
showall = False
lexer = None
options = {}

if __name__ == '__main__':
    import getopt
    opts, args = getopt.getopt(sys.argv[1:], 'n:l:aO:')
    for opt, val in opts:
        if opt == '-n':
            num = int(val)
        elif opt == '-a':
            showall = True
        elif opt == '-l':
            lexer = val
        elif opt == '-O':
            options = _parse_options([val])
    ret = 0
    for f in args:
        ret += main(f, lexer, options)
    sys.exit(bool(ret))
<commit_msg>Fix find_error script if error is in very beginning of file.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
    Lexing error finder
    ~~~~~~~~~~~~~~~~~~~

    For the source files given on the command line, display
    the text where Error tokens are being generated, along
    with some context.

    :copyright: Copyright 2006-2011 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import sys, os

try:
    import pygments
except ImportError:
    # try parent path
    sys.path.append(os.path.join(os.path.dirname(__file__), '..'))

from pygments.lexer import RegexLexer
from pygments.lexers import get_lexer_for_filename, get_lexer_by_name
from pygments.token import Error, Text, _TokenType
from pygments.cmdline import _parse_options

class DebuggingRegexLexer(RegexLexer):
    """Make the state stack, position and current match instance attributes."""

    def get_tokens_unprocessed(self, text, stack=('root',)):
        """
        Split ``text`` into (tokentype, text) pairs.

        ``stack`` is the inital stack (default: ``['root']``)
        """
        self.pos = 0
        tokendefs = self._tokens
        self.statestack = list(stack)
        statetokens = tokendefs[self.statestack[-1]]
        while 1:
            for rexmatch, action, new_state in statetokens:
                self.m = m = rexmatch(text, self.pos)
                if m:
                    if type(action) is _TokenType:
                        yield self.pos, action, m.group()
                    else:
                        for item in action(self, m):
                            yield item
                    self.pos = m.end()
                    if new_state is not None:
                        # state transition
                        if isinstance(new_state, tuple):
                            for state in new_state:
                                if state == '#pop':
                                    self.statestack.pop()
                                elif state == '#push':
                                    self.statestack.append(self.statestack[-1])
                                else:
                                    self.statestack.append(state)
                        elif isinstance(new_state, int):
                            # pop
                            del self.statestack[new_state:]
                        elif new_state == '#push':
                            self.statestack.append(self.statestack[-1])
                        else:
                            assert False, 'wrong state def: %r' % new_state
                        statetokens = tokendefs[self.statestack[-1]]
                    break
            else:
                try:
                    if text[self.pos] == '\n':
                        # at EOL, reset state to 'root'
                        self.pos += 1
                        self.statestack = ['root']
                        statetokens = tokendefs['root']
                        yield self.pos, Text, u'\n'
                        continue
                    yield self.pos, Error, text[self.pos]
                    self.pos += 1
                except IndexError:
                    break

def main(fn, lexer=None, options={}):
    if lexer is not None:
        lx = get_lexer_by_name(lexer)
    else:
        try:
            lx = get_lexer_for_filename(os.path.basename(fn), **options)
        except ValueError:
            try:
                name, rest = fn.split('_', 1)
                lx = get_lexer_by_name(name, **options)
            except ValueError:
                raise AssertionError('no lexer found for file %r' % fn)
    debug_lexer = False
    # does not work for e.g. ExtendedRegexLexers
    if lx.__class__.__bases__ == (RegexLexer,):
        lx.__class__.__bases__ = (DebuggingRegexLexer,)
        debug_lexer = True
    lno = 1
    text = file(fn, 'U').read()
    text = text.strip('\n') + '\n'
    tokens = []
    states = []

    def show_token(tok, state):
        reprs = map(repr, tok)
        print '   ' + reprs[1] + ' ' + ' ' * (29-len(reprs[1])) + reprs[0],
        if debug_lexer:
            print ' ' + ' ' * (29-len(reprs[0])) + repr(state),
        print

    for type, val in lx.get_tokens(text):
        lno += val.count('\n')
        if type == Error:
            print 'Error parsing', fn, 'on line', lno
            print 'Previous tokens' + (debug_lexer and ' and states' or '') + ':'
            if showall:
                for tok, state in map(None, tokens, states):
                    show_token(tok, state)
            else:
                for i in range(max(len(tokens) - num, 0), len(tokens)):
                    show_token(tokens[i], states[i])
            print 'Error token:'
            l = len(repr(val))
            print '   ' + repr(val),
            if debug_lexer and hasattr(lx, 'statestack'):
                print ' ' * (60-l) + repr(lx.statestack),
            print
            print
            return 1
        tokens.append((type, val))
        if debug_lexer:
            if hasattr(lx, 'statestack'):
                states.append(lx.statestack[:])
            else:
                states.append(None)
    if showall:
        for tok, state in map(None, tokens, states):
            show_token(tok, state)
    return 0

num = 10
showall = False
lexer = None
options = {}

if __name__ == '__main__':
    import getopt
    opts, args = getopt.getopt(sys.argv[1:], 'n:l:aO:')
    for opt, val in opts:
        if opt == '-n':
            num = int(val)
        elif opt == '-a':
            showall = True
        elif opt == '-l':
            lexer = val
        elif opt == '-O':
            options = _parse_options([val])
    ret = 0
    for f in args:
        ret += main(f, lexer, options)
    sys.exit(bool(ret))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#############################################################################
##                                                                         ##
##                                                                         ##
## Copyright (C) 2015 ADS CyberSecurity. All rights reserved.              ##
## This document is the property of ADS CyberSecurity, it is               ##
## licensed under the Apache Open Source License                           ##
##                                                                         ##
##  Author: Andy Dove <andrew.dove@airbus.com>                             ##
##                                                                         ##
#############################################################################

"""
@author:       Andy Dove
@contact:      andrew.dove@airbus.com
@organization: Airbus Defence and Space CyberSecurity
"""

import tempfile
from shutil import rmtree
import subprocess
import os
import logging

def nsis_unpack(data):

    if not os.path.isfile('/usr/bin/7z'):
        logging.debug("7 Zip not installed.  Cannot continue.")
        return []
    # Open Temporary Storage

    f = tempfile.NamedTemporaryFile()
    d = tempfile.mkdtemp()

    # Save a copy of the payload
    f.write(data)
    f.flush()

    # unpack files using 7zip
    try:
        cmd = ['/usr/bin/7z', 'e', f.name, '-o'+ d]
        subprocess.call(cmd)
    except:
        logging.debug("Error unpacking Nullsoft Installer file")
        f.close()
        rmtree(d)
        return []

    # read payload files

    payloads = []

    for file in os.listdir(d):
        filename = os.path.join(d,file)
        payloads.append(open(filename,'rb').read())
        return payloads

    # tidy up
    f.close()
    rmtree(d)

class nullsoft_installer():
    RELEASE = "beta"

    def __init__(self):
        self.name = 'Nullsoft Installer'
        self.description = 'Nullsoft Installer'
        self.description_threatintelligence = ""
        self.malware = 'zcryptor'
        self.category = 'CATEGORY_DROPPER'
        self.payload = set()

    def identify(self, data):
        s = 'EFBEADDE4E756C6C736F6674496E7374'
        ident = s.decode('hex')
        if(data[0,2] == 'MZ' and hex in data):
            return 'PACKER_IDENTIFIED'
            logging.debug("Nullsoft Installer file identified.")
        else:
            return 'PACKER_UNKNOWN'

    def run(self, data):
        payloads = nsis_unpack(data)
        for p in payloads:
            self.payload.add(p)

    def get_payload(self):
        if len(self.payload) == 0:
            return None
        return self.payload.pop()

    def extract_payload(self, fname):
        if len(self.payload) == 0:
            return False
        fd = open(fname, "wb")
        fd.write(self.payload.pop())
        fd.close()
        return True

<commit_msg>Delete nsis_unpack.py<commit_after><|endoftext|>


<commit_before>from django.contrib import admin
from devilry.apps.core.models import AssignmentGroup, Node, Subject, Period
from django.utils.translation import ugettext_lazy as _

class BaseNodeAdmin(admin.ModelAdmin):
    filter_horizontal = ['admins']
    raw_id_fields = [
        'parentnode'
    ]
    list_display = (
        'id',
        'short_name',
        'long_name',
        'admins_as_string'
    )
    search_fields = [
        'id',
        'short_name',
        'long_name',
        'admins__username',
        'admins__email',
        'admins__fullname',
    ]

    def admins_as_string(self, obj):
        return u', '.join([user.username for user in obj.admins.all()])
    admins_as_string.short_description = _("Admins")

    def admins_as_string(self, obj):
        return obj.get_path()
    admins_as_string.short_description = _("Path")

    def get_queryset(self, request):
        return super(BaseNodeAdmin, self).get_queryset(request)\
            .prefetch_related(
                'admins')

class NodeAdmin(BaseNodeAdmin):
    def get_queryset(self, request):
        return super(NodeAdmin, self).get_queryset(request)\
            .select_related('parentnode')\

admin.site.register(Node, NodeAdmin)

class SubjectAdmin(BaseNodeAdmin):
    pass

admin.site.register(Subject, SubjectAdmin)

class PeriodAdmin(BaseNodeAdmin):
    pass

admin.site.register(Period, PeriodAdmin)

class AssignmentGroupAdmin(admin.ModelAdmin):
    list_display = ('id', 'assignment', 'long_displayname')
    search_fields = [
        'id',
    ]
    readonly_fields = [
        'parentnode',
        'feedback',
        'last_deadline'
    ]

    def get_queryset(self, request):
        return super(AssignmentGroupAdmin, self).get_queryset(request)\
            .select_related(
                'parentnode', 'parentnode__parentnode',
                'parentnode__parentnode__parentnode')

    # def admins_as_string(self, obj):
    #     return ', '.join([user.username for user in obj.admins.all()])
    # admins_as_string.short_description = "Admins"

admin.site.register(AssignmentGroup, AssignmentGroupAdmin)
<commit_msg>core...admin: Added admin UI for Assignment.<commit_after>from django.contrib import admin
from devilry.apps.core.models import AssignmentGroup, Node, Subject, Period, Assignment
from django.utils.translation import ugettext_lazy as _

class BaseNodeAdmin(admin.ModelAdmin):
    filter_horizontal = ['admins']
    raw_id_fields = [
        'parentnode'
    ]
    list_display = (
        'id',
        'short_name',
        'long_name',
        'path_as_string',
        'admins_as_string'
    )
    search_fields = [
        'id',
        'short_name',
        'long_name',
        'admins__shortname',
        'admins__fullname',
    ]

    def admins_as_string(self, obj):
        return u', '.join([user.username for user in obj.admins.all()])

    admins_as_string.short_description = _("Admins")

    def path_as_string(self, obj):
        return obj.get_path()

    path_as_string.short_description = _("Path")

    def get_queryset(self, request):
        return super(BaseNodeAdmin, self).get_queryset(request) \
            .prefetch_related(
            'admins')

class NodeAdmin(BaseNodeAdmin):
    def get_queryset(self, request):
        return super(NodeAdmin, self).get_queryset(request) \
            .select_related('parentnode')

admin.site.register(Node, NodeAdmin)

class SubjectAdmin(BaseNodeAdmin):
    pass

admin.site.register(Subject, SubjectAdmin)

class PeriodAdmin(BaseNodeAdmin):
    pass

admin.site.register(Period, PeriodAdmin)

class AssignmentAdmin(BaseNodeAdmin):
    search_fields = BaseNodeAdmin.search_fields + [
        'parentnode__long_name',
        'parentnode__short_name',
        'parentnode__parentnode__long_name',
        'parentnode__parentnode__short_name',
    ]

admin.site.register(Assignment, AssignmentAdmin)

class AssignmentGroupAdmin(admin.ModelAdmin):
    list_display = ('id', 'assignment', 'long_displayname')
    search_fields = [
        'id',
    ]
    readonly_fields = [
        'parentnode',
        'feedback',
        'last_deadline'
    ]

    def get_queryset(self, request):
        return super(AssignmentGroupAdmin, self).get_queryset(request) \
            .select_related(
            'parentnode', 'parentnode__parentnode',
            'parentnode__parentnode__parentnode')

        # def admins_as_string(self, obj):
        #     return ', '.join([user.username for user in obj.admins.all()])
        # admins_as_string.short_description = "Admins"

admin.site.register(AssignmentGroup, AssignmentGroupAdmin)
<|endoftext|>