<commit_before>#!/usr/bin/env python
#
# Copyright 2015 Flavio Garcia
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4:

from __future__ import (absolute_import, division,
                        print_function, with_statement)
import yaml
import os

# Setting root path
ROOT = None
if os.path.abspath(__file__).endswith('.py') or \
        os.path.abspath(__file__).endswith('.pyc'):
    ROOT = os.path.dirname(os.path.abspath(__file__))
else:
    ROOT = os.path.abspath(__file__)

# Getting configuration paths and files from the environment
FIRENADO_CONFIG_FILE = None
try:
    FIRENADO_CONFIG_FILE = os.environ['FIRENADO_CONFIG_FILE']
except KeyError:
    FIRENADO_CONFIG_FILE = 'firenado.yaml'

stack = []

LIB_CONFIG_FILE = os.path.join(ROOT, 'conf', FIRENADO_CONFIG_FILE)

# Application file
APP_CONFIG_ROOT_PATH = os.path.join(os.getcwd())
# If FIRENADO_CURRENT_APP_PATH is not set than return current directory
# conf dir
APP_CONFIG_PATH = os.getenv('FIRENADO_CURRENT_APP_CONFIG_PATH',
                            os.path.join(APP_CONFIG_ROOT_PATH, 'conf'))
APP_CONFIG_FILE = os.path.join(APP_CONFIG_PATH, FIRENADO_CONFIG_FILE)

HAS_LIB_CONFIG_FILE = False
HAS_APP_CONFIG_FILE = False

if os.path.isfile(LIB_CONFIG_FILE):
    HAS_LIB_CONFIG_FILE = True
    stack.append(LIB_CONFIG_FILE)

if os.path.isfile(APP_CONFIG_FILE):
    if APP_CONFIG_FILE != LIB_CONFIG_FILE:
        HAS_APP_CONFIG_FILE = True
        stack.append(APP_CONFIG_FILE)

# Setting firenado's default variables

# Application section configuration
app = {}
# Key to to be used on on the session context to store and retrieve the current
# logged user
app['current_user_key'] = '__FIRENADO_CURRENT_USER_KEY__'
app['data'] = {}
app['data']['sources'] = []
app['pythonpath'] = None
app['port'] = 8888
app['login'] = {}
app['login']['urls'] = {}
app['login']['urls']['default'] = '/login'
app['is_on_dir'] = False

# Component section configutaion
components = {}

# Data section configuration
data = {}
data['connectors'] = {}
data['sources'] = {}

# Management section configuration
management = {}
management['commands'] = {}

def process_config(config):
    """ Populates firenado.conf attributes from the loaded configuration
    dict. It handles data and management aspects from the configuration.

    :param config: Loaded configuration dict
    """
    if 'components' in config:
        process_components_config_section(config['components'])
    if 'data' in config:
        process_data_config_section(config['data'])
    if 'management' in config:
        process_management_config_section(config['management'])

def process_app_config(config):
    """ Populates firenado.conf attributes from the loaded configuration
    dict. It handles everything that process_config does plus application
    configuation.

    :param config: Loaded configuration dict
    """
    process_config(config)
    if 'app' in config:
        process_app_config_section(config['app'])

def process_app_config_section(app_config):
    """ Processes the app section from a configuration dict.

    :param app_config: App section from a config dict
    data.
    """
    global app
    if 'data' in app_config:
        if 'sources' in app_config['data']:
            app['data']['sources'] = app_config['data']['sources']
    if 'pythonpath' in app_config:
        app['pythonpath'] = app_config['pythonpath']
    if 'port' in app_config:
        app['port'] = app_config['port']

def process_components_config_section(components_config):
    """ Processes the components section from a configuration dict.

    :param components_config: Data section from a config dict.
    """
    global components
    for component_config in components_config:
        if 'id' not in component_config:
            raise Exception('The component %s was defined without an id.' %
                            component_config)
        component_id = component_config['id']
        if component_id not in components:
            components[component_id] = {}
            components[component_id]['enabled'] = False
            components[component_id]['config'] = {}
        if 'class' in component_config:
            class_config_x = component_config['class'].split('.')
            components[component_id]['class'] = class_config_x[1]
            components[component_id]['module'] = '.'.join(class_config_x[:-1])
        if 'enabled' in component_config:
            components[component_id]['enabled'] = bool(
                component_config['enabled'])

def process_data_config_section(data_config):
    """ Processes the data configuration section from the configuration
    dict.

    :param data_config: Data configuration section from a config dict.
    """
    global data
    if 'connectors' in data_config:
        for connector in data_config['connectors']:
            connector_class_x = connector['class'].split('.')
            connector['class'] = connector_class_x[-1]
            connector['module'] = '.'.join(connector_class_x[:-1][:])
            data['connectors'][connector['name']] = connector
            del data['connectors'][connector['name']]['name']

def process_management_config_section(management_config):
    """ Processes the management section from a configuration dict.

    :param management_config: Management section from a config dict.
    """
    global management
    if 'commands' in management_config:
        management['commands'] = management_config['commands']

if HAS_LIB_CONFIG_FILE:
    lib_config = yaml.safe_load(file(LIB_CONFIG_FILE, 'r'))
    process_config(lib_config)

if HAS_APP_CONFIG_FILE:
    app_config = yaml.safe_load(file(APP_CONFIG_FILE, 'r'))
    process_app_config(app_config)
<commit_msg>Introduced data sources to the process data confuration function.<commit_after>#!/usr/bin/env python
#
# Copyright 2015 Flavio Garcia
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4:

from __future__ import (absolute_import, division,
                        print_function, with_statement)
import yaml
import os

# Setting root path
ROOT = None
if os.path.abspath(__file__).endswith('.py') or \
        os.path.abspath(__file__).endswith('.pyc'):
    ROOT = os.path.dirname(os.path.abspath(__file__))
else:
    ROOT = os.path.abspath(__file__)

# Getting configuration paths and files from the environment
FIRENADO_CONFIG_FILE = None
try:
    FIRENADO_CONFIG_FILE = os.environ['FIRENADO_CONFIG_FILE']
except KeyError:
    FIRENADO_CONFIG_FILE = 'firenado.yaml'

stack = []

LIB_CONFIG_FILE = os.path.join(ROOT, 'conf', FIRENADO_CONFIG_FILE)

# Application file
APP_CONFIG_ROOT_PATH = os.path.join(os.getcwd())
# If FIRENADO_CURRENT_APP_PATH is not set than return current directory
# conf dir
APP_CONFIG_PATH = os.getenv('FIRENADO_CURRENT_APP_CONFIG_PATH',
                            os.path.join(APP_CONFIG_ROOT_PATH, 'conf'))
APP_CONFIG_FILE = os.path.join(APP_CONFIG_PATH, FIRENADO_CONFIG_FILE)

HAS_LIB_CONFIG_FILE = False
HAS_APP_CONFIG_FILE = False

if os.path.isfile(LIB_CONFIG_FILE):
    HAS_LIB_CONFIG_FILE = True
    stack.append(LIB_CONFIG_FILE)

if os.path.isfile(APP_CONFIG_FILE):
    if APP_CONFIG_FILE != LIB_CONFIG_FILE:
        HAS_APP_CONFIG_FILE = True
        stack.append(APP_CONFIG_FILE)

# Setting firenado's default variables

# Application section configuration
app = {}
# Key to to be used on on the session context to store and retrieve the current
# logged user
app['current_user_key'] = '__FIRENADO_CURRENT_USER_KEY__'
app['data'] = {}
app['data']['sources'] = []
app['pythonpath'] = None
app['port'] = 8888
app['login'] = {}
app['login']['urls'] = {}
app['login']['urls']['default'] = '/login'
app['is_on_dir'] = False

# Component section configutaion
components = {}

# Data section configuration
data = {}
data['connectors'] = {}
data['sources'] = {}

# Management section configuration
management = {}
management['commands'] = {}

def process_config(config):
    """ Populates firenado.conf attributes from the loaded configuration
    dict. It handles data and management aspects from the configuration.

    :param config: Loaded configuration dict
    """
    if 'components' in config:
        process_components_config_section(config['components'])
    if 'data' in config:
        process_data_config_section(config['data'])
    if 'management' in config:
        process_management_config_section(config['management'])

def process_app_config(config):
    """ Populates firenado.conf attributes from the loaded configuration
    dict. It handles everything that process_config does plus application
    configuation.

    :param config: Loaded configuration dict
    """
    process_config(config)
    if 'app' in config:
        process_app_config_section(config['app'])

def process_app_config_section(app_config):
    """ Processes the app section from a configuration dict.

    :param app_config: App section from a config dict
    data.
    """
    global app
    if 'data' in app_config:
        if 'sources' in app_config['data']:
            app['data']['sources'] = app_config['data']['sources']
    if 'pythonpath' in app_config:
        app['pythonpath'] = app_config['pythonpath']
    if 'port' in app_config:
        app['port'] = app_config['port']

def process_components_config_section(components_config):
    """ Processes the components section from a configuration dict.

    :param components_config: Data section from a config dict.
    """
    global components
    for component_config in components_config:
        if 'id' not in component_config:
            raise Exception('The component %s was defined without an id.' %
                            component_config)
        component_id = component_config['id']
        if component_id not in components:
            components[component_id] = {}
            components[component_id]['enabled'] = False
            components[component_id]['config'] = {}
        if 'class' in component_config:
            class_config_x = component_config['class'].split('.')
            components[component_id]['class'] = class_config_x[1]
            components[component_id]['module'] = '.'.join(class_config_x[:-1])
        if 'enabled' in component_config:
            components[component_id]['enabled'] = bool(
                component_config['enabled'])

def process_data_config_section(data_config):
    """ Processes the data configuration section from the configuration
    dict.

    :param data_config: Data configuration section from a config dict.
    """
    global data
    if 'connectors' in data_config:
        for connector in data_config['connectors']:
            connector_class_x = connector['class'].split('.')
            connector['class'] = connector_class_x[-1]
            connector['module'] = '.'.join(connector_class_x[:-1][:])
            data['connectors'][connector['name']] = connector
            del data['connectors'][connector['name']]['name']
    if 'sources' in data_config:
        for source in data_config['sources']:
            data['sources'][source['name']] = source
            del data['sources'][source['name']]['name']

def process_management_config_section(management_config):
    """ Processes the management section from a configuration dict.

    :param management_config: Management section from a config dict.
    """
    global management
    if 'commands' in management_config:
        management['commands'] = management_config['commands']

if HAS_LIB_CONFIG_FILE:
    lib_config = yaml.safe_load(file(LIB_CONFIG_FILE, 'r'))
    process_config(lib_config)

if HAS_APP_CONFIG_FILE:
    app_config = yaml.safe_load(file(APP_CONFIG_FILE, 'r'))
    process_app_config(app_config)
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import print_function
import math
from copy import deepcopy

def get_lineup(players):
    if not isinstance(players, tuple):
        raise TypeError

    n = int(round(math.log(sum(players)) / math.log(2)))
    if 2 ** n != sum(players):
        raise ValueError

    lineup_dict = build_lineup_dict(n)
    lineups = sorted([lineup_dict[n][winner] for winner in ('R', 'P', 'S')])

    for lineup in lineups:
        count = count_players(lineup)
        if players == count:
            return lineup

    return 'IMPOSSIBLE'

def build_lineup_dict(n):
    lineup_dict = dict.fromkeys(range(n+1), {})

    for winner in ('R', 'P', 'S'):
        lineup_dict[0][winner] = winner

    for i in range(1, n+1):
        r = deepcopy(lineup_dict[i-1]['R'])
        p = deepcopy(lineup_dict[i-1]['P'])
        s = deepcopy(lineup_dict[i-1]['S'])

        lineup_dict[i]['R'] = min(r, s) + max(r, s)
        lineup_dict[i]['P'] = min(p, r) + max(p, r)
        lineup_dict[i]['S'] = min(p, s) + max(p , s)

    return lineup_dict

def count_players(lineup):
    r = lineup.count('R')
    p = lineup.count('P')
    s = lineup.count('S')
    return r, p, s

if __name__ == '__main__':
    import os

    samples = [
        (1,1,0),
        (2,0,0),
        (1,1,2),
        (2,0,2)
    ]

    for sample in samples:
        print(get_lineup(sample))

    data_files = ['A-small-practice', 'A-large-practice']
    for f in data_files:
        with open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                  '{0}.in'.format(f)), 'r') as input_file:
            lines = input_file.readlines()
        input_count = int(lines[0].replace('\n' ,''))
        inputs = [line.replace('\n', '') for line in lines[1:]]

        i = 1
        with open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                  '{0}.out'.format(f)), 'w') as output_file:
            for in_ in inputs:
                players = tuple([int(_) for _ in in_.split(' ')[1:]])
                output_file.write('Case #{0}: {1}\n'.format(i, get_lineup(players)))
                i += 1
<commit_msg>Remove space<commit_after>#!/usr/bin/env python

from __future__ import print_function
import math
from copy import deepcopy

def get_lineup(players):
    if not isinstance(players, tuple):
        raise TypeError

    n = int(round(math.log(sum(players)) / math.log(2)))
    if 2 ** n != sum(players):
        raise ValueError

    lineup_dict = build_lineup_dict(n)
    lineups = sorted([lineup_dict[n][winner] for winner in ('R', 'P', 'S')])

    for lineup in lineups:
        count = count_players(lineup)
        if players == count:
            return lineup

    return 'IMPOSSIBLE'

def build_lineup_dict(n):
    lineup_dict = dict.fromkeys(range(n+1), {})

    for winner in ('R', 'P', 'S'):
        lineup_dict[0][winner] = winner

    for i in range(1, n+1):
        r = deepcopy(lineup_dict[i-1]['R'])
        p = deepcopy(lineup_dict[i-1]['P'])
        s = deepcopy(lineup_dict[i-1]['S'])

        lineup_dict[i]['R'] = min(r, s) + max(r, s)
        lineup_dict[i]['P'] = min(p, r) + max(p, r)
        lineup_dict[i]['S'] = min(p, s) + max(p, s)

    return lineup_dict

def count_players(lineup):
    r = lineup.count('R')
    p = lineup.count('P')
    s = lineup.count('S')
    return r, p, s

if __name__ == '__main__':
    import os

    samples = [
        (1,1,0),
        (2,0,0),
        (1,1,2),
        (2,0,2)
    ]

    for sample in samples:
        print(get_lineup(sample))

    data_files = ['A-small-practice', 'A-large-practice']
    for f in data_files:
        with open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                  '{0}.in'.format(f)), 'r') as input_file:
            lines = input_file.readlines()
        input_count = int(lines[0].replace('\n' ,''))
        inputs = [line.replace('\n', '') for line in lines[1:]]

        i = 1
        with open(os.path.join(os.path.dirname(os.path.realpath(__file__)),
                  '{0}.out'.format(f)), 'w') as output_file:
            for in_ in inputs:
                players = tuple([int(_) for _ in in_.split(' ')[1:]])
                output_file.write('Case #{0}: {1}\n'.format(i, get_lineup(players)))
                i += 1
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2012-2015 Spotify AB
# Ported to Airflow by Bolke de Bruin
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This is a port of Luigi's ssh implementation. All credits go there.
import subprocess
from contextlib import contextmanager

from airflow.hooks.base_hook import BaseHook
from airflow.exceptions import AirflowException

import logging

class SSHHook(BaseHook):
    """
    Light-weight remote execution library and utilities.

    Using this hook (which is just a convenience wrapper for subprocess),
    is created to let you stream data from a remotely stored file.

    As a bonus, :class:`SSHHook` also provides a really cool feature that let's you
    set up ssh tunnels super easily using a python context manager (there is an example
    in the integration part of unittests).

    :param key_file: Typically the SSHHook uses the keys that are used by the user
        airflow is running under. This sets the behavior to use another file instead.
    :type key_file: str
    :param connect_timeout: sets the connection timeout for this connection.
    :type connect_timeout: int
    :param no_host_key_check: whether to check to host key. If True host keys will not
        be checked, but are also not stored in the current users's known_hosts file.
    :type no_host_key_check: bool
    :param tty: allocate a tty.
    :type tty: bool
    :param sshpass: Use to non-interactively perform password authentication by using
        sshpass.
    :type sshpass: bool
    """
    def __init__(self, conn_id='ssh_default'):
        conn = self.get_connection(conn_id)
        self.key_file = conn.extra_dejson.get('key_file', None)
        self.connect_timeout = conn.extra_dejson.get('connect_timeout', None)
        self.no_host_key_check = conn.extra_dejson.get('no_host_key_check', False)
        self.tty = conn.extra_dejson.get('tty', False)
        self.sshpass = conn.extra_dejson.get('sshpass', False)
        self.conn = conn

    def get_conn(self):
        pass

    def _host_ref(self):
        if self.conn.login:
            return "{0}@{1}".format(self.conn.login, self.conn.host)
        else:
            return self.conn.host

    def _prepare_command(self, cmd):
        connection_cmd = ["ssh", self._host_ref(), "-o", "ControlMaster=no"]
        if self.sshpass:
            connection_cmd = ["sshpass", "-e"] + connection_cmd
        else:
            connection_cmd += ["-o", "BatchMode=yes"]  # no password prompts

        if self.conn.port:
            connection_cmd += ["-p", str(self.conn.port)]

        if self.connect_timeout:
            connection_cmd += ["-o", "ConnectionTimeout={}".format(self.connect_timeout)]

        if self.no_host_key_check:
            connection_cmd += ["-o", "UserKnownHostsFile=/dev/null",
                               "-o", "StrictHostKeyChecking=no"]

        if self.key_file:
            connection_cmd += ["-i", self.key_file]

        if self.tty:
            connection_cmd += ["-t"]

        connection_cmd += cmd
        logging.debug("SSH cmd: {} ".format(connection_cmd))

        return connection_cmd

    def Popen(self, cmd, **kwargs):
        """
        Remote Popen

        :param cmd: command to remotely execute
        :param kwargs: extra arguments to Popen (see subprocess.Popen)
        :return: handle to subprocess
        """
        prefixed_cmd = self._prepare_command(cmd)
        return subprocess.Popen(prefixed_cmd, **kwargs)

    def check_output(self, cmd):
        """
        Executes a remote command and returns the stdout a remote process.
        Simplified version of Popen when you only want the output as a string and detect any errors.

        :param cmd: command to remotely execute
        :return: stdout
        """
        p = self.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        output, stderr = p.communicate()

        if p.returncode != 0:
            # I like this better: RemoteCalledProcessError(p.returncode, cmd, self.host, output=output)
            raise AirflowException("Cannot execute {} on {}. Error code is: {}. Output: {}, Stderr: {}".format(
                                   cmd, self.conn.host, p.returncode, output, stderr))

        return output

    @contextmanager
    def tunnel(self, local_port, remote_port=None, remote_host="localhost"):
        """
        Creates a tunnel between two hosts. Like ssh -L <LOCAL_PORT>:host:<REMOTE_PORT>.
        Remember to close() the returned "tunnel" object in order to clean up
        after yourself when you are done with the tunnel.

        :param local_port:
        :type local_port: int
        :param remote_port:
        :type remote_port: int
        :param remote_host:
        :type remote_host: str
        :return:
        """
        tunnel_host = "{0}:{1}:{2}".format(local_port, remote_host, remote_port)
        proc = self.Popen(["-L", tunnel_host, "echo -n ready && cat"],
                          stdin=subprocess.PIPE, stdout=subprocess.PIPE)

        ready = proc.stdout.read(5)
        assert ready == b"ready", "Did not get 'ready' from remote"
        yield
        proc.communicate()
        assert proc.returncode == 0, "Tunnel process did unclean exit (returncode {}".format(proc.returncode)

<commit_msg>[AIRFLOW-691] Add SSH KeepAlive option to SSH_hook<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2012-2015 Spotify AB
# Ported to Airflow by Bolke de Bruin
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This is a port of Luigi's ssh implementation. All credits go there.
import subprocess
from contextlib import contextmanager

from airflow.hooks.base_hook import BaseHook
from airflow.exceptions import AirflowException

import logging

class SSHHook(BaseHook):
    """
    Light-weight remote execution library and utilities.

    Using this hook (which is just a convenience wrapper for subprocess),
    is created to let you stream data from a remotely stored file.

    As a bonus, :class:`SSHHook` also provides a really cool feature that let's you
    set up ssh tunnels super easily using a python context manager (there is an example
    in the integration part of unittests).

    :param key_file: Typically the SSHHook uses the keys that are used by the user
        airflow is running under. This sets the behavior to use another file instead.
    :type key_file: str
    :param connect_timeout: sets the connection timeout for this connection.
    :type connect_timeout: int
    :param no_host_key_check: whether to check to host key. If True host keys will not
        be checked, but are also not stored in the current users's known_hosts file.
    :type no_host_key_check: bool
    :param tty: allocate a tty.
    :type tty: bool
    :param sshpass: Use to non-interactively perform password authentication by using
        sshpass.
    :type sshpass: bool
    """
    def __init__(self, conn_id='ssh_default'):
        conn = self.get_connection(conn_id)
        self.key_file = conn.extra_dejson.get('key_file', None)
        self.connect_timeout = conn.extra_dejson.get('connect_timeout', None)
        self.tcp_keepalive = conn.extra_dejson.get('tcp_keepalive', False)
        self.server_alive_interval = conn.extra_dejson.get('server_alive_interval', 60)
        self.no_host_key_check = conn.extra_dejson.get('no_host_key_check', False)
        self.tty = conn.extra_dejson.get('tty', False)
        self.sshpass = conn.extra_dejson.get('sshpass', False)
        self.conn = conn

    def get_conn(self):
        pass

    def _host_ref(self):
        if self.conn.login:
            return "{0}@{1}".format(self.conn.login, self.conn.host)
        else:
            return self.conn.host

    def _prepare_command(self, cmd):
        connection_cmd = ["ssh", self._host_ref(), "-o", "ControlMaster=no"]
        if self.sshpass:
            connection_cmd = ["sshpass", "-e"] + connection_cmd
        else:
            connection_cmd += ["-o", "BatchMode=yes"]  # no password prompts

        if self.conn.port:
            connection_cmd += ["-p", str(self.conn.port)]

        if self.connect_timeout:
            connection_cmd += ["-o", "ConnectionTimeout={}".format(self.connect_timeout)]

        if self.tcp_keepalive:
            connection_cmd += ["-o", "TCPKeepAlive=yes"]
            connection_cmd += ["-o", "ServerAliveInterval={}".format(self.server_alive_interval)]

        if self.no_host_key_check:
            connection_cmd += ["-o", "UserKnownHostsFile=/dev/null",
                               "-o", "StrictHostKeyChecking=no"]

        if self.key_file:
            connection_cmd += ["-i", self.key_file]

        if self.tty:
            connection_cmd += ["-t"]

        connection_cmd += cmd
        logging.debug("SSH cmd: {} ".format(connection_cmd))

        return connection_cmd

    def Popen(self, cmd, **kwargs):
        """
        Remote Popen

        :param cmd: command to remotely execute
        :param kwargs: extra arguments to Popen (see subprocess.Popen)
        :return: handle to subprocess
        """
        prefixed_cmd = self._prepare_command(cmd)
        return subprocess.Popen(prefixed_cmd, **kwargs)

    def check_output(self, cmd):
        """
        Executes a remote command and returns the stdout a remote process.
        Simplified version of Popen when you only want the output as a string and detect any errors.

        :param cmd: command to remotely execute
        :return: stdout
        """
        p = self.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        output, stderr = p.communicate()

        if p.returncode != 0:
            # I like this better: RemoteCalledProcessError(p.returncode, cmd, self.host, output=output)
            raise AirflowException("Cannot execute {} on {}. Error code is: {}. Output: {}, Stderr: {}".format(
                                   cmd, self.conn.host, p.returncode, output, stderr))

        return output

    @contextmanager
    def tunnel(self, local_port, remote_port=None, remote_host="localhost"):
        """
        Creates a tunnel between two hosts. Like ssh -L <LOCAL_PORT>:host:<REMOTE_PORT>.
        Remember to close() the returned "tunnel" object in order to clean up
        after yourself when you are done with the tunnel.

        :param local_port:
        :type local_port: int
        :param remote_port:
        :type remote_port: int
        :param remote_host:
        :type remote_host: str
        :return:
        """
        tunnel_host = "{0}:{1}:{2}".format(local_port, remote_host, remote_port)
        proc = self.Popen(["-L", tunnel_host, "echo -n ready && cat"],
                          stdin=subprocess.PIPE, stdout=subprocess.PIPE)

        ready = proc.stdout.read(5)
        assert ready == b"ready", "Did not get 'ready' from remote"
        yield
        proc.communicate()
        assert proc.returncode == 0, "Tunnel process did unclean exit (returncode {}".format(proc.returncode)

<|endoftext|>


<commit_before>#!/usr/bin/python
from lxml import etree
from StringIO import StringIO

from sfa.util.faults import InvalidXML

class XpathFilter:
    @staticmethod
    def xpath(filter={}):
        xpath = ""
        if filter:
            filter_list = []
            for (key, value) in filter.items():
                if key == 'text':
                    key = 'text()'
                else:
                    key = '@'+key
                if isinstance(value, str):
                    filter_list.append('%s="%s"' % (key, value))
                elif isinstance(value, list):
                    filter_list.append('contains("%s", %s)' % (' '.join(map(str, value)), key))
            if filter_list:
                xpath = ' and '.join(filter_list)
                xpath = '[' + xpath + ']'
        return xpath

class XML:

    def __init__(self, xml=None):
        self.root = None
        self.namespaces = None
        self.default_namespace = None
        self.schema = None
        if isinstance(xml, basestring):
            self.parse_xml(xml)
        elif isinstance(xml, etree._ElementTree):
            self.root = xml.getroot()
        elif isinstance(xml, etree._Element):
            self.root = xml

    def parse_xml(self, xml):
        """
        parse rspec into etree
        """
        parser = etree.XMLParser(remove_blank_text=True)
        try:
            tree = etree.parse(xml, parser)
        except IOError:
            # 'rspec' file doesnt exist. 'rspec' is proably an xml string
            try:
                tree = etree.parse(StringIO(xml), parser)
            except Exception, e:
                raise InvalidXML(str(e))
        self.root = tree.getroot()
        # set namespaces map
        self.namespaces = dict(self.root.nsmap)
        if 'default' not in self.namespaces and None in self.namespaces:
            # If the 'None' exist, then it's pointing to the default namespace. This makes
            # it hard for us to write xpath queries for the default naemspace because lxml
            # wont understand a None prefix. We will just associate the default namespeace
            # with a key named 'default'.
            self.namespaces['default'] = self.namespaces[None]
        else:
            self.namespaces['default'] = 'default'

        # set schema
        for key in self.root.attrib.keys():
            if key.endswith('schemaLocation'):
                # schema location should be at the end of the list
                schema_parts  = self.root.attrib[key].split(' ')
                self.schema = schema_parts[1]
                namespace, schema  = schema_parts[0], schema_parts[1]
                break

    def parse_dict(self, d, root_tag_name='xml', element = None):
        if element is None:
            if self.root is None:
                self.parse_xml('<%s/>' % root_tag_name)
            element = self.root

        if 'text' in d:
            text = d.pop('text')
            element.text = text

        # handle repeating fields
        for (key, value) in d.items():
            if isinstance(value, list):
                value = d.pop(key)
                for val in value:
                    if isinstance(val, dict):
                        child_element = etree.SubElement(element, key)
                        self.parse_dict(val, key, child_element)
                    elif isinstance(val, basestring):
                        child_element = etree.SubElement(element, key).text = val

            elif isinstance(value, int):
                d[key] = unicode(d[key])
            elif value is None:
                d.pop(key)

        element.attrib.update(d)

    def validate(self, schema):
        """
        Validate against rng schema
        """
        relaxng_doc = etree.parse(schema)
        relaxng = etree.RelaxNG(relaxng_doc)
        if not relaxng(self.root):
            error = relaxng.error_log.last_error
            message = "%s (line %s)" % (error.message, error.line)
            raise InvalidXML(message)
        return True

    def xpath(self, xpath, namespaces=None):
        if not namespaces:
            namespaces = self.namespaces
        return self.root.xpath(xpath, namespaces=namespaces)

    def set(self, key, value):
        return self.root.set(key, value)

    def add_attribute(self, elem, name, value):
        """
        Add attribute to specified etree element
        """
        opt = etree.SubElement(elem, name)
        opt.text = value

    def add_element(self, name, attrs={}, parent=None, text=""):
        """
        Generic wrapper around etree.SubElement(). Adds an element to
        specified parent node. Adds element to root node is parent is
        not specified.
        """
        if parent == None:
            parent = self.root
        element = etree.SubElement(parent, name)
        if text:
            element.text = text
        if isinstance(attrs, dict):
            for attr in attrs:
                element.set(attr, attrs[attr])
        return element

    def remove_attribute(self, elem, name, value):
        """
        Removes an attribute from an element
        """
        if elem is not None:
            opts = elem.iterfind(name)
            if opts is not None:
                for opt in opts:
                    if opt.text == value:
                        elem.remove(opt)

    def remove_element(self, element_name, root_node = None):
        """
        Removes all occurences of an element from the tree. Start at
        specified root_node if specified, otherwise start at tree's root.
        """
        if not root_node:
            root_node = self.root

        if not element_name.startswith('//'):
            element_name = '//' + element_name

        elements = root_node.xpath('%s ' % element_name, namespaces=self.namespaces)
        for element in elements:
            parent = element.getparent()
            parent.remove(element)

    def attributes_list(self, elem):
        # convert a list of attribute tags into list of tuples
        # (tagnme, text_value)
        opts = []
        if elem is not None:
            for e in elem:
                opts.append((e.tag, str(e.text).strip()))
        return opts

    def get_element_attributes(self, elem=None, depth=0):
        if elem == None:
            elem = self.root_node
        if not hasattr(elem, 'attrib'):
            # this is probably not an element node with attribute. could be just and an
            # attribute, return it
            return elem
        attrs = dict(elem.attrib)
        attrs['text'] = str(elem.text).strip()
        attrs['parent'] = elem.getparent()
        if isinstance(depth, int) and depth > 0:
            for child_elem in list(elem):
                key = str(child_elem.tag)
                if key not in attrs:
                    attrs[key] = [self.get_element_attributes(child_elem, depth-1)]
                else:
                    attrs[key].append(self.get_element_attributes(child_elem, depth-1))
        else:
            attrs['child_nodes'] = list(elem)
        return attrs

    def merge(self, in_xml):
        pass

    def __str__(self):
        return self.toxml()

    def toxml(self):
        return etree.tostring(self.root, encoding='UTF-8', pretty_print=True)

    def todict(self, elem=None):
        if elem is None:
            elem = self.root
        d = {}
        d.update(elem.attrib)
        d['text'] = elem.text
        for child in elem.iterchildren():
            if child.tag not in d:
                d[child.tag] = []
            d[child.tag].append(self.todict(child))
        return d

    def save(self, filename):
        f = open(filename, 'w')
        f.write(self.toxml())
        f.close()

# no RSpec in scope
#if __name__ == '__main__':
#    rspec = RSpec('/tmp/resources.rspec')
#    print rspec

<commit_msg>fix error with datetime structs in parse_dict<commit_after>#!/usr/bin/python
from lxml import etree
from StringIO import StringIO

from sfa.util.faults import InvalidXML

class XpathFilter:
    @staticmethod
    def xpath(filter={}):
        xpath = ""
        if filter:
            filter_list = []
            for (key, value) in filter.items():
                if key == 'text':
                    key = 'text()'
                else:
                    key = '@'+key
                if isinstance(value, str):
                    filter_list.append('%s="%s"' % (key, value))
                elif isinstance(value, list):
                    filter_list.append('contains("%s", %s)' % (' '.join(map(str, value)), key))
            if filter_list:
                xpath = ' and '.join(filter_list)
                xpath = '[' + xpath + ']'
        return xpath

class XML:

    def __init__(self, xml=None):
        self.root = None
        self.namespaces = None
        self.default_namespace = None
        self.schema = None
        if isinstance(xml, basestring):
            self.parse_xml(xml)
        elif isinstance(xml, etree._ElementTree):
            self.root = xml.getroot()
        elif isinstance(xml, etree._Element):
            self.root = xml

    def parse_xml(self, xml):
        """
        parse rspec into etree
        """
        parser = etree.XMLParser(remove_blank_text=True)
        try:
            tree = etree.parse(xml, parser)
        except IOError:
            # 'rspec' file doesnt exist. 'rspec' is proably an xml string
            try:
                tree = etree.parse(StringIO(xml), parser)
            except Exception, e:
                raise InvalidXML(str(e))
        self.root = tree.getroot()
        # set namespaces map
        self.namespaces = dict(self.root.nsmap)
        if 'default' not in self.namespaces and None in self.namespaces:
            # If the 'None' exist, then it's pointing to the default namespace. This makes
            # it hard for us to write xpath queries for the default naemspace because lxml
            # wont understand a None prefix. We will just associate the default namespeace
            # with a key named 'default'.
            self.namespaces['default'] = self.namespaces[None]
        else:
            self.namespaces['default'] = 'default'

        # set schema
        for key in self.root.attrib.keys():
            if key.endswith('schemaLocation'):
                # schema location should be at the end of the list
                schema_parts  = self.root.attrib[key].split(' ')
                self.schema = schema_parts[1]
                namespace, schema  = schema_parts[0], schema_parts[1]
                break

    def parse_dict(self, d, root_tag_name='xml', element = None):
        if element is None:
            if self.root is None:
                self.parse_xml('<%s/>' % root_tag_name)
            element = self.root

        if 'text' in d:
            text = d.pop('text')
            element.text = text

        # handle repeating fields
        for (key, value) in d.items():
            if isinstance(value, list):
                value = d.pop(key)
                for val in value:
                    if isinstance(val, dict):
                        child_element = etree.SubElement(element, key)
                        self.parse_dict(val, key, child_element)
                    elif isinstance(val, basestring):
                        child_element = etree.SubElement(element, key).text = val

            elif isinstance(value, int):
                d[key] = unicode(d[key])
            elif value is None:
                d.pop(key)

        # element.attrib.update will explode if DateTimes are in the
        # dcitionary.
        d=d.copy()
        for k in d.keys():
            if (type(d[k]) != str) and (type(d[k]) != unicode):
                del d[k]

        element.attrib.update(d)

    def validate(self, schema):
        """
        Validate against rng schema
        """
        relaxng_doc = etree.parse(schema)
        relaxng = etree.RelaxNG(relaxng_doc)
        if not relaxng(self.root):
            error = relaxng.error_log.last_error
            message = "%s (line %s)" % (error.message, error.line)
            raise InvalidXML(message)
        return True

    def xpath(self, xpath, namespaces=None):
        if not namespaces:
            namespaces = self.namespaces
        return self.root.xpath(xpath, namespaces=namespaces)

    def set(self, key, value):
        return self.root.set(key, value)

    def add_attribute(self, elem, name, value):
        """
        Add attribute to specified etree element
        """
        opt = etree.SubElement(elem, name)
        opt.text = value

    def add_element(self, name, attrs={}, parent=None, text=""):
        """
        Generic wrapper around etree.SubElement(). Adds an element to
        specified parent node. Adds element to root node is parent is
        not specified.
        """
        if parent == None:
            parent = self.root
        element = etree.SubElement(parent, name)
        if text:
            element.text = text
        if isinstance(attrs, dict):
            for attr in attrs:
                element.set(attr, attrs[attr])
        return element

    def remove_attribute(self, elem, name, value):
        """
        Removes an attribute from an element
        """
        if elem is not None:
            opts = elem.iterfind(name)
            if opts is not None:
                for opt in opts:
                    if opt.text == value:
                        elem.remove(opt)

    def remove_element(self, element_name, root_node = None):
        """
        Removes all occurences of an element from the tree. Start at
        specified root_node if specified, otherwise start at tree's root.
        """
        if not root_node:
            root_node = self.root

        if not element_name.startswith('//'):
            element_name = '//' + element_name

        elements = root_node.xpath('%s ' % element_name, namespaces=self.namespaces)
        for element in elements:
            parent = element.getparent()
            parent.remove(element)

    def attributes_list(self, elem):
        # convert a list of attribute tags into list of tuples
        # (tagnme, text_value)
        opts = []
        if elem is not None:
            for e in elem:
                opts.append((e.tag, str(e.text).strip()))
        return opts

    def get_element_attributes(self, elem=None, depth=0):
        if elem == None:
            elem = self.root_node
        if not hasattr(elem, 'attrib'):
            # this is probably not an element node with attribute. could be just and an
            # attribute, return it
            return elem
        attrs = dict(elem.attrib)
        attrs['text'] = str(elem.text).strip()
        attrs['parent'] = elem.getparent()
        if isinstance(depth, int) and depth > 0:
            for child_elem in list(elem):
                key = str(child_elem.tag)
                if key not in attrs:
                    attrs[key] = [self.get_element_attributes(child_elem, depth-1)]
                else:
                    attrs[key].append(self.get_element_attributes(child_elem, depth-1))
        else:
            attrs['child_nodes'] = list(elem)
        return attrs

    def merge(self, in_xml):
        pass

    def __str__(self):
        return self.toxml()

    def toxml(self):
        return etree.tostring(self.root, encoding='UTF-8', pretty_print=True)

    def todict(self, elem=None):
        if elem is None:
            elem = self.root
        d = {}
        d.update(elem.attrib)
        d['text'] = elem.text
        for child in elem.iterchildren():
            if child.tag not in d:
                d[child.tag] = []
            d[child.tag].append(self.todict(child))
        return d

    def save(self, filename):
        f = open(filename, 'w')
        f.write(self.toxml())
        f.close()

# no RSpec in scope
#if __name__ == '__main__':
#    rspec = RSpec('/tmp/resources.rspec')
#    print rspec

<|endoftext|>


<commit_before><commit_msg>Attempt to gracefully handle failed MLE PCA component selection<commit_after><|endoftext|>


<commit_before># Filip Keri
# Average_Epochs.py
# A script that takes in a .par file, a .tim file, and an output directory
# as a result, it creates a new file with TOAs compressed in epochs, stored in output directory
# sample input:
# python Average_Epochs.py /Users/fkeri/Desktop/B1855+09_NANOGrav_9yv0.par /Users/fkeri/Desktop/B1855+09_NANOGrav_9yv0.tim /Users/fkeri/Desktop/
# we can see that it takes in 3 line arguments: [INPUT .par FILE], [INPUT .tim FILE], [OUTPUT DIRECTORY]
# the output file will have the same name as the input file, with "Output_for_plotting_" as a prefix: "Output_for_plotting_B1855+09_NANOGrav_9yv0.tim"
# it is possible to name the output file differently by putting the file name in [OUTPUT DIRECTORY]: /Users/fkeri/Desktop/filename.tim

import sys
import math
import numpy as N
import matplotlib.pyplot as P
import libstempo as T

def freq_idx( F ):
	freq=[ [100, 360], [361, 600], [601, 999], [1000, 2100], [2101, 3000] ]
	for i in range( 5 ):
		if ( F >= freq[ i ][ 0 ] ) and ( F <= freq[ i ][ 1 ] ):
			return i
	return 0

F = [ 350, 450, 800, 1400, 2300 ]

output_dir = str(sys.argv[ 3 ])
psr = T.tempopulsar( parfile = str( sys.argv[ 1 ] ), timfile = str( sys.argv[ 2 ] ) )
sort_cmp = N.argsort( psr.stoas )
bin = [] # format: residuals, error, TOA, frequency

# sorting residuals, errors, frequencies, and TOAs by TOAs
Residuals = psr.residuals()[ sort_cmp ]
Errors = psr.toaerrs[ sort_cmp ]
TOA = psr.stoas[ sort_cmp ]
FREQ = psr.freqs[ sort_cmp ]

curr = [ Residuals[ 0 ], Errors[ 0 ], TOA[ 0 ], FREQ[ 0 ] ]
for i in range( 1, len( TOA ) ):
	if( TOA[ i ]-TOA[ i-1 ] > (3.0/144.0) ):
		bin.append( curr )
		curr = []
	curr.append( Residuals[ i ] )
	curr.append( Errors[ i ] )
	curr.append( TOA[ i ] )
	curr.append( FREQ[ i ] )
if len( curr ) > 0:
	bin.append( curr )

avgResidual = []
avgError = []
avgTOA = []
avgFREQ = []

for i in range( len( bin ) ):
	sum1 = [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
	sum2 = [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
	sum3 = [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
	sumTOA = [ 0, 0, 0, 0, 0 ]
	for j in range( 0, len( bin[ i ] ), 4 ):
		currRes = bin[ i ][ j ]
		currErr = bin[ i ][ j+1 ]
		currTOA = bin[ i ][ j+2 ]
		idx = freq_idx( bin[ i ][ j+3 ] )
		sum1[ idx ] += currRes/( currErr**2 )
		sum2[ idx ] += 1/( currErr**2 )
		sum3[ idx ] += currTOA
		sumTOA[ idx ] += 1
	for j in range( 5 ):
		if sumTOA[ j ] > 0:
			avgResidual.append( sum1[ j ]/sum2[ j ] )
			avgError.append( math.sqrt( 1/sum2[ j ] ) )
			avgTOA.append( sum3[ j ]/sumTOA[ j ] )
			avgFREQ.append( F[ j ] )

pulsar_path = str( sys.argv[ 1 ] )
pulsar_file = pulsar_path.split('/')
pulsar_name = pulsar_file[-1].split('.')

if sys.argv[3][-4] != '.':
    outFile = open( os.path.join( str(sys.argv[ 3 ]), "Output_for_plotting_"+pulsar_name[0]+".tim", "w" )
else:
    outFile = open( sys.argv[3], "w" )

outFile.write( psr.name+"\n" )
outFile.write( "This program comes with ABSOLUTELY NO WARRANTY.\nThis is free software, and you are welcome to redistribute it\n" )
outFile.write( "under conditions of GPL license.\n\n\n\n" )

for i in range( len( avgTOA ) ):
	outFile.write( str("{0:.15f}".format(avgTOA[ i ]))+"\t"+str("{0:.19f}".format(avgResidual[ i ]))+"\t"+str(avgFREQ[ i ])+"\t"+str("{0:.19f}".format(avgError[ i ]))+"\n" )

outFile.close()
<commit_msg>add import and add missing parentheses<commit_after># Filip Keri
# Average_Epochs.py
# A script that takes in a .par file, a .tim file, and an output directory
# as a result, it creates a new file with TOAs compressed in epochs, stored in output directory
# sample input:
# python Average_Epochs.py /Users/fkeri/Desktop/B1855+09_NANOGrav_9yv0.par /Users/fkeri/Desktop/B1855+09_NANOGrav_9yv0.tim /Users/fkeri/Desktop/
# we can see that it takes in 3 line arguments: [INPUT .par FILE], [INPUT .tim FILE], [OUTPUT DIRECTORY]
# the output file will have the same name as the input file, with "Output_for_plotting_" as a prefix: "Output_for_plotting_B1855+09_NANOGrav_9yv0.tim"
# it is possible to name the output file differently by putting the file name in [OUTPUT DIRECTORY]: /Users/fkeri/Desktop/filename.tim

import os
import sys
import math
import numpy as N
import matplotlib.pyplot as P
import libstempo as T

def freq_idx( F ):
	freq=[ [100, 360], [361, 600], [601, 999], [1000, 2100], [2101, 3000] ]
	for i in range( 5 ):
		if ( F >= freq[ i ][ 0 ] ) and ( F <= freq[ i ][ 1 ] ):
			return i
	return 0

F = [ 350, 450, 800, 1400, 2300 ]

output_dir = str(sys.argv[ 3 ])
psr = T.tempopulsar( parfile = str( sys.argv[ 1 ] ), timfile = str( sys.argv[ 2 ] ) )
sort_cmp = N.argsort( psr.stoas )
bin = [] # format: residuals, error, TOA, frequency

# sorting residuals, errors, frequencies, and TOAs by TOAs
Residuals = psr.residuals()[ sort_cmp ]
Errors = psr.toaerrs[ sort_cmp ]
TOA = psr.stoas[ sort_cmp ]
FREQ = psr.freqs[ sort_cmp ]

curr = [ Residuals[ 0 ], Errors[ 0 ], TOA[ 0 ], FREQ[ 0 ] ]
for i in range( 1, len( TOA ) ):
	if( TOA[ i ]-TOA[ i-1 ] > (3.0/144.0) ):
		bin.append( curr )
		curr = []
	curr.append( Residuals[ i ] )
	curr.append( Errors[ i ] )
	curr.append( TOA[ i ] )
	curr.append( FREQ[ i ] )
if len( curr ) > 0:
	bin.append( curr )

avgResidual = []
avgError = []
avgTOA = []
avgFREQ = []

for i in range( len( bin ) ):
	sum1 = [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
	sum2 = [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
	sum3 = [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
	sumTOA = [ 0, 0, 0, 0, 0 ]
	for j in range( 0, len( bin[ i ] ), 4 ):
		currRes = bin[ i ][ j ]
		currErr = bin[ i ][ j+1 ]
		currTOA = bin[ i ][ j+2 ]
		idx = freq_idx( bin[ i ][ j+3 ] )
		sum1[ idx ] += currRes/( currErr**2 )
		sum2[ idx ] += 1/( currErr**2 )
		sum3[ idx ] += currTOA
		sumTOA[ idx ] += 1
	for j in range( 5 ):
		if sumTOA[ j ] > 0:
			avgResidual.append( sum1[ j ]/sum2[ j ] )
			avgError.append( math.sqrt( 1/sum2[ j ] ) )
			avgTOA.append( sum3[ j ]/sumTOA[ j ] )
			avgFREQ.append( F[ j ] )

pulsar_path = str( sys.argv[ 1 ] )
pulsar_file = pulsar_path.split('/')
pulsar_name = pulsar_file[-1].split('.')

if sys.argv[3][-4] != '.':
    outFile = open( os.path.join( str(sys.argv[ 3 ]), "Output_for_plotting_"+pulsar_name[0]+".tim"), "w" )
else:
    outFile = open( sys.argv[3], "w" )

outFile.write( psr.name+"\n" )
outFile.write( "This program comes with ABSOLUTELY NO WARRANTY.\nThis is free software, and you are welcome to redistribute it\n" )
outFile.write( "under conditions of GPL license.\n\n\n\n" )

for i in range( len( avgTOA ) ):
	outFile.write( str("{0:.15f}".format(avgTOA[ i ]))+"\t"+str("{0:.19f}".format(avgResidual[ i ]))+"\t"+str(avgFREQ[ i ])+"\t"+str("{0:.19f}".format(avgError[ i ]))+"\n" )

outFile.close()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# A Sphinx extension to generate list of bug descriptions from findbugs.xml and messages.xml.
# It is necessary to generate localized list, because Sphinx i18n feature based on .po file
# does not support translating raw HTML block.

from xml.etree.ElementTree import *
import codecs

def generate_category_title(bug_category):
    return "%s (%s)" % (bug_category.findtext('.//Description'), bug_category.get('category'))

def generate_pattern_title(bug_pattern, message):
    return "%s: %s (%s)" % (bug_pattern.get('abbrev'), message.findtext('.//ShortDescription'), bug_pattern.get('type'))

def generate_bug_description(language):
    print("Generating bug description page for %s..." % language)
    findbugs = parse('../spotbugs/etc/findbugs.xml')
    if language == 'ja':
        messages = parse('../spotbugs/etc/messages_ja.xml')
    else:
        messages = parse('../spotbugs/etc/messages.xml')

    with codecs.open('generated/bugDescriptionList.inc', 'w', encoding='UTF-8') as bug_description_page:
        for bug_category in sorted(messages.getiterator('BugCategory'), key=lambda element: element.get('category')):
            category = bug_category.get('category')
            category_title = generate_category_title(bug_category)
            bug_description_page.write(category_title)
            bug_description_page.write('\n')
            bug_description_page.write('-' * len(category_title))
            bug_description_page.write('\n\n')
            for line in bug_category.findtext('.//Details').splitlines():
                bug_description_page.write(line.strip())
                bug_description_page.write('\n')
            bug_description_page.write('\n\n')

            for bug_pattern in findbugs.findall(".//BugPattern[@category='%s']" % category):
                if (bug_pattern.get('deprecated') == 'true'):
                    continue
                type = bug_pattern.get('type')
                bug_description_page.write(".. _%s:\n\n" % type)
                message = messages.find(".//BugPattern[@type='%s']" % type)
                pattern_title = generate_pattern_title(bug_pattern, message)
                bug_description_page.write("%s\n%s\n\n" % (pattern_title, '^' * len(pattern_title)))
                details = message.findtext('.//Details')
                bug_description_page.write('.. raw:: html\n')
                for line in details.splitlines():
                    bug_description_page.write('  ')
                    bug_description_page.write(line)
                    bug_description_page.write('\n')
                bug_description_page.write('\n')

def setup(app):
    app.connect('builder-inited', lambda app: generate_bug_description(app.config.language))
<commit_msg>close #430: use raw-block instead of explicit link target<commit_after># -*- coding: utf-8 -*-
# A Sphinx extension to generate list of bug descriptions from findbugs.xml and messages.xml.
# It is necessary to generate localized list, because Sphinx i18n feature based on .po file
# does not support translating raw HTML block.

from xml.etree.ElementTree import *
import codecs

def generate_category_title(bug_category):
    return "%s (%s)" % (bug_category.findtext('.//Description'), bug_category.get('category'))

def generate_pattern_title(bug_pattern, message):
    return "%s: %s (%s)" % (bug_pattern.get('abbrev'), message.findtext('.//ShortDescription'), bug_pattern.get('type'))

def generate_bug_description(language):
    print("Generating bug description page for %s..." % language)
    findbugs = parse('../spotbugs/etc/findbugs.xml')
    if language == 'ja':
        messages = parse('../spotbugs/etc/messages_ja.xml')
    else:
        messages = parse('../spotbugs/etc/messages.xml')

    with codecs.open('generated/bugDescriptionList.inc', 'w', encoding='UTF-8') as bug_description_page:
        for bug_category in sorted(messages.getiterator('BugCategory'), key=lambda element: element.get('category')):
            category = bug_category.get('category')
            category_title = generate_category_title(bug_category)
            bug_description_page.write(category_title)
            bug_description_page.write('\n')
            bug_description_page.write('-' * len(category_title))
            bug_description_page.write('\n\n')
            for line in bug_category.findtext('.//Details').splitlines():
                bug_description_page.write(line.strip())
                bug_description_page.write('\n')
            bug_description_page.write('\n\n')

            for bug_pattern in findbugs.findall(".//BugPattern[@category='%s']" % category):
                if (bug_pattern.get('deprecated') == 'true'):
                    continue
                type = bug_pattern.get('type')
                bug_description_page.write(".. raw:: html\n\n  \n\n" % type)
                message = messages.find(".//BugPattern[@type='%s']" % type)
                pattern_title = generate_pattern_title(bug_pattern, message)
                bug_description_page.write("%s\n%s\n\n" % (pattern_title, '^' * len(pattern_title)))
                details = message.findtext('.//Details')
                bug_description_page.write('.. raw:: html\n')
                for line in details.splitlines():
                    bug_description_page.write('  ')
                    bug_description_page.write(line)
                    bug_description_page.write('\n')
                bug_description_page.write('\n')

def setup(app):
    app.connect('builder-inited', lambda app: generate_bug_description(app.config.language))
<|endoftext|>


<commit_before>import urllib

from django.forms.models import modelform_factory
from django.shortcuts import get_object_or_404
from django.http import QueryDict, HttpResponse, HttpResponseNotFound, HttpResponseBadRequest
from django.template.defaultfilters import linebreaksbr
from django.utils import simplejson, translation
from django.views.decorators.http import require_POST
from django.views.decorators.csrf import csrf_exempt

from dajax.core import Dajax
from dajaxice.core import dajaxice_functions

from tagging.models import Tag
from tagging.utils import get_tag_list, parse_tag_input

from .models import I4pProject, I4pProjectTranslation
from .forms import I4pProjectObjectiveForm, I4pProjectThemesForm
from .utils import get_or_create_project, get_or_create_project_translation_by_slug, get_project_translation_by_slug

TEXTFIELD_MAPPINGS = {
    'about_section_txt': 'about_section',
    'project_uniqueness_txt': 'uniqueness_section',
    'project_value_txt': 'value_section',
    'project_scalability_txt': 'scalability_section'
    }

def project_textfield_load(request, project_slug=None):
    """
    Load the source of a text field (project description, ...),
    without the markup rendered (useful for JS editing, to prevent
    html tags from being edited without interpretation).
    """
    if not project_slug:
        return HttpResponse('')

    try:
        language_code = request.GET['language_code']
        section = request.GET['id']
    except KeyError:
        raise HttpResponseBadRequest()

    # Check if we allow this field
    if not section in TEXTFIELD_MAPPINGS:
        return HttpResponseNotFound()

    # Activate requested language
    translation.activate(language_code)

    # get the project translation and its base
    project_translation = get_project_translation_by_slug(project_translation_slug=project_slug,
                                                          language_code=language_code)

    # Get the text
    text = getattr(project_translation, TEXTFIELD_MAPPINGS[section])

    return HttpResponse(text or '')

@require_POST
@csrf_exempt
def project_textfield_save(request, project_slug=None):
    """
    Edit a text field
    """
    language_code = request.POST['language_code']
    section = request.POST['id']
    value = request.POST['value']

    # Activate requested language
    translation.activate(language_code)

    project_translation = get_or_create_project_translation_by_slug(project_translation_slug=project_slug,
                                                                    language_code=language_code)

    # Check if we allow this field
    if not section in TEXTFIELD_MAPPINGS:
        return HttpResponseNotFound()

    # Resolve the fieldname
    fieldname = TEXTFIELD_MAPPINGS[section]

    FieldForm = modelform_factory(I4pProjectTranslation, fields=(fieldname,))

    form = FieldForm({fieldname: value}, instance=project_translation)

    if form.is_valid():
        form.save()
        return HttpResponse(simplejson.dumps({'text': linebreaksbr(value),
                                              'redirect': project_slug is None,
                                              'redirect_url': project_translation.get_absolute_url()}), 'application/json')
    else:
        return HttpResponseNotFound()

def project_update_related(request, language_code, related_form, project_slug):
    """
    Update themes and objective of a given project, in a given language
    """
    # Activate requested language
    translation.activate(language_code)

    # get the project translation and its base
    project_translation = get_project_translation_by_slug(project_translation_slug=project_slug,
                                                          language_code=language_code)

    parent_project = project_translation.project

    project_objective_form = I4pProjectObjectiveForm(related_form,
                                                     instance=parent_project)

    # Convert tags to string list, separated by comma
    if isinstance(related_form['themes'], list):
        related_form['themes'] = ", ".join(related_form['themes'])

    project_themes_form = I4pProjectThemesForm(related_form,
                                               instance=project_translation)

    if project_themes_form.is_valid() and project_objective_form.is_valid():
        # Use Tag otherwise it doesn't work because of the proxy model
        project_themes_form.save()
        #Tag.objects.update_tags(project_sheet.get_translation(), related_form['themes'])

        # Save objective
        project_objective_form.save()

    return simplejson.dumps({})

#dajaxice_functions.register(project_sheet_edit_field)
dajaxice_functions.register(project_update_related)

<commit_msg>use return instead of raise (colon dumbass period).<commit_after>import urllib

from django.forms.models import modelform_factory
from django.shortcuts import get_object_or_404
from django.http import QueryDict, HttpResponse, HttpResponseNotFound, HttpResponseBadRequest
from django.template.defaultfilters import linebreaksbr
from django.utils import simplejson, translation
from django.views.decorators.http import require_POST
from django.views.decorators.csrf import csrf_exempt

from dajax.core import Dajax
from dajaxice.core import dajaxice_functions

from tagging.models import Tag
from tagging.utils import get_tag_list, parse_tag_input

from .models import I4pProject, I4pProjectTranslation
from .forms import I4pProjectObjectiveForm, I4pProjectThemesForm
from .utils import get_or_create_project, get_or_create_project_translation_by_slug, get_project_translation_by_slug

TEXTFIELD_MAPPINGS = {
    'about_section_txt': 'about_section',
    'project_uniqueness_txt': 'uniqueness_section',
    'project_value_txt': 'value_section',
    'project_scalability_txt': 'scalability_section'
    }

def project_textfield_load(request, project_slug=None):
    """
    Load the source of a text field (project description, ...),
    without the markup rendered (useful for JS editing, to prevent
    html tags from being edited without interpretation).
    """
    if not project_slug:
        return HttpResponse('')

    try:
        language_code = request.GET['language_code']
        section = request.GET['id']
    except KeyError:
        return HttpResponseBadRequest()

    # Check if we allow this field
    if not section in TEXTFIELD_MAPPINGS:
        return HttpResponseNotFound()

    # Activate requested language
    translation.activate(language_code)

    # get the project translation and its base
    project_translation = get_project_translation_by_slug(project_translation_slug=project_slug,
                                                          language_code=language_code)

    # Get the text
    text = getattr(project_translation, TEXTFIELD_MAPPINGS[section])

    return HttpResponse(text or '')

@require_POST
@csrf_exempt
def project_textfield_save(request, project_slug=None):
    """
    Edit a text field
    """
    language_code = request.POST['language_code']
    section = request.POST['id']
    value = request.POST['value']

    # Activate requested language
    translation.activate(language_code)

    project_translation = get_or_create_project_translation_by_slug(project_translation_slug=project_slug,
                                                                    language_code=language_code)

    # Check if we allow this field
    if not section in TEXTFIELD_MAPPINGS:
        return HttpResponseNotFound()

    # Resolve the fieldname
    fieldname = TEXTFIELD_MAPPINGS[section]

    FieldForm = modelform_factory(I4pProjectTranslation, fields=(fieldname,))

    form = FieldForm({fieldname: value}, instance=project_translation)

    if form.is_valid():
        form.save()
        return HttpResponse(simplejson.dumps({'text': linebreaksbr(value),
                                              'redirect': project_slug is None,
                                              'redirect_url': project_translation.get_absolute_url()}), 'application/json')
    else:
        return HttpResponseNotFound()

def project_update_related(request, language_code, related_form, project_slug):
    """
    Update themes and objective of a given project, in a given language
    """
    # Activate requested language
    translation.activate(language_code)

    # get the project translation and its base
    project_translation = get_project_translation_by_slug(project_translation_slug=project_slug,
                                                          language_code=language_code)

    parent_project = project_translation.project

    project_objective_form = I4pProjectObjectiveForm(related_form,
                                                     instance=parent_project)

    # Convert tags to string list, separated by comma
    if isinstance(related_form['themes'], list):
        related_form['themes'] = ", ".join(related_form['themes'])

    project_themes_form = I4pProjectThemesForm(related_form,
                                               instance=project_translation)

    if project_themes_form.is_valid() and project_objective_form.is_valid():
        # Use Tag otherwise it doesn't work because of the proxy model
        project_themes_form.save()
        #Tag.objects.update_tags(project_sheet.get_translation(), related_form['themes'])

        # Save objective
        project_objective_form.save()

    return simplejson.dumps({})

#dajaxice_functions.register(project_sheet_edit_field)
dajaxice_functions.register(project_update_related)

<|endoftext|>


<commit_before># Reading and processing tabular file formats
#
# Copyright (c) 2016 Aubrey Barnard.  This is free software released
# under the MIT License.  See `LICENSE.txt` for details.

import io
import pathlib
import re

from . import records

class Format:
    pass

class DelimitedText(records.RecordStream):
    pass

# Functional file API

def open(file, mode='rt', compression='auto'):
    """Open a file with optional (de)compression.

    file: Filename (str) or pathlib.Path object

    mode: Mode as in the standard io.open

    compression: Compression type.  Expects one of {None, 'auto',
        'gzip', 'bzip2', 'lzma', <extension>:str}.  None means do not
        perform any compression, 'auto' means detect the type of
        compression from the filename extension, and the others specify
        the specific compression to use, either as the compression name
        or a commonly-associated filename extension.

    """
    # Check and convert arguments
    path = None
    if isinstance(file, str):
        path = pathlib.Path(file)
    elif isinstance(file, pathlib.Path):
        path = file
    else:
        raise TypeError('Not a file: {}'.format(file))
    # Detect compression type by extension
    auto = (compression == 'auto')
    if auto:
        compression = path.suffix.lstrip('.')
        if not compression:
            raise ValueError(
                'Cannot detect compression type from filename '
                'extension: {}'.format(path.suffix))

    # Open file based on compression type.  Only import modules as
    # needed.  (I know buried imports are bad style, but it makes this
    # function more compatible with older Pythons that didn't have
    # support for various compressions in the standard library.)
    filename = str(path)
    if compression is None:
        return io.open(filename, mode=mode)
    # Use gzip for all common Lempel-Ziv compression suffixes
    elif compression in ('gz', 'z', 'Z', 'gzip'):
        import gzip
        return gzip.open(filename, mode=mode)
    elif compression in ('bz2', 'bzip2'):
        import bz2
        return bz2.open(filename, mode=mode)
    elif compression in ('xz', 'lzma'):
        import lzma
        return lzma.open(filename, mode=mode)
    elif auto:
        # No compression detected
        return io.open(filename, mode=mode)
    else:
        raise ValueError(
            'Unrecognized compression type: {}'.format(compression))

def read_lines(file, compression='auto'):
    """Open a text file for reading lines and automatically close when done.

    Returns an iterator over lines after opening the file with
    `open(file, mode='rt', compression)`.  See `open` in this module for
    details.

    """
    with open(file, mode='rt', compression=compression) as lines:
        for line in lines:
            yield line
    # File closes as soon as reading is finished

def read_content_lines(
        file,
        compression='auto',
        comment_char='#',
        skip_blank_lines=True,
        enumerate_lines=False,
        ):
    """Read the non-comment, non-blank lines of a file.

    Returns an iterator over the non-comment, non-blank lines of a file
    with optional line numbers.

    file: Filename (str), pathlib.Path object, or iterable of strings
        (e.g. open text file)

    compression: Compression type.  See `open` in this module.

    comment_char: Initial character (after whitespace) that identifies
        comment lines

    skip_blank_lines: Whether to skip blank lines or treat them as
        records

    enumerate_lines: Whether to return (line number, line) pairs.

    """
    # Open file if needed
    if isinstance(file, (str, pathlib.Path)):
        file = read_lines(file, compression=compression)
    # Otherwise assume open file or other iterable of lines

    # Create pattern for detecting comments.  Quote the comment
    # character to avoid regex injection.
    comment_pattern = re.compile('\s*' + re.escape(comment_char))

    # Iterate over lines in the file.  Line numbers start at 1.
    line_num = 0
    for line in file:
        line_num += 1
        # Ignore comment lines
        if comment_pattern.match(line) is not None:
            continue
        # Ignore blank lines (if desired)
        elif skip_blank_lines and line.isspace():
            continue
        # Return content lines with or without line numbers
        elif enumerate_lines:
            yield (line_num, line)
        else:
            yield line
<commit_msg>Implemented `read_content_lines` functionality as a class.<commit_after># Reading and processing tabular file formats
#
# Copyright (c) 2016 Aubrey Barnard.  This is free software released
# under the MIT License.  See `LICENSE.txt` for details.

import io
import pathlib
import re

from . import records

class Format:
    pass

class DelimitedText(records.RecordStream):
    pass

# Functional file API

def open(file, mode='rt', compression='auto'):
    """Open a file with optional (de)compression.

    file: Filename (str) or pathlib.Path object

    mode: Mode as in the standard io.open

    compression: Compression type.  Expects one of {None, 'auto',
        'gzip', 'bzip2', 'lzma', <extension>:str}.  None means do not
        perform any compression, 'auto' means detect the type of
        compression from the filename extension, and the others specify
        the specific compression to use, either as the compression name
        or a commonly-associated filename extension.

    """
    # Check and convert arguments
    path = None
    if isinstance(file, str):
        path = pathlib.Path(file)
    elif isinstance(file, pathlib.Path):
        path = file
    else:
        raise TypeError('Not a file: {}'.format(file))
    # Detect compression type by extension
    auto = (compression == 'auto')
    if auto:
        compression = path.suffix.lstrip('.')
        if not compression:
            raise ValueError(
                'Cannot detect compression type from filename '
                'extension: {}'.format(path.suffix))

    # Open file based on compression type.  Only import modules as
    # needed.  (I know buried imports are bad style, but it makes this
    # function more compatible with older Pythons that didn't have
    # support for various compressions in the standard library.)
    filename = str(path)
    if compression is None:
        return io.open(filename, mode=mode)
    # Use gzip for all common Lempel-Ziv compression suffixes
    elif compression in ('gz', 'z', 'Z', 'gzip'):
        import gzip
        return gzip.open(filename, mode=mode)
    elif compression in ('bz2', 'bzip2'):
        import bz2
        return bz2.open(filename, mode=mode)
    elif compression in ('xz', 'lzma'):
        import lzma
        return lzma.open(filename, mode=mode)
    elif auto:
        # No compression detected
        return io.open(filename, mode=mode)
    else:
        raise ValueError(
            'Unrecognized compression type: {}'.format(compression))

def read_lines(file, compression='auto'):
    """Open a text file for reading lines and automatically close when done.

    Returns an iterator over lines after opening the file with
    `open(file, mode='rt', compression)`.  See `open` in this module for
    details.

    """
    with open(file, mode='rt', compression=compression) as lines:
        for line in lines:
            yield line
    # File closes as soon as reading is finished

def read_content_lines(
        file,
        compression='auto',
        comment_char='#',
        skip_blank_lines=True,
        enumerate_lines=False,
        ):
    """Read the non-comment, non-blank lines of a file.

    Returns an iterator over the non-comment, non-blank lines of a file
    with optional line numbers.

    file: Filename (str), pathlib.Path object, or iterable of strings
        (e.g. open text file)

    compression: Compression type.  See `open` in this module.

    comment_char: Initial character (after whitespace) that identifies
        comment lines

    skip_blank_lines: Whether to skip blank lines or treat them as
        records

    enumerate_lines: Whether to return (line number, line) pairs.

    """
    # Open file if needed
    if isinstance(file, (str, pathlib.Path)):
        file = read_lines(file, compression=compression)
    # Otherwise assume open file or other iterable of lines

    # Create pattern for detecting comments.  Quote the comment
    # character to avoid regex injection.
    comment_pattern = re.compile('\s*' + re.escape(comment_char))

    # Iterate over lines in the file.  Line numbers start at 1.
    line_num = 0
    for line in file:
        line_num += 1
        # Ignore comment lines
        if comment_pattern.match(line) is not None:
            continue
        # Ignore blank lines (if desired)
        elif skip_blank_lines and line.isspace():
            continue
        # Return content lines with or without line numbers
        elif enumerate_lines:
            yield (line_num, line)
        else:
            yield line

class ContentReader:
    """Reader for the non-comment, non-blank lines of a file.

    This is basically a class version of `read_content_lines`.  The
    difference is that the reader exposes the line number as a property
    whereas it is part of the iterable values in `read_content_lines`.

    """

    def __init__(
            self,
            file,
            compression='auto',
            comment_char='#',
            skip_blank_lines=True,
            ):
        """Create a new ContentReader.

        file: Filename (str), pathlib.Path object, or iterable of
            strings (e.g. open text file)

        compression: Compression type.  See `open` in this module.

        comment_char: Initial character (after whitespace) that
            identifies comment lines

        skip_blank_lines: Whether to skip blank lines or treat them as
            records

        """
        self._file = file
        self._compression = compression
        self._comment_char = comment_char
        self._skip_blank_lines = skip_blank_lines
        self._line_num = 0

    def __iter__(self):
        """Return an iterator over non-comment, non-blank lines."""
        # Open file if needed
        if isinstance(self._file, (str, pathlib.Path)):
            self._file = read_lines(
                self._file, compression=self._compression)
        # Otherwise assume open file or other iterable of lines

        # Create pattern for detecting comments.  Quote the comment
        # character to avoid regex injection.
        comment_pattern = re.compile('\s*' + re.escape(self._comment_char))

        # Iterate over lines in the file.  Line numbers start at 1.
        self._line_num = 0
        for line in self._file:
            self._line_num += 1
            # Ignore comment lines
            if comment_pattern.match(line) is not None:
                continue
            # Ignore blank lines (if desired)
            elif self._skip_blank_lines and line.isspace():
                continue
            # Return content lines
            else:
                yield line

    @property
    def line_num(self):
        return self._line_num
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from fractions import Fraction

def vnmax(features):
    return sum([max([j['value'] for j in i['enum']]) for i in features])

def calculate_coeficient(features, parameters):
    vmax = Fraction(vnmax(features))
    vn = sum([Fraction(i['value']) for i in parameters])
    return 1 + vn / (1 - vmax)

def cooking(price, features=None, parameters=None):
    """
    MEAT price normalization

    features = [
        {
            "code": "OCDS-123454-AIR-INTAKE",
            "description": "\u0415\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u0430 \u043f\u043e\u0442\u0443\u0436\u043d\u0456\u0441\u0442\u044c \u0432\u0441\u043c\u043e\u043a\u0442\u0443\u0432\u0430\u043d\u043d\u044f \u043f\u0438\u043b\u043e\u0441\u043e\u0441\u0430,\u0432 \u0432\u0430\u0442\u0430\u0445 (\u0430\u0435\u0440\u043e\u0432\u0430\u0442\u0430\u0445)",
            "title": "\u041f\u043e\u0442\u0443\u0436\u043d\u0456\u0441\u0442\u044c \u0432\u0441\u043c\u043e\u043a\u0442\u0443\u0432\u0430\u043d\u043d\u044f",
            "enum": [
                {
                    "value": 0.1,
                    "title": "\u0414\u043e 1000 \u0412\u0442"
                },
                {
                    "value": 0.15,
                    "title": "\u0411\u0456\u043b\u044c\u0448\u0435 1000 \u0412\u0442"
                }
            ],
            "title_en": "Air Intake",
            "relatedItem": "123456",
            "featureOf": "item"
        },
        {
            "code": "OCDS-123454-YEARS",
            "description": "\u041a\u0456\u043b\u044c\u043a\u0456\u0441\u0442\u044c \u0440\u043e\u043a\u0456\u0432, \u044f\u043a\u0456 \u043e\u0440\u0433\u0430\u043d\u0456\u0437\u0430\u0446\u0456\u044f \u0443\u0447\u0430\u0441\u043d\u0438\u043a \u043f\u0440\u0430\u0446\u044e\u0454 \u043d\u0430 \u0440\u0438\u043d\u043a\u0443",
            "title": "\u0420\u043e\u043a\u0456\u0432 \u043d\u0430 \u0440\u0438\u043d\u043a\u0443",
            "enum": [
                {
                    "value": 0.05,
                    "title": "\u0414\u043e 3 \u0440\u043e\u043a\u0456\u0432"
                },
                {
                    "value": 0.1,
                    "title": "\u0412\u0456\u0434 3 \u0434\u043e 5 \u0440\u043e\u043a\u0456\u0432"
                },
                {
                    "value": 0.15,
                    "title": "\u0411\u0456\u043b\u044c\u0448\u0435 5 \u0440\u043e\u043a\u0456\u0432"
                }
            ],
            "title_en": "Years trading",
            "featureOf": "tenderer"
        }
    ]

    parameters = [
        {
            "code": "OCDS-123454-AIR-INTAKE",
            "value": 0.1
        },
        {
            "code": "OCDS-123454-YEARS",
            "value": 0.1
        }
    ]

    coef = 1 + Fraction(1, 5) / (1 - Fraction(3, 10)) = Fraction(9, 7)
    """
    if not features or not parameters:
        return price
    coef = calculate_coeficient(features, parameters)
    return Fraction(price) * coef

def chef(bids, features=None, ignore=[]):
    """
    MEAT bids sorting
    """
    sorted_bids = sorted(bids, key=lambda i: (cooking(i['value']['amount'], features, i['parameters']), i['date']))
    return [i for i in sorted_bids if i['id'] not in ignore]
<commit_msg>Fixed getting params<commit_after># -*- coding: utf-8 -*-

from fractions import Fraction

def vnmax(features):
    return sum([max([j['value'] for j in i['enum']]) for i in features])

def calculate_coeficient(features, parameters):
    vmax = Fraction(vnmax(features))
    vn = sum([Fraction(i['value']) for i in parameters])
    return 1 + vn / (1 - vmax)

def cooking(price, features=None, parameters=None):
    """
    MEAT price normalization

    features = [
        {
            "code": "OCDS-123454-AIR-INTAKE",
            "description": "\u0415\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u0430 \u043f\u043e\u0442\u0443\u0436\u043d\u0456\u0441\u0442\u044c \u0432\u0441\u043c\u043e\u043a\u0442\u0443\u0432\u0430\u043d\u043d\u044f \u043f\u0438\u043b\u043e\u0441\u043e\u0441\u0430,\u0432 \u0432\u0430\u0442\u0430\u0445 (\u0430\u0435\u0440\u043e\u0432\u0430\u0442\u0430\u0445)",
            "title": "\u041f\u043e\u0442\u0443\u0436\u043d\u0456\u0441\u0442\u044c \u0432\u0441\u043c\u043e\u043a\u0442\u0443\u0432\u0430\u043d\u043d\u044f",
            "enum": [
                {
                    "value": 0.1,
                    "title": "\u0414\u043e 1000 \u0412\u0442"
                },
                {
                    "value": 0.15,
                    "title": "\u0411\u0456\u043b\u044c\u0448\u0435 1000 \u0412\u0442"
                }
            ],
            "title_en": "Air Intake",
            "relatedItem": "123456",
            "featureOf": "item"
        },
        {
            "code": "OCDS-123454-YEARS",
            "description": "\u041a\u0456\u043b\u044c\u043a\u0456\u0441\u0442\u044c \u0440\u043e\u043a\u0456\u0432, \u044f\u043a\u0456 \u043e\u0440\u0433\u0430\u043d\u0456\u0437\u0430\u0446\u0456\u044f \u0443\u0447\u0430\u0441\u043d\u0438\u043a \u043f\u0440\u0430\u0446\u044e\u0454 \u043d\u0430 \u0440\u0438\u043d\u043a\u0443",
            "title": "\u0420\u043e\u043a\u0456\u0432 \u043d\u0430 \u0440\u0438\u043d\u043a\u0443",
            "enum": [
                {
                    "value": 0.05,
                    "title": "\u0414\u043e 3 \u0440\u043e\u043a\u0456\u0432"
                },
                {
                    "value": 0.1,
                    "title": "\u0412\u0456\u0434 3 \u0434\u043e 5 \u0440\u043e\u043a\u0456\u0432"
                },
                {
                    "value": 0.15,
                    "title": "\u0411\u0456\u043b\u044c\u0448\u0435 5 \u0440\u043e\u043a\u0456\u0432"
                }
            ],
            "title_en": "Years trading",
            "featureOf": "tenderer"
        }
    ]

    parameters = [
        {
            "code": "OCDS-123454-AIR-INTAKE",
            "value": 0.1
        },
        {
            "code": "OCDS-123454-YEARS",
            "value": 0.1
        }
    ]

    coef = 1 + Fraction(1, 5) / (1 - Fraction(3, 10)) = Fraction(9, 7)
    """
    if not features or not parameters:
        return price
    coef = calculate_coeficient(features, parameters)
    return Fraction(price) * coef

def chef(bids, features=None, ignore=[]):
    """
    MEAT bids sorting
    """
    sorted_bids = sorted(bids, key=lambda i: (
        cooking(i['value']['amount'], features, i.get('parameters', [])), i['date'])
    )
    return [i for i in sorted_bids if i['id'] not in ignore]
<|endoftext|>


<commit_before>import os
import logging
from flask import current_app

from aleph.core import db, archive, celery
from aleph.core import USER_QUEUE, USER_ROUTING_KEY
from aleph.core import WORKER_QUEUE, WORKER_ROUTING_KEY
from aleph.model import Document, Role
from aleph.ingest.manager import DocumentManager
from aleph.notify import notify_role_template

log = logging.getLogger(__name__)

def get_manager():
    """Get an ingestor manager, as a singleton instance."""
    if not hasattr(DocumentManager, '_instance'):
        DocumentManager._instance = DocumentManager(current_app.config,
                                                    archive)
        log.info("Loaded ingestors: %r", DocumentManager._instance.ingestors)
    return DocumentManager._instance

def ingest_document(document, file_path, role_id=None):
    """Given a stub document and file path, extract information.
    This does not attempt to infer metadata such as a file name."""
    document.status = Document.STATUS_PENDING
    if os.path.isdir(file_path):
        manager = get_manager()
        manager.ingest_document(document, file_path=file_path, role_id=role_id)
    else:
        ch = archive.archive_file(file_path,
                                  content_hash=document.content_hash)
        document.content_hash = ch or document.content_hash
        db.session.commit()
        if role_id is not None:
            queue = USER_QUEUE
            routing_key = USER_ROUTING_KEY
        else:
            queue = WORKER_QUEUE
            routing_key = WORKER_ROUTING_KEY
        ingest.apply_async(args=[document.id],
                           kwargs=dict(role_id=role_id),
                           queue=queue,
                           routing_key=routing_key)

@celery.task()
def ingest(document_id, role_id=None):
    """Process a given document by extracting its contents.
    This may include creating or updating child documents."""
    document = Document.by_id(document_id)
    if document is None:
        log.error("Could not find document: %s", document_id)
        return

    get_manager().ingest_document(document, role_id=role_id)

    pending = Document.pending_count(collection_id=document.collection.id)
    if pending == 0:
        ingest_complete(document.collection, role_id=role_id)

def ingest_complete(collection, role_id=None):
    """Operations supposed to be performed when an ingest process completes."""
    from aleph.logic.collections import update_collection, collection_url  # noqa
    update_collection(collection)
    role = Role.by_id(role_id)
    if role is not None:
        # notify the user that their import is completed.
        notify_role_template(role,
                             collection.label,
                             'email/ingest.html',
                             collection=collection,
                             url=collection_url(collection.id))
<commit_msg>dont list ingestors upon boot<commit_after>import os
import logging
from flask import current_app

from aleph.core import db, archive, celery
from aleph.core import USER_QUEUE, USER_ROUTING_KEY
from aleph.core import WORKER_QUEUE, WORKER_ROUTING_KEY
from aleph.model import Document, Role
from aleph.ingest.manager import DocumentManager
from aleph.notify import notify_role_template

log = logging.getLogger(__name__)

def get_manager():
    """Get an ingestor manager, as a singleton instance."""
    if not hasattr(DocumentManager, '_instance'):
        DocumentManager._instance = DocumentManager(current_app.config,
                                                    archive)
        # log.info("Loaded ingestors: %r", DocumentManager._instance.ingestors)
    return DocumentManager._instance

def ingest_document(document, file_path, role_id=None):
    """Given a stub document and file path, extract information.
    This does not attempt to infer metadata such as a file name."""
    document.status = Document.STATUS_PENDING
    if os.path.isdir(file_path):
        manager = get_manager()
        manager.ingest_document(document, file_path=file_path, role_id=role_id)
    else:
        ch = archive.archive_file(file_path,
                                  content_hash=document.content_hash)
        document.content_hash = ch or document.content_hash
        db.session.commit()
        queue = WORKER_QUEUE
        routing_key = WORKER_ROUTING_KEY
        if role_id is not None:
            queue = USER_QUEUE
            routing_key = USER_ROUTING_KEY
        ingest.apply_async(args=[document.id],
                           kwargs={'role_id': role_id},
                           queue=queue,
                           routing_key=routing_key)

@celery.task()
def ingest(document_id, role_id=None):
    """Process a given document by extracting its contents.
    This may include creating or updating child documents."""
    document = Document.by_id(document_id)
    if document is None:
        log.error("Could not find document: %s", document_id)
        return

    get_manager().ingest_document(document, role_id=role_id)

    pending = Document.pending_count(collection_id=document.collection.id)
    if pending == 0:
        ingest_complete(document.collection, role_id=role_id)

def ingest_complete(collection, role_id=None):
    """Operations supposed to be performed when an ingest process completes."""
    from aleph.logic.collections import update_collection, collection_url  # noqa
    update_collection(collection)
    role = Role.by_id(role_id)
    if role is not None:
        # notify the user that their import is completed.
        notify_role_template(role,
                             collection.label,
                             'email/ingest.html',
                             collection=collection,
                             url=collection_url(collection.id))
<|endoftext|>