<commit_before>import random
from contextlib import contextmanager
from urllib import urlencode

from django.conf import settings
import sqlalchemy
from sqlalchemy.orm.scoping import scoped_session
from sqlalchemy.orm.session import sessionmaker
from django.core import signals

DEFAULT_ENGINE_ID = 'default'
UCR_ENGINE_ID = 'ucr'
ICDS_UCR_ENGINE_ID = 'icds-ucr'
ICDS_TEST_UCR_ENGINE_ID = 'icds-test-ucr'

def create_engine(connection_string):
    # paramstyle='format' allows you to use column names that include the ')' character
    # otherwise queries will sometimes be misformated/error when formatting
    # https://github.com/zzzeek/sqlalchemy/blob/ff20903/lib/sqlalchemy/dialects/postgresql/psycopg2.py#L173
    return sqlalchemy.create_engine(connection_string, paramstyle='format')

class SessionHelper(object):
    """
    Shim class helper for a single connection/session factory
    """

    def __init__(self, connection_string):
        self.engine = create_engine(connection_string)
        self._session_factory = sessionmaker(bind=self.engine)
        # Session is the actual constructor object
        self.Session = scoped_session(self._session_factory)

    @property
    def session_context(self):
        @contextmanager
        def session_scope():
            """Provide a transactional scope around a series of operations."""
            session = self.Session()
            try:
                yield session
                session.commit()
            except:
                session.rollback()
                raise
            finally:
                session.close()

        return session_scope

class ConnectionManager(object):
    """
    Object for dealing with sqlalchemy engines and sessions.
    """

    def __init__(self):
        self._session_helpers = {}
        self.db_connection_map = {}
        self.read_database_mapping = {}
        self._populate_connection_map()

    def _get_or_create_helper(self, engine_id):
        if engine_id not in self._session_helpers:
            self._session_helpers[engine_id] = SessionHelper(self.get_connection_string(engine_id))
        return self._session_helpers[engine_id]

    def get_session_helper(self, engine_id=DEFAULT_ENGINE_ID):
        """
        Returns the SessionHelper object associated with this engine id
        """
        # making this separate from _get_or_create in case we ever want to fail differently here
        return self._get_or_create_helper(engine_id)

    def get_scoped_session(self, engine_id=DEFAULT_ENGINE_ID):
        """
        This returns a handle to a thread-locally scoped session object.
        """
        return self.get_session_helper(engine_id).Session

    def get_engine(self, engine_id=DEFAULT_ENGINE_ID):
        """
        Get an engine by ID. This should be a unique field identifying the connection,
        e.g. "report-db-1"
        """
        return self._get_or_create_helper(engine_id).engine

    def get_load_balanced_read_engine_id(self, engine_id):
        read_dbs = self.read_database_mapping.get(engine_id, [])
        if read_dbs:
            return random.choice(read_dbs)
        return engine_id

    def close_scoped_sessions(self):
        for helper in self._session_helpers.values():
            helper.Session.remove()

    def dispose_engine(self, engine_id=DEFAULT_ENGINE_ID):
        """
        If found, closes the active sessions associated with an an engine and disposes it.
        Also removes it from the session manager.
        If not found, does nothing.
        """
        if engine_id in self._session_helpers:
            helper = self._session_helpers.pop(engine_id)
            helper.Session.remove()
            helper.engine.dispose()

    def dispose_all(self):
        """
        Dispose all engines associated with this. Useful for tests.
        """
        for engine_id in self._session_helpers.keys():
            self.dispose_engine(engine_id)

    def get_connection_string(self, engine_id):
        connection_string = self.db_connection_map.get(engine_id)
        return connection_string or self.db_connection_map['default']

    def _populate_connection_map(self):
        reporting_db_config = self._get_reporting_db_config()
        if not reporting_db_config:
            self._populate_from_legacy_settings()
        else:
            for engine_id, db_config in reporting_db_config.items():
                write_db = db_config
                read = None
                if isinstance(db_config, dict):
                    write_db = db_config['WRITE']
                    read = db_config['READ']
                    for db_alias, weighting in read:
                        assert isinstance(weighting, int), 'weighting must be int'
                        assert db_alias in settings.DATABASES, db_alias

                self._add_django_db(engine_id, write_db)
                if read:
                    self.read_database_mapping[engine_id] = []
                    for read_db, weighting in read:
                        assert read_db == write_db or read_db not in self.db_connection_map, read_db
                        self.read_database_mapping[engine_id].extend([read_db] * weighting)
                        if read_db != write_db:
                            self._add_django_db(read_db, read_db)

        if DEFAULT_ENGINE_ID not in self.db_connection_map:
            self._add_django_db(DEFAULT_ENGINE_ID, 'default')
        if UCR_ENGINE_ID not in self.db_connection_map:
            self._add_django_db(UCR_ENGINE_ID, 'default')

        self._add_django_db_from_settings_key(ICDS_UCR_ENGINE_ID, 'ICDS_UCR_DATABASE_ALIAS')
        self._add_django_db_from_settings_key(ICDS_TEST_UCR_ENGINE_ID, 'ICDS_UCR_TEST_DATABASE_ALIAS')

    def _populate_from_legacy_settings(self):
        default_db = self._connection_string_from_django('default')
        ucr_db_reporting_url = getattr(settings, 'UCR_DATABASE_URL', None)
        self.db_connection_map[DEFAULT_ENGINE_ID] = default_db
        self.db_connection_map[UCR_ENGINE_ID] = ucr_db_reporting_url or default_db

    def _get_reporting_db_config(self):
        return getattr(settings, 'REPORTING_DATABASES', None)

    def _add_django_db_from_settings_key(self, engine_id, db_alias_settings_key):
        db_alias = self._get_db_alias_from_settings_key(db_alias_settings_key)
        if db_alias:
            self._add_django_db(engine_id, db_alias)

    def _add_django_db(self, engine_id, db_alias):
            connection_string = self._connection_string_from_django(db_alias)
            self.db_connection_map[engine_id] = connection_string

    def _connection_string_from_django(self, django_alias):
        db_settings = settings.DATABASES[django_alias].copy()
        db_settings['PORT'] = db_settings.get('PORT', '5432')
        options = db_settings.get('OPTIONS')
        db_settings['OPTIONS'] = '?{}'.format(urlencode(options)) if options else ''

        return "postgresql+psycopg2://{USER}:{PASSWORD}@{HOST}:{PORT}/{NAME}{OPTIONS}".format(
            **db_settings
        )

    def _get_db_alias_from_settings_key(self, db_alias_settings_key):
        db_alias = getattr(settings, db_alias_settings_key, None)
        if db_alias in settings.DATABASES:
            return db_alias

connection_manager = ConnectionManager()
Session = connection_manager.get_scoped_session(DEFAULT_ENGINE_ID)

# Register an event that closes the database connection
# when a Django request is finished.
# This will rollback any open transactions.
def _close_connections(**kwargs):
    Session.remove()  # todo: unclear whether this is necessary
    connection_manager.close_scoped_sessions()

signals.request_finished.connect(_close_connections)

@contextmanager
def override_engine(engine_id, connection_url):
    original_url = connection_manager.get_connection_string(engine_id)
    connection_manager.db_connection_map[engine_id] = connection_url
    try:
        yield
    finally:
        connection_manager.db_connection_map[engine_id] = original_url
<commit_msg>remove icd-ucr engine configs<commit_after>import random
from contextlib import contextmanager
from urllib import urlencode

from django.conf import settings
import sqlalchemy
from sqlalchemy.orm.scoping import scoped_session
from sqlalchemy.orm.session import sessionmaker
from django.core import signals

DEFAULT_ENGINE_ID = 'default'
UCR_ENGINE_ID = 'ucr'
ICDS_UCR_ENGINE_ID = 'icds-ucr'
ICDS_TEST_UCR_ENGINE_ID = 'icds-test-ucr'

def create_engine(connection_string):
    # paramstyle='format' allows you to use column names that include the ')' character
    # otherwise queries will sometimes be misformated/error when formatting
    # https://github.com/zzzeek/sqlalchemy/blob/ff20903/lib/sqlalchemy/dialects/postgresql/psycopg2.py#L173
    return sqlalchemy.create_engine(connection_string, paramstyle='format')

class SessionHelper(object):
    """
    Shim class helper for a single connection/session factory
    """

    def __init__(self, connection_string):
        self.engine = create_engine(connection_string)
        self._session_factory = sessionmaker(bind=self.engine)
        # Session is the actual constructor object
        self.Session = scoped_session(self._session_factory)

    @property
    def session_context(self):
        @contextmanager
        def session_scope():
            """Provide a transactional scope around a series of operations."""
            session = self.Session()
            try:
                yield session
                session.commit()
            except:
                session.rollback()
                raise
            finally:
                session.close()

        return session_scope

class ConnectionManager(object):
    """
    Object for dealing with sqlalchemy engines and sessions.
    """

    def __init__(self):
        self._session_helpers = {}
        self.db_connection_map = {}
        self.read_database_mapping = {}
        self._populate_connection_map()

    def _get_or_create_helper(self, engine_id):
        if engine_id not in self._session_helpers:
            self._session_helpers[engine_id] = SessionHelper(self.get_connection_string(engine_id))
        return self._session_helpers[engine_id]

    def get_session_helper(self, engine_id=DEFAULT_ENGINE_ID):
        """
        Returns the SessionHelper object associated with this engine id
        """
        # making this separate from _get_or_create in case we ever want to fail differently here
        return self._get_or_create_helper(engine_id)

    def get_scoped_session(self, engine_id=DEFAULT_ENGINE_ID):
        """
        This returns a handle to a thread-locally scoped session object.
        """
        return self.get_session_helper(engine_id).Session

    def get_engine(self, engine_id=DEFAULT_ENGINE_ID):
        """
        Get an engine by ID. This should be a unique field identifying the connection,
        e.g. "report-db-1"
        """
        return self._get_or_create_helper(engine_id).engine

    def get_load_balanced_read_engine_id(self, engine_id):
        read_dbs = self.read_database_mapping.get(engine_id, [])
        if read_dbs:
            return random.choice(read_dbs)
        return engine_id

    def close_scoped_sessions(self):
        for helper in self._session_helpers.values():
            helper.Session.remove()

    def dispose_engine(self, engine_id=DEFAULT_ENGINE_ID):
        """
        If found, closes the active sessions associated with an an engine and disposes it.
        Also removes it from the session manager.
        If not found, does nothing.
        """
        if engine_id in self._session_helpers:
            helper = self._session_helpers.pop(engine_id)
            helper.Session.remove()
            helper.engine.dispose()

    def dispose_all(self):
        """
        Dispose all engines associated with this. Useful for tests.
        """
        for engine_id in self._session_helpers.keys():
            self.dispose_engine(engine_id)

    def get_connection_string(self, engine_id):
        connection_string = self.db_connection_map.get(engine_id)
        return connection_string or self.db_connection_map['default']

    def _populate_connection_map(self):
        reporting_db_config = self._get_reporting_db_config()
        if not reporting_db_config:
            self._populate_from_legacy_settings()
        else:
            for engine_id, db_config in reporting_db_config.items():
                write_db = db_config
                read = None
                if isinstance(db_config, dict):
                    write_db = db_config['WRITE']
                    read = db_config['READ']
                    for db_alias, weighting in read:
                        assert isinstance(weighting, int), 'weighting must be int'
                        assert db_alias in settings.DATABASES, db_alias

                self._add_django_db(engine_id, write_db)
                if read:
                    self.read_database_mapping[engine_id] = []
                    for read_db, weighting in read:
                        assert read_db == write_db or read_db not in self.db_connection_map, read_db
                        self.read_database_mapping[engine_id].extend([read_db] * weighting)
                        if read_db != write_db:
                            self._add_django_db(read_db, read_db)

        if DEFAULT_ENGINE_ID not in self.db_connection_map:
            self._add_django_db(DEFAULT_ENGINE_ID, 'default')
        if UCR_ENGINE_ID not in self.db_connection_map:
            self._add_django_db(UCR_ENGINE_ID, 'default')

    def _populate_from_legacy_settings(self):
        default_db = self._connection_string_from_django('default')
        ucr_db_reporting_url = getattr(settings, 'UCR_DATABASE_URL', None)
        self.db_connection_map[DEFAULT_ENGINE_ID] = default_db
        self.db_connection_map[UCR_ENGINE_ID] = ucr_db_reporting_url or default_db

    def _get_reporting_db_config(self):
        return getattr(settings, 'REPORTING_DATABASES', None)

    def _add_django_db_from_settings_key(self, engine_id, db_alias_settings_key):
        db_alias = self._get_db_alias_from_settings_key(db_alias_settings_key)
        if db_alias:
            self._add_django_db(engine_id, db_alias)

    def _add_django_db(self, engine_id, db_alias):
            connection_string = self._connection_string_from_django(db_alias)
            self.db_connection_map[engine_id] = connection_string

    def _connection_string_from_django(self, django_alias):
        db_settings = settings.DATABASES[django_alias].copy()
        db_settings['PORT'] = db_settings.get('PORT', '5432')
        options = db_settings.get('OPTIONS')
        db_settings['OPTIONS'] = '?{}'.format(urlencode(options)) if options else ''

        return "postgresql+psycopg2://{USER}:{PASSWORD}@{HOST}:{PORT}/{NAME}{OPTIONS}".format(
            **db_settings
        )

    def _get_db_alias_from_settings_key(self, db_alias_settings_key):
        db_alias = getattr(settings, db_alias_settings_key, None)
        if db_alias in settings.DATABASES:
            return db_alias

connection_manager = ConnectionManager()
Session = connection_manager.get_scoped_session(DEFAULT_ENGINE_ID)

# Register an event that closes the database connection
# when a Django request is finished.
# This will rollback any open transactions.
def _close_connections(**kwargs):
    Session.remove()  # todo: unclear whether this is necessary
    connection_manager.close_scoped_sessions()

signals.request_finished.connect(_close_connections)

@contextmanager
def override_engine(engine_id, connection_url):
    original_url = connection_manager.get_connection_string(engine_id)
    connection_manager.db_connection_map[engine_id] = connection_url
    try:
        yield
    finally:
        connection_manager.db_connection_map[engine_id] = original_url
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2011 MORITA Kazutaka.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from sqlalchemy import Column, Table, MetaData, Boolean, String

meta = MetaData()

fixed_ips_host = Column('host', String(255))

networks_multi_host = Column('multi_host', Boolean, default=False)

def upgrade(migrate_engine):
    # Upgrade operations go here. Don't create your own engine;
    # bind migrate_engine to your metadata
    meta.bind = migrate_engine

    # Add columns to existing tables
    fixed_ips = Table('fixed_ips', meta, autoload=True)
    fixed_ips.create_column(fixed_ips_host)

    networks = Table('networks', meta, autoload=True)
    networks.create_column(networks_multi_host)
<commit_msg>add downgrade<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2011 OpenStack, LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from sqlalchemy import Column, Table, MetaData, Boolean, String

meta = MetaData()

fixed_ips_host = Column('host', String(255))

networks_multi_host = Column('multi_host', Boolean, default=False)

def upgrade(migrate_engine):
    meta.bind = migrate_engine

    fixed_ips = Table('fixed_ips', meta, autoload=True)
    fixed_ips.create_column(fixed_ips_host)

    networks = Table('networks', meta, autoload=True)
    networks.create_column(networks_multi_host)

def downgrade(migrate_engine):
    meta.bind = migrate_engine

    fixed_ips = Table('fixed_ips', meta, autoload=True)
    fixed_ips.drop_column(fixed_ips_host)

    networks = Table('networks', meta, autoload=True)
    networks.drop_column(networks_multi_host)
<|endoftext|>


<commit_before>import rethinkdb as r
r.connect( "localhost", 24455).repl()
import requests
results = []
for i in range(10):
    results.extend(requests.get('http://api.us.socrata.com/api/catalog/v1?only=datasets&limit=10000&offset='+str(10000*i)).json()['results'])
data = results
modified_data = []
for row in data:
    d = {}
    d.update(row)
    d.update(row['resource'])
    d.update(row['resource']['view_count'])
    d.update(row['classification'])
    modified_data.append(d)
r.db('public').table('datasets').insert(modified_data).run(conflict='update')
import rethinkdb as r
r.connect( "localhost", 28015).repl()
import requests
app_token = r.db('nonpublic').table('third_party_creds').get('socrata').run()['app_token']
t = list(r.db('public').table('police_response_events').order_by(r.desc('socrata_created_at'), index=r.desc('socrata_created_at')).limit(int(1)).run())[0]['socrata_created_at'].isoformat()[:-9]
data = requests.get('https://data.seattle.gov/resource/pu5n-trf4.json?$select=:*,*&$limit=2000000&$where=:created_at%%20>%%20"%s"&$$app_token=%s' % (t, app_token)).json()
import dateutil
import dateutil.parser
from pytz import timezone
from datetime import date
from datetime import datetime
tz = timezone('America/Los_Angeles')
for row in data:
    row['organization_id'] = 'cfee2384-fb23-450a-90bf-2cbfa2912876'
    row['is_cad_event'] = True
    row['is_rms_event'] = False
    for key in row.keys():
        if key.startswith(':id'):
            row['id'] = row[':id']
            del row[':id']
        elif key.startswith(':'):
            row['socrata_'+key[1:]] = row[key]
            del row[key]
    for key in row.keys():
        if not '_at' in key and not 'date' in key and not 'time' in key:
            continue

        row[key] = dateutil.parser.parse(row[key])
        if row[key].tzinfo == None:
            row[key] = row[key].replace(tzinfo = tz)
print r.db('public').table('police_response_events').insert(data).run(conflict='update')

t = list(r.db('public').table('police_response_events').order_by(r.desc('socrata_rms_created_at'), index=r.desc('socrata_rms_created_at')).limit(int(1)).run())[0]['socrata_rms_created_at'].isoformat()[:-9]
data = requests.get('https://data.seattle.gov/resource/y7pv-r3kh.json?$select=:*,*&$limit=2000000&$where=:created_at%%20>%%20"%s"&$$app_token=%s' % (t, app_token)).json()
print 'got the data'
#print data
import dateutil
from pytz import timezone
from datetime import date
from datetime import datetime
tz = timezone('America/Los_Angeles')
for i, row in enumerate(data):
    row['organization_id'] = 'cfee2384-fb23-450a-90bf-2cbfa2912876'
    row['is_rms_event'] = True
    #print 'gon', row['general_offense_number']
    try:
        row['id'] = list(r.db('public').table('police_response_events').get_all(row['general_offense_number'], index='general_offense_number').run())[0]['id']
    except:
        #print row.keys()
        row['id'] = row[':id']
    for key in row.keys():
        if key.startswith(':'):
            row['socrata_rms_'+key[1:]] = row[key]
            del row[key]
    for key in row.keys():
        if not '_at' in key and not 'date' in key and not 'time' in key:
            continue

        row[key] = dateutil.parser.parse(row[key])
        if row[key].tzinfo == None:
            row[key] = row[key].replace(tzinfo = tz)
    change = r.db('public').table('police_response_events').insert(row).run(conflict='update')
    print change<commit_msg>auto commit<commit_after>import rethinkdb as r
r.connect( "localhost", 24455).repl()
import requests
results = []
for i in range(10):
    results.extend(requests.get('http://api.us.socrata.com/api/catalog/v1?only=datasets&limit=10000&offset='+str(10000*i)).json()['results'])
data = results
modified_data = []
for row in data:
    d = {}
    d.update(row)
    d.update(row['resource'])
    d.update(row['resource']['view_count'])
    d.update(row['classification'])
    modified_data.append(d)
r.db('public').table('datasets').insert(modified_data).run(conflict='update')
import rethinkdb as r
r.connect( "localhost", 28015).repl()
import requests
app_token = r.db('nonpublic').table('third_party_creds').get('socrata').run()['app_token']
t = list(r.db('public').table('police_response_events').order_by(r.desc('socrata_created_at'), index=r.desc('socrata_created_at')).limit(int(1)).run())[0]['socrata_created_at'].isoformat()[:-9]
data = requests.get('https://data.seattle.gov/resource/pu5n-trf4.json?$select=:*,*&$limit=2000000&$where=:created_at%%20>%%20"%s"&$$app_token=%s' % (t, app_token)).json()
import dateutil
import dateutil.parser
from pytz import timezone
from datetime import date
from datetime import datetime
tz = timezone('America/Los_Angeles')
for row in data:
    row['organization_id'] = 'cfee2384-fb23-450a-90bf-2cbfa2912876'
    row['is_cad_event'] = True
    row['is_rms_event'] = False
    for key in row.keys():
        if key.startswith(':id'):
            row['id'] = row[':id']
            del row[':id']
        elif key.startswith(':'):
            row['socrata_'+key[1:]] = row[key]
            del row[key]
    for key in row.keys():
        if not '_at' in key and not 'date' in key and not 'time' in key:
            continue

        row[key] = dateutil.parser.parse(row[key])
        if row[key].tzinfo == None:
            row[key] = row[key].replace(tzinfo = tz)
print r.db('public').table('police_response_events').insert(data).run(conflict='update')

t = list(r.db('public').table('police_response_events').order_by(r.desc('socrata_rms_created_at'), index=r.desc('socrata_rms_created_at')).limit(int(1)).run())[0]['socrata_rms_created_at'].isoformat()[:-9]
data = requests.get('https://data.seattle.gov/resource/y7pv-r3kh.json?$select=:*,*&$limit=2000000&$where=:created_at%%20>%%20"%s"&$$app_token=%s' % (t, app_token)).json()
print 'got the data'
#print data
import dateutil
from pytz import timezone
from datetime import date
from datetime import datetime
tz = timezone('America/Los_Angeles')
for i, row in enumerate(data):
    row['organization_id'] = 'cfee2384-fb23-450a-90bf-2cbfa2912876'
    row['is_rms_event'] = True
    #print 'gon', row['general_offense_number']
    try:
        row['id'] = list(r.db('public').table('police_response_events').get_all(row['general_offense_number'], index='general_offense_number').run())[0]['id']
    except:
        #print row.keys()
        row['id'] = row[':id']
    for key in row.keys():
        if key.startswith(':'):
            row['socrata_rms_'+key[1:]] = row[key]
            del row[key]
    for key in row.keys():
        if not '_at' in key and not 'date' in key and not 'time' in key:
            continue

        row[key] = dateutil.parser.parse(row[key])
        if row[key].tzinfo == None:
            row[key] = row[key].replace(tzinfo = tz)
    change = r.db('public').table('police_response_events').insert(row).run(conflict='update')
    print change<|endoftext|>


<commit_before>#####################################################################################
#
#  Copyright (C) Tavendo GmbH
#
#  Unless a separate license agreement exists between you and Tavendo GmbH (e.g. you
#  have purchased a commercial license), the license terms below apply.
#
#  Should you enter into a separate license agreement after having received a copy of
#  this software, then the terms of such license agreement replace the terms below at
#  the time at which such license agreement becomes effective.
#
#  In case a separate license agreement ends, and such agreement ends without being
#  replaced by another separate license agreement, the license terms below apply
#  from the time at which said agreement ends.
#
#  LICENSE TERMS
#
#  This program is free software: you can redistribute it and/or modify it under the
#  terms of the GNU Affero General Public License, version 3, as published by the
#  Free Software Foundation. This program is distributed in the hope that it will be
#  useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
#  See the GNU Affero General Public License Version 3 for more details.
#
#  You should have received a copy of the GNU Affero General Public license along
#  with this program. If not, see <http://www.gnu.org/licenses/agpl-3.0.en.html>.
#
#####################################################################################

from __future__ import absolute_import, division

import six
from os import environ
from os.path import join, abspath

from twisted.internet import defer
from twisted.internet._sslverify import OpenSSLCertificateAuthorities
from twisted.internet.ssl import CertificateOptions, PrivateCertificate, Certificate, KeyPair
from twisted.internet.ssl import optionsForClientTLS, DiffieHellmanParameters
from twisted.internet.endpoints import TCP4ServerEndpoint, \
    TCP6ServerEndpoint, \
    TCP4ClientEndpoint, \
    TCP6ClientEndpoint, \
    UNIXServerEndpoint, \
    UNIXClientEndpoint
from twisted.python.filepath import FilePath

from crossbar._logging import make_logger
from crossbar.twisted.sharedport import SharedPort

try:
    from twisted.internet.endpoints import SSL4ServerEndpoint, \
        SSL4ClientEndpoint
    from OpenSSL import crypto
    _HAS_TLS = True
    _LACKS_TLS_MSG = None
except ImportError as e:
    _HAS_TLS = False
    _LACKS_TLS_MSG = "{}".format(e)

__all__ = ('create_listening_endpoint_from_config',
           'create_listening_port_from_config',
           'create_connecting_endpoint_from_config',
           'create_connecting_port_from_config')

def create_listening_endpoint_from_config(config, cbdir, reactor):
    """
    Create a Twisted stream server endpoint from a Crossbar.io transport configuration.

    See: https://twistedmatrix.com/documents/current/api/twisted.internet.interfaces.IStreamServerEndpoint.html

    :param config: The transport configuration.
    :type config: dict
    :param cbdir: Crossbar.io node directory (we need this for TLS key/certificates).
    :type cbdir: str
    :param reactor: The reactor to use for endpoint creation.
    :type reactor: obj

    :returns obj -- An instance implementing IStreamServerEndpoint
    """
    log = make_logger()
    endpoint = None

    # a TCP endpoint
    #
    if config['type'] == 'tcp':

        # the TCP protocol version (v4 or v6)
        #
        version = int(config.get('version', 4))

        # the listening port
        #
        if type(config['port']) is six.text_type:
            # read port from environment variable ..
            try:
                port = int(environ[config['port'][1:]])
            except Exception as e:
                print("Could not read listening port from env var: {}".format(e))
                raise e
        else:
            port = config['port']

        # the listening interface
        #
        interface = str(config.get('interface', '').strip())

        # the TCP accept queue depth
        #
        backlog = int(config.get('backlog', 50))

        if 'tls' in config:
            if _HAS_TLS:
                key_filepath = abspath(join(cbdir, config['tls']['key']))
                cert_filepath = abspath(join(cbdir, config['tls']['certificate']))

                with open(key_filepath) as key_file:
                    with open(cert_filepath) as cert_file:

                        if 'dhparam' in config['tls']:
                            dhpath = FilePath(
                                abspath(join(cbdir, config['tls']['dhparam']))
                            )
                            dh_params = DiffieHellmanParameters.fromFile(dhpath)
                        else:
                            # XXX won't be doing ANY EDH
                            # curves... maybe make dhparam required?
                            # or do "whatever tlxctx was doing"
                            dh_params = None
                            self.log.warn("OpenSSL DH modes not active (no 'dhparam')")

                        # create a TLS context factory
                        #
                        key = key_file.read()
                        cert = cert_file.read()
                        ciphers = config['tls'].get('ciphers', None)
                        ca_certs = None
                        if 'ca_certificates' in config['tls']:
                            ca_certs = []
                            for fname in config['tls']['ca_certificates']:
                                with open(fname, 'r') as f:
                                    ca_certs.append(Certificate.loadPEM(f.read()).original)

                        ctx = CertificateOptions(
                            privateKey=KeyPair.load(key, crypto.FILETYPE_PEM).original,
                            certificate=Certificate.loadPEM(cert).original,
                            verify=True,
                            caCerts=ca_certs,
                            dhParameters=dh_params,
                        )
                        if ctx._ecCurve is None:
                            log.warn("OpenSSL failed to set ECDH default curve")
                        else:
                            log.info(
                                "Ok, OpenSSL is using ECDH elliptic curve {curve}",
                                curve=ctx._ecCurve.snName,
                            )

                # create a TLS server endpoint
                #
                if version == 4:
                    endpoint = SSL4ServerEndpoint(reactor,
                                                  port,
                                                  ctx,
                                                  backlog=backlog,
                                                  interface=interface)
                elif version == 6:
                    raise Exception("TLS on IPv6 not implemented")
                else:
                    raise Exception("invalid TCP protocol version {}".format(version))
            else:
                raise Exception("TLS transport requested, but TLS packages not available:\n{}".format(_LACKS_TLS_MSG))

        else:
            # create a non-TLS server endpoint
            #
            if version == 4:
                endpoint = TCP4ServerEndpoint(reactor,
                                              port,
                                              backlog=backlog,
                                              interface=interface)
            elif version == 6:
                endpoint = TCP6ServerEndpoint(reactor,
                                              port,
                                              backlog=backlog,
                                              interface=interface)
            else:
                raise Exception("invalid TCP protocol version {}".format(version))

    # a Unix Domain Socket endpoint
    #
    elif config['type'] == 'unix':

        # the accept queue depth
        #
        backlog = int(config.get('backlog', 50))

        # the path
        #
        path = FilePath(join(cbdir, config['path']))

        # if there is already something there, delete it.
        #
        if path.exists():
            log.info(("{path} exists, attempting to remove before using as a "
                     "UNIX socket"), path=path)
            path.remove()

        # create the endpoint
        #
        endpoint = UNIXServerEndpoint(reactor, path.path, backlog=backlog)

    else:
        raise Exception("invalid endpoint type '{}'".format(config['type']))

    return endpoint

def create_listening_port_from_config(config, factory, cbdir, reactor):
    """
    Create a Twisted listening port from a Crossbar.io transport configuration.

    See: https://twistedmatrix.com/documents/current/api/twisted.internet.interfaces.IListeningPort.html

    :param config: The transport configuration.
    :type config: dict
    :param factory: The transport factory to use (a provider of IProtocolFactory).
    :type factory: obj
    :param cbdir: Crossbar.io node directory (we need this for TLS key/certificates).
    :type cbdir: str
    :param reactor: The reactor to use for endpoint creation.
    :type reactor: obj

    :returns obj -- A Deferred that results in an IListeningPort or an CannotListenError
    """
    if config['type'] == 'tcp' and config.get('shared', False):

        # the TCP protocol version (v4 or v6)
        # FIXME: handle v6
        # version = int(config.get('version', 4))

        # the listening port
        #
        port = int(config['port'])

        # the listening interface
        #
        interface = str(config.get('interface', '').strip())

        # the TCP accept queue depth
        #
        backlog = int(config.get('backlog', 50))

        listening_port = SharedPort(port, factory, backlog, interface, reactor, shared=True)
        try:
            listening_port.startListening()
            return defer.succeed(listening_port)
        except Exception as e:
            return defer.fail(e)

    else:
        try:
            endpoint = create_listening_endpoint_from_config(config, cbdir, reactor)
            return endpoint.listen(factory)
        except Exception:
            return defer.fail()

def create_connecting_endpoint_from_config(config, cbdir, reactor):
    """
    Create a Twisted stream client endpoint from a Crossbar.io transport configuration.

    See: https://twistedmatrix.com/documents/current/api/twisted.internet.interfaces.IStreamClientEndpoint.html

    :param config: The transport configuration.
    :type config: dict
    :param cbdir: Crossbar.io node directory (we need this for Unix domain socket paths and TLS key/certificates).
    :type cbdir: str
    :param reactor: The reactor to use for endpoint creation.
    :type reactor: obj

    :returns obj -- An instance implementing IStreamClientEndpoint
    """
    endpoint = None
    log = make_logger()

    # a TCP endpoint
    #
    if config['type'] == 'tcp':

        # the TCP protocol version (v4 or v6)
        #
        version = int(config.get('version', 4))

        # the host to connect to
        #
        host = str(config['host'])

        # the port to connect to
        #
        port = int(config['port'])

        # connection timeout in seconds
        #
        timeout = int(config.get('timeout', 10))

        if 'tls' in config:
            if _HAS_TLS:
                # if the config specified any CA certificates, we use those (only!)
                if 'ca_certificates' in config['tls']:
                    ca_certs = []
                    for cert_fname in config['tls']['ca_certificates']:
                        cert = crypto.load_certificate(
                            crypto.FILETYPE_PEM,
                            six.u(open(cert_fname, 'r').read())
                        )
                        log.info("Loaded CA certificate '{fname}'", fname=cert_fname)
                        ca_certs.append(cert)

                    client_cert = None
                    if 'key' in config['tls']:
                        with open(config['tls']['certificate'], 'r') as f:
                            cert = Certificate.loadPEM(
                                f.read(),
                            )
                            log.info(
                                "{fname}: CN={subj.CN}, sha={sha}",
                                fname=config['tls']['certificate'],
                                subj=cert.getSubject(),
                                sha=cert.digest('sha'),
                            )

                        with open(config['tls']['key'], 'r') as f:
                            private_key = KeyPair.load(
                                f.read(),
                                format=crypto.FILETYPE_PEM,
                            )

                            log.info(
                                "{fname}: {key}",
                                fname=config['tls']['key'],
                                key=private_key.inspect(),
                            )

                        client_cert = PrivateCertificate.fromCertificateAndKeyPair(
                            cert, private_key)

                    # XXX FIXME private class OpenSSLCertificateAuthorities...
                    options = optionsForClientTLS(
                        config['tls']['hostname'],
                        OpenSSLCertificateAuthorities(ca_certs),
                        client_cert,
                    )
                else:
                    options = optionsForClientTLS(config['tls']['hostname'])

                # create a TLS client endpoint
                #
                if version == 4:
                    endpoint = SSL4ClientEndpoint(
                        reactor,
                        host,
                        port,
                        options,
                        timeout=timeout,
                    )
                elif version == 6:
                    raise Exception("TLS on IPv6 not implemented")
                else:
                    raise Exception("invalid TCP protocol version {}".format(version))

            else:
                raise Exception("TLS transport requested, but TLS packages not available:\n{}".format(_LACKS_TLS_MSG))

        else:
            # create a non-TLS client endpoint
            #
            if version == 4:
                endpoint = TCP4ClientEndpoint(reactor,
                                              host,
                                              port,
                                              timeout=timeout)
            elif version == 6:
                endpoint = TCP6ClientEndpoint(reactor,
                                              host,
                                              port,
                                              timeout=timeout)
            else:
                raise Exception("invalid TCP protocol version {}".format(version))

    # a Unix Domain Socket endpoint
    #
    elif config['type'] == 'unix':

        # the path
        #
        path = abspath(join(cbdir, config['path']))

        # connection timeout in seconds
        #
        timeout = int(config.get('timeout', 10))

        # create the endpoint
        #
        endpoint = UNIXClientEndpoint(reactor, path, timeout=timeout)

    else:
        raise Exception("invalid endpoint type '{}'".format(config['type']))

    return endpoint

def create_connecting_port_from_config(config, factory, cbdir, reactor):
    """
    Create a Twisted connecting port from a Crossbar.io transport configuration.

    See: https://twistedmatrix.com/documents/current/api/twisted.internet.interfaces.IListeningPort.html

    :param config: The transport configuration.
    :type config: dict
    :param factory: The transport factory to use (a provider of IProtocolFactory).
    :type factory: obj
    :param cbdir: Crossbar.io node directory (we need this for Unix domain socket paths and TLS key/certificates).
    :type cbdir: str
    :param reactor: The reactor to use for endpoint creation.
    :type reactor: obj

    :returns obj -- A Deferred that results in an IProtocol upon successful connection otherwise a ConnectError
    """
    endpoint = create_connecting_endpoint_from_config(config, cbdir, reactor)
    return endpoint.connect(factory)
<commit_msg>Tell Twisted about ciphers crossbar likes<commit_after>#####################################################################################
#
#  Copyright (C) Tavendo GmbH
#
#  Unless a separate license agreement exists between you and Tavendo GmbH (e.g. you
#  have purchased a commercial license), the license terms below apply.
#
#  Should you enter into a separate license agreement after having received a copy of
#  this software, then the terms of such license agreement replace the terms below at
#  the time at which such license agreement becomes effective.
#
#  In case a separate license agreement ends, and such agreement ends without being
#  replaced by another separate license agreement, the license terms below apply
#  from the time at which said agreement ends.
#
#  LICENSE TERMS
#
#  This program is free software: you can redistribute it and/or modify it under the
#  terms of the GNU Affero General Public License, version 3, as published by the
#  Free Software Foundation. This program is distributed in the hope that it will be
#  useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
#  See the GNU Affero General Public License Version 3 for more details.
#
#  You should have received a copy of the GNU Affero General Public license along
#  with this program. If not, see <http://www.gnu.org/licenses/agpl-3.0.en.html>.
#
#####################################################################################

from __future__ import absolute_import, division

import six
from os import environ
from os.path import join, abspath

from twisted.internet import defer
from twisted.internet._sslverify import OpenSSLCertificateAuthorities
from twisted.internet.ssl import CertificateOptions, PrivateCertificate, Certificate, KeyPair
from twisted.internet.ssl import optionsForClientTLS, DiffieHellmanParameters
from twisted.internet.ssl import AcceptableCiphers
from twisted.internet.endpoints import TCP4ServerEndpoint, \
    TCP6ServerEndpoint, \
    TCP4ClientEndpoint, \
    TCP6ClientEndpoint, \
    UNIXServerEndpoint, \
    UNIXClientEndpoint
from twisted.python.filepath import FilePath

from crossbar._logging import make_logger
from crossbar.twisted.sharedport import SharedPort

try:
    from twisted.internet.endpoints import SSL4ServerEndpoint, \
        SSL4ClientEndpoint
    from OpenSSL import crypto
    _HAS_TLS = True
    _LACKS_TLS_MSG = None
except ImportError as e:
    _HAS_TLS = False
    _LACKS_TLS_MSG = "{}".format(e)

__all__ = ('create_listening_endpoint_from_config',
           'create_listening_port_from_config',
           'create_connecting_endpoint_from_config',
           'create_connecting_port_from_config')

def create_listening_endpoint_from_config(config, cbdir, reactor):
    """
    Create a Twisted stream server endpoint from a Crossbar.io transport configuration.

    See: https://twistedmatrix.com/documents/current/api/twisted.internet.interfaces.IStreamServerEndpoint.html

    :param config: The transport configuration.
    :type config: dict
    :param cbdir: Crossbar.io node directory (we need this for TLS key/certificates).
    :type cbdir: str
    :param reactor: The reactor to use for endpoint creation.
    :type reactor: obj

    :returns obj -- An instance implementing IStreamServerEndpoint
    """
    log = make_logger()
    endpoint = None

    # a TCP endpoint
    #
    if config['type'] == 'tcp':

        # the TCP protocol version (v4 or v6)
        #
        version = int(config.get('version', 4))

        # the listening port
        #
        if type(config['port']) is six.text_type:
            # read port from environment variable ..
            try:
                port = int(environ[config['port'][1:]])
            except Exception as e:
                print("Could not read listening port from env var: {}".format(e))
                raise e
        else:
            port = config['port']

        # the listening interface
        #
        interface = str(config.get('interface', '').strip())

        # the TCP accept queue depth
        #
        backlog = int(config.get('backlog', 50))

        if 'tls' in config:
            if _HAS_TLS:
                key_filepath = abspath(join(cbdir, config['tls']['key']))
                cert_filepath = abspath(join(cbdir, config['tls']['certificate']))

                with open(key_filepath) as key_file:
                    with open(cert_filepath) as cert_file:

                        if 'dhparam' in config['tls']:
                            dhpath = FilePath(
                                abspath(join(cbdir, config['tls']['dhparam']))
                            )
                            dh_params = DiffieHellmanParameters.fromFile(dhpath)
                        else:
                            # XXX won't be doing ANY EDH
                            # curves... maybe make dhparam required?
                            # or do "whatever tlxctx was doing"
                            dh_params = None
                            self.log.warn("OpenSSL DH modes not active (no 'dhparam')")

                        # create a TLS context factory
                        #
                        key = key_file.read()
                        cert = cert_file.read()
                        ciphers = config['tls'].get('ciphers', None)
                        ca_certs = None
                        if 'ca_certificates' in config['tls']:
                            ca_certs = []
                            for fname in config['tls']['ca_certificates']:
                                with open(fname, 'r') as f:
                                    ca_certs.append(Certificate.loadPEM(f.read()).original)

                        crossbar_ciphers = AcceptableCiphers.fromOpenSSLCipherString(
                            'ECDHE-RSA-AES128-GCM-SHA256:'
                            'DHE-RSA-AES128-GCM-SHA256:'
                            'ECDHE-RSA-AES128-SHA256:'
                            'DHE-RSA-AES128-SHA256:'
                            'ECDHE-RSA-AES128-SHA:'
                            'DHE-RSA-AES128-SHA'
                        )

                        ctx = CertificateOptions(
                            privateKey=KeyPair.load(key, crypto.FILETYPE_PEM).original,
                            certificate=Certificate.loadPEM(cert).original,
                            verify=True,
                            caCerts=ca_certs,
                            dhParameters=dh_params,
                            acceptableCiphers=crossbar_ciphers,
                        )
                        if ctx._ecCurve is None:
                            log.warn("OpenSSL failed to set ECDH default curve")
                        else:
                            log.info(
                                "Ok, OpenSSL is using ECDH elliptic curve {curve}",
                                curve=ctx._ecCurve.snName,
                            )

                # create a TLS server endpoint
                #
                if version == 4:
                    endpoint = SSL4ServerEndpoint(reactor,
                                                  port,
                                                  ctx,
                                                  backlog=backlog,
                                                  interface=interface)
                elif version == 6:
                    raise Exception("TLS on IPv6 not implemented")
                else:
                    raise Exception("invalid TCP protocol version {}".format(version))
            else:
                raise Exception("TLS transport requested, but TLS packages not available:\n{}".format(_LACKS_TLS_MSG))

        else:
            # create a non-TLS server endpoint
            #
            if version == 4:
                endpoint = TCP4ServerEndpoint(reactor,
                                              port,
                                              backlog=backlog,
                                              interface=interface)
            elif version == 6:
                endpoint = TCP6ServerEndpoint(reactor,
                                              port,
                                              backlog=backlog,
                                              interface=interface)
            else:
                raise Exception("invalid TCP protocol version {}".format(version))

    # a Unix Domain Socket endpoint
    #
    elif config['type'] == 'unix':

        # the accept queue depth
        #
        backlog = int(config.get('backlog', 50))

        # the path
        #
        path = FilePath(join(cbdir, config['path']))

        # if there is already something there, delete it.
        #
        if path.exists():
            log.info(("{path} exists, attempting to remove before using as a "
                     "UNIX socket"), path=path)
            path.remove()

        # create the endpoint
        #
        endpoint = UNIXServerEndpoint(reactor, path.path, backlog=backlog)

    else:
        raise Exception("invalid endpoint type '{}'".format(config['type']))

    return endpoint

def create_listening_port_from_config(config, factory, cbdir, reactor):
    """
    Create a Twisted listening port from a Crossbar.io transport configuration.

    See: https://twistedmatrix.com/documents/current/api/twisted.internet.interfaces.IListeningPort.html

    :param config: The transport configuration.
    :type config: dict
    :param factory: The transport factory to use (a provider of IProtocolFactory).
    :type factory: obj
    :param cbdir: Crossbar.io node directory (we need this for TLS key/certificates).
    :type cbdir: str
    :param reactor: The reactor to use for endpoint creation.
    :type reactor: obj

    :returns obj -- A Deferred that results in an IListeningPort or an CannotListenError
    """
    if config['type'] == 'tcp' and config.get('shared', False):

        # the TCP protocol version (v4 or v6)
        # FIXME: handle v6
        # version = int(config.get('version', 4))

        # the listening port
        #
        port = int(config['port'])

        # the listening interface
        #
        interface = str(config.get('interface', '').strip())

        # the TCP accept queue depth
        #
        backlog = int(config.get('backlog', 50))

        listening_port = SharedPort(port, factory, backlog, interface, reactor, shared=True)
        try:
            listening_port.startListening()
            return defer.succeed(listening_port)
        except Exception as e:
            return defer.fail(e)

    else:
        try:
            endpoint = create_listening_endpoint_from_config(config, cbdir, reactor)
            return endpoint.listen(factory)
        except Exception:
            return defer.fail()

def create_connecting_endpoint_from_config(config, cbdir, reactor):
    """
    Create a Twisted stream client endpoint from a Crossbar.io transport configuration.

    See: https://twistedmatrix.com/documents/current/api/twisted.internet.interfaces.IStreamClientEndpoint.html

    :param config: The transport configuration.
    :type config: dict
    :param cbdir: Crossbar.io node directory (we need this for Unix domain socket paths and TLS key/certificates).
    :type cbdir: str
    :param reactor: The reactor to use for endpoint creation.
    :type reactor: obj

    :returns obj -- An instance implementing IStreamClientEndpoint
    """
    endpoint = None
    log = make_logger()

    # a TCP endpoint
    #
    if config['type'] == 'tcp':

        # the TCP protocol version (v4 or v6)
        #
        version = int(config.get('version', 4))

        # the host to connect to
        #
        host = str(config['host'])

        # the port to connect to
        #
        port = int(config['port'])

        # connection timeout in seconds
        #
        timeout = int(config.get('timeout', 10))

        if 'tls' in config:
            if _HAS_TLS:
                # if the config specified any CA certificates, we use those (only!)
                if 'ca_certificates' in config['tls']:
                    ca_certs = []
                    for cert_fname in config['tls']['ca_certificates']:
                        cert = crypto.load_certificate(
                            crypto.FILETYPE_PEM,
                            six.u(open(cert_fname, 'r').read())
                        )
                        log.info("Loaded CA certificate '{fname}'", fname=cert_fname)
                        ca_certs.append(cert)

                    client_cert = None
                    if 'key' in config['tls']:
                        with open(config['tls']['certificate'], 'r') as f:
                            cert = Certificate.loadPEM(
                                f.read(),
                            )
                            log.info(
                                "{fname}: CN={subj.CN}, sha={sha}",
                                fname=config['tls']['certificate'],
                                subj=cert.getSubject(),
                                sha=cert.digest('sha'),
                            )

                        with open(config['tls']['key'], 'r') as f:
                            private_key = KeyPair.load(
                                f.read(),
                                format=crypto.FILETYPE_PEM,
                            )

                            log.info(
                                "{fname}: {key}",
                                fname=config['tls']['key'],
                                key=private_key.inspect(),
                            )

                        client_cert = PrivateCertificate.fromCertificateAndKeyPair(
                            cert, private_key)

                    # XXX FIXME private class OpenSSLCertificateAuthorities...
                    options = optionsForClientTLS(
                        config['tls']['hostname'],
                        OpenSSLCertificateAuthorities(ca_certs),
                        client_cert,
                    )
                else:
                    options = optionsForClientTLS(config['tls']['hostname'])

                # create a TLS client endpoint
                #
                if version == 4:
                    endpoint = SSL4ClientEndpoint(
                        reactor,
                        host,
                        port,
                        options,
                        timeout=timeout,
                    )
                elif version == 6:
                    raise Exception("TLS on IPv6 not implemented")
                else:
                    raise Exception("invalid TCP protocol version {}".format(version))

            else:
                raise Exception("TLS transport requested, but TLS packages not available:\n{}".format(_LACKS_TLS_MSG))

        else:
            # create a non-TLS client endpoint
            #
            if version == 4:
                endpoint = TCP4ClientEndpoint(reactor,
                                              host,
                                              port,
                                              timeout=timeout)
            elif version == 6:
                endpoint = TCP6ClientEndpoint(reactor,
                                              host,
                                              port,
                                              timeout=timeout)
            else:
                raise Exception("invalid TCP protocol version {}".format(version))

    # a Unix Domain Socket endpoint
    #
    elif config['type'] == 'unix':

        # the path
        #
        path = abspath(join(cbdir, config['path']))

        # connection timeout in seconds
        #
        timeout = int(config.get('timeout', 10))

        # create the endpoint
        #
        endpoint = UNIXClientEndpoint(reactor, path, timeout=timeout)

    else:
        raise Exception("invalid endpoint type '{}'".format(config['type']))

    return endpoint

def create_connecting_port_from_config(config, factory, cbdir, reactor):
    """
    Create a Twisted connecting port from a Crossbar.io transport configuration.

    See: https://twistedmatrix.com/documents/current/api/twisted.internet.interfaces.IListeningPort.html

    :param config: The transport configuration.
    :type config: dict
    :param factory: The transport factory to use (a provider of IProtocolFactory).
    :type factory: obj
    :param cbdir: Crossbar.io node directory (we need this for Unix domain socket paths and TLS key/certificates).
    :type cbdir: str
    :param reactor: The reactor to use for endpoint creation.
    :type reactor: obj

    :returns obj -- A Deferred that results in an IProtocol upon successful connection otherwise a ConnectError
    """
    endpoint = create_connecting_endpoint_from_config(config, cbdir, reactor)
    return endpoint.connect(factory)
<|endoftext|>


<commit_before>from .args import (
    build_argparser,
    get_students_from_args,
    get_assignments_from_args,
    compute_stogit_url,
)

def args(arglist):
    return vars(build_argparser().parse_args(args=arglist))

students = {
    'my': ['rives'],
    'section-a': ['student-a'],
    'section-b': ['student-b'],
}

def test_all():
    # check that --all includes all students
    assert get_students_from_args(**args(['--all']), _all_students=students) == students['my'] + students['section-a'] + students['section-b']

def test_students():
    # multiple sets of --students should wind up as one flattened list
    assert get_students_from_args(**args(['--students', 'a', 'b', '--students', 'c']), _all_students=students) == ['a', 'b', 'c']

    # it should return a sorted list of student names
    assert get_students_from_args(**args(['--students', 'c', 'b', '--students', 'a']), _all_students=students) == ['a', 'b', 'c']

    # multiple occurences of the same student should be removed
    assert get_students_from_args(**args(['--students', 'a', 'a', '--students', 'a']), _all_students=students) == ['a']

    # if no students are given, it should default to the "my" section
    assert get_students_from_args(**args([]), _all_students=students) == students['my']

def test_section():
    # "--section $name" should return the students for that section
    assert get_students_from_args(**args(['--section', 'a']), _all_students=students) == students['section-a']

def test_record():
    assert get_assignments_from_args(**args(['--record', 'hw4'])) == ['hw4']
<commit_msg>add tests for compute_stogit_url<commit_after>import datetime
from .args import (
    build_argparser,
    get_students_from_args,
    get_assignments_from_args,
    compute_stogit_url,
)

def args(arglist):
    return vars(build_argparser().parse_args(args=arglist))

students = {
    'my': ['rives'],
    'section-a': ['student-a'],
    'section-b': ['student-b'],
}

def test_all():
    # check that --all includes all students
    assert get_students_from_args(**args(['--all']), _all_students=students) == students['my'] + students['section-a'] + students['section-b']

def test_students():
    # multiple sets of --students should wind up as one flattened list
    assert get_students_from_args(**args(['--students', 'a', 'b', '--students', 'c']), _all_students=students) == ['a', 'b', 'c']

    # it should return a sorted list of student names
    assert get_students_from_args(**args(['--students', 'c', 'b', '--students', 'a']), _all_students=students) == ['a', 'b', 'c']

    # multiple occurences of the same student should be removed
    assert get_students_from_args(**args(['--students', 'a', 'a', '--students', 'a']), _all_students=students) == ['a']

    # if no students are given, it should default to the "my" section
    assert get_students_from_args(**args([]), _all_students=students) == students['my']

def test_section():
    # "--section $name" should return the students for that section
    assert get_students_from_args(**args(['--section', 'a']), _all_students=students) == students['section-a']

def test_record():
    assert get_assignments_from_args(**args(['--record', 'hw4'])) == ['hw4']

def test_stogit_url_computation():
    assert compute_stogit_url(stogit=None, course='sd', _now=datetime.date(2017, 1, 31)) \
        == 'git@stogit.cs.stolaf.edu:sd-s17'

    assert compute_stogit_url(stogit=None, course='sd', _now=datetime.date(2016, 9, 15)) \
        == 'git@stogit.cs.stolaf.edu:sd-f16'

    assert compute_stogit_url(stogit=None, course='sd', _now=datetime.date(2016, 4, 15)) \
        == 'git@stogit.cs.stolaf.edu:sd-s16'

    assert compute_stogit_url(stogit='blah', course='sd', _now=datetime.date.today()) \
        == 'blah'

    assert compute_stogit_url(stogit=None, course='hd', _now=datetime.date(2016, 4, 15)) \
        == 'git@stogit.cs.stolaf.edu:hd-s16'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# OpenFisca -- A versatile microsimulation software
# By: OpenFisca Team <contact@openfisca.fr>
#
# Copyright (C) 2011, 2012, 2013, 2014 OpenFisca Team
# https://github.com/openfisca
#
# This file is part of OpenFisca.
#
# OpenFisca is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# OpenFisca is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import division

import datetime
import logging
from numpy import logical_not as not_, logical_or as or_, maximum as max_, minimum as min_

from openfisca_core.accessors import law
from openfisca_core.columns import FloatCol
from openfisca_core.formulas import dated_function, DatedFormulaColumn, SimpleFormulaColumn

from ..base import CAT, QUIFAM, QUIFOY, QUIMEN
from ..base import Individus, reference_formula

CHEF = QUIFAM['chef']
PREF = QUIMEN['pref']
VOUS = QUIFOY['vous']

log = logging.getLogger(__name__)

@reference_formula
class smic_proratise(SimpleFormulaColumn):
    column = FloatCol
    entity_class = Individus
    label = u"SMIC annuel proratisé"

    def function(self, nombre_heures_remunerees, contrat_de_travail,
                 smic_horaire_brut = law.cotsoc.gen.smic_h_b):

        smic_proratise = smic_horaire_brut * nombre_heures_remunerees
        return smic_proratise

    def get_output_period(self, period):
        return period

@reference_formula
class ratio_smic_salaire(DatedFormulaColumn):
    column = FloatCol
    entity_class = Individus
    label = u"Ratio smic/salaire pour le calcul de l'allègement Fillon"

    @dated_function(start = datetime.date(2012, 1, 1))
    def function_2012(self, smic_proratise, salbrut, contrat_de_travail,
                      smic_horaire_brut = law.cotsoc.gen.smic_h_b):
        # salbrut_annuel 2012 nombre_heures_remunerees incluent hsup à partir de 2012
        smic_annuel = smic_proratise  # durée légale du travail = 1820
        ratio_smic_salaire = smic_annuel / (salbrut + 1e-10)
        return ratio_smic_salaire

    @dated_function(start = datetime.date(2011, 1, 1), stop = datetime.date(2011, 12, 31))
    def function_2011(self, smic_proratise, salbrut, contrat_de_travail,
                      smic_horaire_brut = law.cotsoc.gen.smic_h_b):
        # TODO
        # salbrut_annuel 2011 même chose mais avec salbrut sans hsup
        smic_annuel = smic_proratise  # durée légale du travail = 1820
        ratio_smic_salaire = smic_annuel / (salbrut + 1e-10)
        return ratio_smic_salaire

    @dated_function(start = datetime.date(2005, 7, 1), stop = datetime.date(2010, 12, 31))
    def function_2007_2010(self, smic_proratise, salbrut, contrat_de_travail,
                           smic_horaire_brut = law.cotsoc.gen.smic_h_b):
        # TODO: revoir la législation
        smic_annuel = smic_proratise  # durée légale du travail = 1820
        ratio_smic_salaire = smic_annuel / (salbrut + 1e-10)
        return ratio_smic_salaire

    def get_output_period(self, period):
        return period

@reference_formula
class allegement_fillon_anticipe(DatedFormulaColumn):
    column = FloatCol
    entity_class = Individus
    label = u"Allègement de charges patronales sur les bas et moyens salaires (dit allègement Fillon)"

    @dated_function(datetime.date(2005, 7, 1))
    def function(self, ratio_smic_salaire, salbrut, taille_entreprise, type_sal,
                 cotsoc = law.cotsoc):

        majoration = (taille_entreprise <= 2)  # majoration éventuelle pour les petites entreprises
        taux_fillon = taux_exo_fillon(ratio_smic_salaire, majoration, cotsoc)
        allegement_fillon = (
            taux_fillon *
            salbrut *
            ((type_sal == CAT['prive_non_cadre']) | (type_sal == CAT['prive_cadre']))
            )
        return allegement_fillon

    def get_output_period(self, period):
        return period.start.offset('first-of', 'month').period('month')

@reference_formula
class allegement_fillon(DatedFormulaColumn):   # annuel
    column = FloatCol
    entity_class = Individus
    label = u"Allègement de charges patronales sur les bas et moyens salaires (dit allègement Fillon)"

    @dated_function(datetime.date(2005, 7, 1))
    def function(self, ratio_smic_salaire, salbrut, taille_entreprise, type_sal,
                 cotsoc = law.cotsoc):

        majoration = (taille_entreprise <= 2)  # majoration éventuelle pour les petites entreprises
        taux_fillon = taux_exo_fillon(ratio_smic_salaire, majoration, cotsoc)
        allegement_fillon = (
            taux_fillon *
            salbrut *
            ((type_sal == CAT['prive_non_cadre']) | (type_sal == CAT['prive_cadre']))
            )
        return allegement_fillon

    def get_output_period(self, period):
        return period.start.offset('first-of', 'year').period('year')

@reference_formula
class allegement_fillon_bis(DatedFormulaColumn):
    column = FloatCol
    entity_class = Individus
    label = u"Allègement de charges patronales sur les bas et moyens salaires (dit allègement Fillon)"

    @dated_function(datetime.date(2005, 7, 1))
    def function(self, period, allegement_fillon_anticipe, allegement_fillon_annuel,
                 cotsoc = law.cotsoc):
        print "month: ", period.start.month
        if period.start.month < 12:
            return allegement_fillon_anticipe
        else:
            return allegement_fillon_annuel - allegement_fillon_anticipe

    def get_variable_period(self, output_period, variable_name):
        if variable_name in ['allegement_fillon_annuel']:
            return output_period.start.offset('first-of', 'year').period('year')
        elif variable_name in ['allegement_fillon_anticipe'] and output_period.start.month == 12:
            return output_period.start.offset('first-of', 'year').period('month', 11)
        else:
            return output_period

    def get_output_period(self, period):
        return period.start.offset('first-of', 'month').period('month')

@reference_formula
class alleg_cice(DatedFormulaColumn):
    column = FloatCol
    entity_class = Individus
    label = u"Crédit d'imôt pour la compétitivité et l'emploi"

    @dated_function(datetime.date(2013, 1, 1))
    def function_2013_(self, ratio_smic_salaire, salbrut, taille_entreprise, type_sal,
                       cotsoc = law.cotsoc):

        taux_cice = taux_exo_cice(ratio_smic_salaire, cotsoc)
        alleg_cice = (
            taux_cice
            * salbrut
            * or_((type_sal == CAT['prive_non_cadre']), (type_sal == CAT['prive_cadre']))
            )
        return alleg_cice

    def get_output_period(self, period):
        return period.start.offset('first-of', 'month').period('month')

# Helper functions

def taux_exo_fillon(ratio_smic_salaire, majoration, P):
    '''
    Exonération Fillon
    http://www.securite-sociale.fr/comprendre/dossiers/exocotisations/exoenvigueur/fillon.htm
    '''
    # La divison par zéro engendre un warning
    # Le montant maximum de l’allègement dépend de l’effectif de l’entreprise.
    # Le montant est calculé chaque année civile, pour chaque salarié ;
    # il est égal au produit de la totalité de la rémunération annuelle telle
    # que visée à l’article L. 242-1 du code de la Sécurité sociale par un
    # coefficient.
    # Ce montant est majoré de 10 % pour les entreprises de travail temporaire
    # au titre des salariés temporaires pour lesquels elle est tenue à
    # l’obligation d’indemnisation compensatrice de congés payés.

    Pf = P.exo_bas_sal.fillon
    seuil = Pf.seuil
    tx_max = (Pf.tx_max * not_(majoration) + Pf.tx_max2 * majoration)
    if seuil <= 1:
        return 0
    # règle d'arrondi: 4 décimales au dix-millième le plus proche
    taux_fillon = round(tx_max * min_(1, max_(seuil * ratio_smic_salaire - 1, 0) / (seuil - 1)), 4)
    return taux_fillon

def taux_exo_cice(sal_h_b, P):
    Pc = P.exo_bas_sal.cice
    taux_cice = (1 / ratio_smic_salaire <= Pc.max) * Pc.taux
    return taux_cice
<commit_msg>Fix cice<commit_after># -*- coding: utf-8 -*-

# OpenFisca -- A versatile microsimulation software
# By: OpenFisca Team <contact@openfisca.fr>
#
# Copyright (C) 2011, 2012, 2013, 2014 OpenFisca Team
# https://github.com/openfisca
#
# This file is part of OpenFisca.
#
# OpenFisca is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# OpenFisca is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import division

import datetime
import logging
from numpy import logical_not as not_, logical_or as or_, maximum as max_, minimum as min_

from openfisca_core.accessors import law
from openfisca_core.columns import FloatCol
from openfisca_core.formulas import dated_function, DatedFormulaColumn, SimpleFormulaColumn

from ..base import CAT, QUIFAM, QUIFOY, QUIMEN
from ..base import Individus, reference_formula

CHEF = QUIFAM['chef']
PREF = QUIMEN['pref']
VOUS = QUIFOY['vous']

log = logging.getLogger(__name__)

@reference_formula
class smic_proratise(SimpleFormulaColumn):
    column = FloatCol
    entity_class = Individus
    label = u"SMIC annuel proratisé"

    def function(self, nombre_heures_remunerees, contrat_de_travail,
                 smic_horaire_brut = law.cotsoc.gen.smic_h_b):

        smic_proratise = smic_horaire_brut * nombre_heures_remunerees
        return smic_proratise

    def get_output_period(self, period):
        return period

@reference_formula
class ratio_smic_salaire(DatedFormulaColumn):
    column = FloatCol
    entity_class = Individus
    label = u"Ratio smic/salaire pour le calcul de l'allègement Fillon"

    @dated_function(start = datetime.date(2012, 1, 1))
    def function_2012(self, smic_proratise, salbrut, contrat_de_travail,
                      smic_horaire_brut = law.cotsoc.gen.smic_h_b):
        # salbrut_annuel 2012 nombre_heures_remunerees incluent hsup à partir de 2012
        smic_annuel = smic_proratise  # durée légale du travail = 1820
        ratio_smic_salaire = smic_annuel / (salbrut + 1e-10)
        return ratio_smic_salaire

    @dated_function(start = datetime.date(2011, 1, 1), stop = datetime.date(2011, 12, 31))
    def function_2011(self, smic_proratise, salbrut, contrat_de_travail,
                      smic_horaire_brut = law.cotsoc.gen.smic_h_b):
        # TODO
        # salbrut_annuel 2011 même chose mais avec salbrut sans hsup
        smic_annuel = smic_proratise  # durée légale du travail = 1820
        ratio_smic_salaire = smic_annuel / (salbrut + 1e-10)
        return ratio_smic_salaire

    @dated_function(start = datetime.date(2005, 7, 1), stop = datetime.date(2010, 12, 31))
    def function_2007_2010(self, smic_proratise, salbrut, contrat_de_travail,
                           smic_horaire_brut = law.cotsoc.gen.smic_h_b):
        # TODO: revoir la législation
        smic_annuel = smic_proratise  # durée légale du travail = 1820
        ratio_smic_salaire = smic_annuel / (salbrut + 1e-10)
        return ratio_smic_salaire

    def get_output_period(self, period):
        return period

@reference_formula
class allegement_fillon_anticipe(DatedFormulaColumn):
    column = FloatCol
    entity_class = Individus
    label = u"Allègement de charges patronales sur les bas et moyens salaires (dit allègement Fillon)"

    @dated_function(datetime.date(2005, 7, 1))
    def function(self, ratio_smic_salaire, salbrut, taille_entreprise, type_sal,
                 cotsoc = law.cotsoc):

        majoration = (taille_entreprise <= 2)  # majoration éventuelle pour les petites entreprises
        taux_fillon = taux_exo_fillon(ratio_smic_salaire, majoration, cotsoc)
        allegement_fillon = (
            taux_fillon *
            salbrut *
            ((type_sal == CAT['prive_non_cadre']) | (type_sal == CAT['prive_cadre']))
            )
        return allegement_fillon

    def get_output_period(self, period):
        return period.start.offset('first-of', 'month').period('month')

@reference_formula
class allegement_fillon(DatedFormulaColumn):   # annuel
    column = FloatCol
    entity_class = Individus
    label = u"Allègement de charges patronales sur les bas et moyens salaires (dit allègement Fillon)"

    @dated_function(datetime.date(2005, 7, 1))
    def function(self, ratio_smic_salaire, salbrut, taille_entreprise, type_sal,
                 cotsoc = law.cotsoc):

        majoration = (taille_entreprise <= 2)  # majoration éventuelle pour les petites entreprises
        taux_fillon = taux_exo_fillon(ratio_smic_salaire, majoration, cotsoc)
        allegement_fillon = (
            taux_fillon *
            salbrut *
            ((type_sal == CAT['prive_non_cadre']) | (type_sal == CAT['prive_cadre']))
            )
        return allegement_fillon

    def get_output_period(self, period):
        return period.start.offset('first-of', 'year').period('year')

@reference_formula
class allegement_fillon_bis(DatedFormulaColumn):
    column = FloatCol
    entity_class = Individus
    label = u"Allègement de charges patronales sur les bas et moyens salaires (dit allègement Fillon)"

    @dated_function(datetime.date(2005, 7, 1))
    def function(self, period, allegement_fillon_anticipe, allegement_fillon_annuel,
                 cotsoc = law.cotsoc):
        print "month: ", period.start.month
        if period.start.month < 12:
            return allegement_fillon_anticipe
        else:
            return allegement_fillon_annuel - allegement_fillon_anticipe

    def get_variable_period(self, output_period, variable_name):
        if variable_name in ['allegement_fillon_annuel']:
            return output_period.start.offset('first-of', 'year').period('year')
        elif variable_name in ['allegement_fillon_anticipe'] and output_period.start.month == 12:
            return output_period.start.offset('first-of', 'year').period('month', 11)
        else:
            return output_period

    def get_output_period(self, period):
        return period.start.offset('first-of', 'month').period('month')

@reference_formula
class alleg_cice(DatedFormulaColumn):
    column = FloatCol
    entity_class = Individus
    label = u"Crédit d'imôt pour la compétitivité et l'emploi"

    @dated_function(datetime.date(2013, 1, 1))
    def function_2013_(self, ratio_smic_salaire, salbrut, taille_entreprise, type_sal,
                       cotsoc = law.cotsoc):

        taux_cice = taux_exo_cice(ratio_smic_salaire, cotsoc)
        alleg_cice = (
            taux_cice
            * salbrut
            * or_((type_sal == CAT['prive_non_cadre']), (type_sal == CAT['prive_cadre']))
            )
        return alleg_cice

    def get_output_period(self, period):
        return period.start.offset('first-of', 'month').period('month')

# Helper functions

def taux_exo_fillon(ratio_smic_salaire, majoration, P):
    '''
    Exonération Fillon
    http://www.securite-sociale.fr/comprendre/dossiers/exocotisations/exoenvigueur/fillon.htm
    '''
    # La divison par zéro engendre un warning
    # Le montant maximum de l’allègement dépend de l’effectif de l’entreprise.
    # Le montant est calculé chaque année civile, pour chaque salarié ;
    # il est égal au produit de la totalité de la rémunération annuelle telle
    # que visée à l’article L. 242-1 du code de la Sécurité sociale par un
    # coefficient.
    # Ce montant est majoré de 10 % pour les entreprises de travail temporaire
    # au titre des salariés temporaires pour lesquels elle est tenue à
    # l’obligation d’indemnisation compensatrice de congés payés.

    Pf = P.exo_bas_sal.fillon
    seuil = Pf.seuil
    tx_max = (Pf.tx_max * not_(majoration) + Pf.tx_max2 * majoration)
    if seuil <= 1:
        return 0
    # règle d'arrondi: 4 décimales au dix-millième le plus proche
    taux_fillon = round(tx_max * min_(1, max_(seuil * ratio_smic_salaire - 1, 0) / (seuil - 1)), 4)
    return taux_fillon

def taux_exo_cice(ratio_smic_salaire, P):
    Pc = P.exo_bas_sal.cice
    taux_cice = (1 / ratio_smic_salaire <= Pc.max) * Pc.taux
    return taux_cice
<|endoftext|>


<commit_before># A job for creating and managing key generation

from os import environ
import subprocess

jenkins_scripts = environ.get("JENKINS_SCRIPTS")
aws_access_key = environ.get("AWS_ACCESS_KEY")
aws_secret_key = environ.get("AWS_SECRET_KEY")

domain_name=environ.get('Domain_Name')
state=environ.get('State')
city=environ.get('City')
legal_name=environ.get('Legal_Name')
bucket=environ.get('Databag_Name')

# Enter the required information about the company
# Run the SSL command
command='openssl req -new -newkey rsa:2048 -nodes -out /tmp/{domain_name}.csr -keyout /tmp/{domain_name}.key -subj'.format(domain_name=domain_name).split(" ")
command+=['/C=US/ST={state}/L={city}/O={legal_name}/CN={domain_name}'.format(state=state, city=city, legal_name=legal_name, domain_name=domain_name)]
print " ".join(command)
out = subprocess.check_output(command)
print out

# Upload the CSR file to EC2
command="s3upload -l DEBUG -k {aws_access_key} -sk {aws_secret_key} -f -np 8 -s 100 /tmp/{domain_name}.csr s3://{bucket}/{domain_name}.csr".format(aws_access_key=aws_access_key,aws_secret_key=aws_secret_key,domain_name=domain_name, bucket=bucket)
out = subprocess.check_output(command.split(" "))
print out

# Upload the KEY file to EC2
command="s3upload -l DEBUG -k {aws_access_key} -sk {aws_secret_key} -f -np 8 -s 100 /tmp/{domain_name}.key s3://{bucket}/{domain_name}.key".format(aws_access_key=aws_access_key,aws_secret_key=aws_secret_key,domain_name=domain_name, bucket=bucket)
out = subprocess.check_output(command.split(" "))
print out

# Clean-up the local files
os.remove("/tmp/{domain_name}.csr")
os.remove("/tmp/{domain_name}.key")

# Input them into the nmdproxy/certs databag? TODO

# Show the nmd file commands to pull the files and share them
print "Your cert files have been uploaded to the nmdarchive. To download them, copy paste the following text into iTerm:\n"
print "mkdir ~/Downloads/{bucket} &&".format(bucket=bucket)
print "cd ~/Downloads/{bucket} &&".format(bucket=bucket)
print "nmd file get {bucket}/{domain_name}.key &&".format(bucket=bucket, domain_name=domain_name)
print "nmd file get {bucket}/{domain_name}.csr &&".format(bucket=bucket, domain_name=domain_name)
print "open ."<commit_msg>Typo.<commit_after># A job for creating and managing key generation

from os import environ
import subprocess

jenkins_scripts = environ.get("JENKINS_SCRIPTS")
aws_access_key = environ.get("AWS_ACCESS_KEY")
aws_secret_key = environ.get("AWS_SECRET_KEY")

domain_name=environ.get('Domain_Name')
state=environ.get('State')
city=environ.get('City')
legal_name=environ.get('Legal_Name')
bucket=environ.get('Databag_Name')

# Enter the required information about the company
# Run the SSL command
command='openssl req -new -newkey rsa:2048 -nodes -out /tmp/{domain_name}.csr -keyout /tmp/{domain_name}.key -subj'.format(domain_name=domain_name).split(" ")
command+=['/C=US/ST={state}/L={city}/O={legal_name}/CN={domain_name}'.format(state=state, city=city, legal_name=legal_name, domain_name=domain_name)]
print " ".join(command)
out = subprocess.check_output(command)
print out

# Upload the CSR file to EC2
command="s3upload -l DEBUG -k {aws_access_key} -sk {aws_secret_key} -f -np 8 -s 100 /tmp/{domain_name}.csr s3://nmdarchive/{bucket}/{domain_name}.csr".format(aws_access_key=aws_access_key,aws_secret_key=aws_secret_key,domain_name=domain_name, bucket=bucket)
out = subprocess.check_output(command.split(" "))
print out

# Upload the KEY file to EC2
command="s3upload -l DEBUG -k {aws_access_key} -sk {aws_secret_key} -f -np 8 -s 100 /tmp/{domain_name}.key s3://nmdarchive/{bucket}/{domain_name}.key".format(aws_access_key=aws_access_key,aws_secret_key=aws_secret_key,domain_name=domain_name, bucket=bucket)
out = subprocess.check_output(command.split(" "))
print out

# Clean-up the local files
os.remove("/tmp/{domain_name}.csr")
os.remove("/tmp/{domain_name}.key")

# Input them into the nmdproxy/certs databag? TODO

# Show the nmd file commands to pull the files and share them
print "Your cert files have been uploaded to the nmdarchive. To download them, copy paste the following text into iTerm:\n"
print "mkdir ~/Downloads/{bucket} &&".format(bucket=bucket)
print "cd ~/Downloads/{bucket} &&".format(bucket=bucket)
print "nmd file get {bucket}/{domain_name}.key &&".format(bucket=bucket, domain_name=domain_name)
print "nmd file get {bucket}/{domain_name}.csr &&".format(bucket=bucket, domain_name=domain_name)
print "open ."<|endoftext|>


<commit_before># Copyright (c) 2014, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Core functionality for the classical damage risk calculator.
"""

from openquake.engine.calculators.risk import (
    base, hazard_getters, validation, writers)
from openquake.engine.calculators import calculators
from openquake.engine.db import models

def classical_damage(workflow, getter, outputdict, params, monitor):
    """
    Celery task for the classical risk calculator.

    :param workflow:
      A :class:`openquake.risklib.workflows.RiskModel` instance
    :param getter:
      A HazardGetter instance
    :param outputdict:
      An instance of :class:`..writers.OutputDict` containing
      output container instances (e.g. a LossCurve)
    :param params:
      An instance of :class:`..base.CalcParams` used to compute
      derived outputs
    :param monitor:
      A monitor instance
    For each calculation unit we compute loss curves, loss maps and
    loss fractions. Then if the number of units are bigger than 1, we
    compute mean and quantile artifacts.
    """
    for loss_type in workflow.loss_types:
        with monitor.copy('computing risk'):
            outputs = workflow.compute_all_outputs(getter, loss_type)
        with monitor.copy('saving risk'):
            for out in outputs:
                damage = models.Damage.objects.get(hazard_output=out.hid)
                writers.classical_damage(
                    out.assets, out.damages, params.damage_state_ids,
                    damage.id)
        # TODO: statistical outputs

@calculators.add('classical_damage')
class ClassicalDamageCalculator(base.RiskCalculator):
    """
    Classical PSHA risk calculator. Computes loss curves and loss maps
    for a given set of assets.
    """

    core = staticmethod(classical_damage)

    validators = base.RiskCalculator.validators + [
        validation.ExposureHasInsuranceBounds]

    output_builders = [writers.DamageCurveBuilder]

    getter_class = hazard_getters.HazardCurveGetter

    def pre_execute(self):
        """
        Create the DmgState objects associated to the current calculation
        """
        super(ClassicalDamageCalculator, self).pre_execute()
        self.oqparam.damage_state_ids = []
        for lsi, dstate in enumerate(self.risk_model.damage_states):
            ds = models.DmgState.objects.create(
                risk_calculation=self.job, dmg_state=dstate, lsi=lsi)
            self.oqparam.damage_state_ids.append(ds.id)
<commit_msg>Fixed docstring<commit_after># Copyright (c) 2015, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""
Core functionality for the classical damage risk calculator.
"""

from openquake.engine.calculators.risk import (
    base, hazard_getters, validation, writers)
from openquake.engine.calculators import calculators
from openquake.engine.db import models

def classical_damage(workflow, getter, outputdict, params, monitor):
    """
    Celery task for the classical risk calculator.

    :param workflow:
      A :class:`openquake.risklib.workflows.RiskModel` instance
    :param getter:
      A HazardGetter instance
    :param outputdict:
      An instance of :class:`..writers.OutputDict` containing
      output container instances (e.g. a LossCurve)
    :param params:
      An instance of :class:`..base.CalcParams` used to compute
      derived outputs
    :param monitor:
      A monitor instance
    """
    for loss_type in workflow.loss_types:
        with monitor.copy('computing risk'):
            outputs = workflow.compute_all_outputs(getter, loss_type)
        with monitor.copy('saving risk'):
            for out in outputs:
                damage = models.Damage.objects.get(hazard_output=out.hid)
                writers.classical_damage(
                    out.assets, out.damages, params.damage_state_ids,
                    damage.id)
        # TODO: statistical outputs

@calculators.add('classical_damage')
class ClassicalDamageCalculator(base.RiskCalculator):
    """
    Classical PSHA risk calculator. Computes loss curves and loss maps
    for a given set of assets.
    """

    core = staticmethod(classical_damage)

    validators = base.RiskCalculator.validators + [
        validation.ExposureHasInsuranceBounds]

    output_builders = [writers.DamageCurveBuilder]

    getter_class = hazard_getters.HazardCurveGetter

    def pre_execute(self):
        """
        Create the DmgState objects associated to the current calculation
        """
        super(ClassicalDamageCalculator, self).pre_execute()

        self.oqparam.damage_state_ids = []
        for lsi, dstate in enumerate(self.risk_model.damage_states):
            ds = models.DmgState.objects.create(
                risk_calculation=self.job, dmg_state=dstate, lsi=lsi)
            self.oqparam.damage_state_ids.append(ds.id)
<|endoftext|>


<commit_before># The Hazard Library
# Copyright (C) 2012-2018 GEM Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
# import unittest

import os
import numpy
from openquake.hazardlib.scalerel.thingbaijam2017 import ThingbaijamInterface
from openquake.hazardlib.tests.scalerel.msr_test import BaseMSRTestCase

DATA_DIR = os.path.join(os.path.dirname(__file__), 'data/thingbaijam2017')

class ThingbaijamInterfaceTestCase(BaseMSRTestCase):
    '''
    Tests for the magnitude-scaling relationship Thingbaijam et al. (2017) for
    interface events.
    '''

    MSR_CLASS = ThingbaijamInterface

    def test_median_area(self):
        """
        Tests against data digitized from a figure and manual data
        """
        # digitized data
        fname = os.path.join(DATA_DIR, 'interface.csv')
        data = numpy.loadtxt(fname, delimiter=',')
        msr = ThingbaijamInterface()
        computed = msr.get_median_area(data[:, 0], 90.)
        numpy.testing.assert_allclose(computed, 10**data[:, 1], rtol=10)
        # manual data
        self._test_get_median_area(8.0, None, 19952.6231496888, places=5)
        self._test_get_median_area(7.0, None, 2243.8819237828, places=5)

    def test_median_magnitude(self):
        """
        This tests the MSR
        """
        self._test_get_median_mag(19952.6231496888, None, 8.00, places=2)
        self._test_get_median_mag(2243.8819237828, None, 7.00, places=2)
<commit_msg>Fixing path<commit_after># The Hazard Library
# Copyright (C) 2012-2018 GEM Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
# import unittest

import os
import numpy
from openquake.hazardlib.scalerel.thingbaijam2017 import ThingbaijamInterface
from openquake.hazardlib.tests.scalerel.msr_test import BaseMSRTestCase

DATA_DIR = os.path.join(os.path.dirname(__file__), 'data', 'thingbaijam2017')

class ThingbaijamInterfaceTestCase(BaseMSRTestCase):
    '''
    Tests for the magnitude-scaling relationship Thingbaijam et al. (2017) for
    interface events.
    '''

    MSR_CLASS = ThingbaijamInterface

    def test_median_area(self):
        """
        Tests against data digitized from a figure and manual data
        """
        # digitized data
        fname = os.path.join(DATA_DIR, 'interface.csv')
        data = numpy.loadtxt(fname, delimiter=',')
        msr = ThingbaijamInterface()
        computed = msr.get_median_area(data[:, 0], 90.)
        numpy.testing.assert_allclose(computed, 10**data[:, 1], rtol=10)
        # manual data
        self._test_get_median_area(8.0, None, 19952.6231496888, places=5)
        self._test_get_median_area(7.0, None, 2243.8819237828, places=5)

    def test_median_magnitude(self):
        """
        This tests the MSR
        """
        self._test_get_median_mag(19952.6231496888, None, 8.00, places=2)
        self._test_get_median_mag(2243.8819237828, None, 7.00, places=2)
<|endoftext|>