<commit_before># Copyright (c) 2014 Cisco Systems Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#
# @author: Arvind Somya (asomya@cisco.com), Cisco Systems Inc.

from apicapi import apic_manager
from apicapi import exceptions as exc
from keystoneclient.v2_0 import client as keyclient
import netaddr
from oslo.config import cfg

from neutron.common import constants as n_constants
from neutron.extensions import portbindings
from neutron.openstack.common import lockutils
from neutron.openstack.common import log
from neutron.plugins.common import constants
from neutron.plugins.ml2 import driver_api as api
from neutron.plugins.ml2 import models

from apic_ml2.neutron.plugins.ml2.drivers.cisco.apic import apic_model
from apic_ml2.neutron.plugins.ml2.drivers.cisco.apic import apic_sync
from apic_ml2.neutron.plugins.ml2.drivers.cisco.apic import config

LOG = log.getLogger(__name__)

class APICMechanismDriver(api.MechanismDriver):

    @staticmethod
    def get_apic_manager(client=True):
        apic_config = cfg.CONF.ml2_cisco_apic
        network_config = {
            'vlan_ranges': cfg.CONF.ml2_type_vlan.network_vlan_ranges,
            'switch_dict': config.create_switch_dictionary(),
            'vpc_dict': config.create_vpc_dictionary(),
            'external_network_dict':
            config.create_external_network_dictionary(),
        }
        apic_system_id = cfg.CONF.apic_system_id
        keyclient_param = keyclient if client else None
        keystone_authtoken = cfg.CONF.keystone_authtoken if client else None
        return apic_manager.APICManager(apic_model.ApicDbModel(), log,
                                        network_config, apic_config,
                                        keyclient_param, keystone_authtoken,
                                        apic_system_id)

    @staticmethod
    def get_base_synchronizer(inst):
        apic_config = cfg.CONF.ml2_cisco_apic
        return apic_sync.ApicBaseSynchronizer(inst,
                                              apic_config.apic_sync_interval)

    @staticmethod
    def get_router_synchronizer(inst):
        apic_config = cfg.CONF.ml2_cisco_apic
        return apic_sync.ApicRouterSynchronizer(inst,
                                                apic_config.apic_sync_interval)

    def initialize(self):
        # initialize apic
        self.apic_manager = APICMechanismDriver.get_apic_manager()
        self.name_mapper = self.apic_manager.apic_mapper
        self.synchronizer = None
        self.apic_manager.ensure_infra_created_on_apic()
        self.apic_manager.ensure_bgp_pod_policy_created_on_apic()

    def sync_init(f):
        def inner(inst, *args, **kwargs):
            if not inst.synchronizer:
                inst.synchronizer = (
                    APICMechanismDriver.get_base_synchronizer(inst))
                inst.synchronizer.sync_base()
            return f(inst, *args, **kwargs)
        return inner

    @lockutils.synchronized('apic-portlock')
    def _perform_path_port_operations(self, context, port):
        # Get network
        network_id = context.network.current['id']
        anetwork_id = self.name_mapper.network(context, network_id)
        # Get tenant details from port context
        tenant_id = context.current['tenant_id']
        tenant_id = self.name_mapper.tenant(context, tenant_id)

        # Get segmentation id
        if not context.bound_segment:
            LOG.debug("Port %s is not bound to a segment", port)
            return
        seg = None
        if (context.bound_segment.get(api.NETWORK_TYPE)
                in [constants.TYPE_VLAN]):
            seg = context.bound_segment.get(api.SEGMENTATION_ID)
        # hosts on which this vlan is provisioned
        host = port.get(portbindings.HOST_ID)
        # Create a static path attachment for the host/epg/switchport combo
        with self.apic_manager.apic.transaction() as trs:
            self.apic_manager.ensure_path_created_for_port(
                tenant_id, anetwork_id, host, seg, transaction=trs)

    def _perform_gw_port_operations(self, context, port):
        router_id = port.get('device_id')
        network = context.network.current
        anetwork_id = self.name_mapper.network(context, network['id'])
        router_info = self.apic_manager.ext_net_dict.get(network['name'])

        if router_id and router_info:
            address = router_info['cidr_exposed']
            next_hop = router_info['gateway_ip']
            encap = router_info.get('encap')  # No encap if None
            switch = router_info['switch']
            module, sport = router_info['port'].split('/')
            with self.apic_manager.apic.transaction() as trs:
                # Get/Create contract
                arouter_id = self.name_mapper.router(context, router_id)
                cid = self.apic_manager.get_router_contract(arouter_id)
                # Ensure that the external ctx exists
                self.apic_manager.ensure_context_enforced()
                # Create External Routed Network and configure it
                self.apic_manager.ensure_external_routed_network_created(
                    anetwork_id, transaction=trs)
                self.apic_manager.ensure_logical_node_profile_created(
                    anetwork_id, switch, module, sport, encap,
                    address, transaction=trs)
                self.apic_manager.ensure_static_route_created(
                    anetwork_id, switch, next_hop, transaction=trs)
                self.apic_manager.ensure_external_epg_created(
                    anetwork_id, transaction=trs)
                self.apic_manager.ensure_external_epg_consumed_contract(
                    anetwork_id, cid, transaction=trs)
                self.apic_manager.ensure_external_epg_provided_contract(
                    anetwork_id, cid, transaction=trs)

    def _perform_port_operations(self, context):
        # Get port
        port = context.current
        # Check if a compute port
        if port.get('device_owner', '').startswith('compute'):
            self._perform_path_port_operations(context, port)
        elif port.get('device_owner') == n_constants.DEVICE_OWNER_ROUTER_GW:
            self._perform_gw_port_operations(context, port)
        elif port.get('device_owner') == n_constants.DEVICE_OWNER_DHCP:
            self._perform_path_port_operations(context, port)

    def _delete_contract(self, context):
        port = context.current
        network_id = self.name_mapper.network(
            context, context.network.current['id'])
        arouter_id = self.name_mapper.router(context,
                                             port.get('device_id'))
        self.apic_manager.delete_external_epg_contract(arouter_id,
                                                       network_id)

    def _get_active_path_count(self, context):
        return (context._plugin_context.session.query(models.PortBinding).
                filter_by(host=context.current.get(portbindings.HOST_ID),
                          segment=context._binding.segment).count())

    @lockutils.synchronized('apic-portlock')
    def _delete_port_path(self, context, atenant_id, anetwork_id):
        if not self._get_active_path_count(context):
            self.apic_manager.ensure_path_deleted_for_port(
                atenant_id, anetwork_id,
                context.current.get(portbindings.HOST_ID))

    def _delete_path_if_last(self, context):
        if not self._get_active_path_count(context):
            tenant_id = context.current['tenant_id']
            atenant_id = self.name_mapper.tenant(context, tenant_id)
            network_id = context.network.current['id']
            anetwork_id = self.name_mapper.network(context, network_id)
            self._delete_port_path(context, atenant_id, anetwork_id)

    def _get_subnet_info(self, context, subnet):
        tenant_id = subnet['tenant_id']
        network_id = subnet['network_id']
        network = context._plugin.get_network(context._plugin_context,
                                              network_id)
        if not network.get('router:external'):
            cidr = netaddr.IPNetwork(subnet['cidr'])
            gateway_ip = '%s/%s' % (subnet['gateway_ip'], str(cidr.prefixlen))

            # Convert to APIC IDs
            tenant_id = self.name_mapper.tenant(context, tenant_id)
            network_id = self.name_mapper.network(context, network_id)
            return tenant_id, network_id, gateway_ip

    @sync_init
    def create_port_postcommit(self, context):
        self._perform_port_operations(context)

    def update_port_precommit(self, context):
        orig = context.original
        curr = context.current
        if (orig['device_owner'] != curr['device_owner']
                or orig['device_id'] != curr['device_id']):
            raise exc.ApicOperationNotSupported(
                resource='Port', msg='Port device owner and id cannot be '
                                     'changed.')

    @sync_init
    def update_port_postcommit(self, context):
        self._perform_port_operations(context)

    def delete_port_postcommit(self, context):
        port = context.current
        # Check if a compute port
        if port.get('device_owner', '').startswith('compute'):
            self._delete_path_if_last(context)
        elif port.get('device_owner') == n_constants.DEVICE_OWNER_ROUTER_GW:
            self._delete_contract(context)
        elif port.get('device_owner') == n_constants.DEVICE_OWNER_DHCP:
            self._delete_path_if_last(context)

    @sync_init
    def create_network_postcommit(self, context):
        if not context.current.get('router:external'):
            tenant_id = context.current['tenant_id']
            network_id = context.current['id']

            # Convert to APIC IDs
            tenant_id = self.name_mapper.tenant(context, tenant_id)
            network_id = self.name_mapper.network(context, network_id)

            # Create BD and EPG for this network
            with self.apic_manager.apic.transaction() as trs:
                self.apic_manager.ensure_bd_created_on_apic(tenant_id,
                                                            network_id,
                                                            transaction=trs)
                self.apic_manager.ensure_epg_created(
                    tenant_id, network_id, transaction=trs)

    @sync_init
    def update_network_postcommit(self, context):
        super(APICMechanismDriver, self).update_network_postcommit(context)

    def delete_network_postcommit(self, context):
        if not context.current.get('router:external'):
            tenant_id = context.current['tenant_id']
            network_id = context.current['id']

            # Convert to APIC IDs
            tenant_id = self.name_mapper.tenant(context, tenant_id)
            network_id = self.name_mapper.network(context, network_id)

            # Delete BD and EPG for this network
            with self.apic_manager.apic.transaction() as trs:
                self.apic_manager.delete_epg_for_network(tenant_id, network_id,
                                                         transaction=trs)
                self.apic_manager.delete_bd_on_apic(tenant_id, network_id,
                                                    transaction=trs)
        else:
            network_name = context.current['name']
            if self.apic_manager.ext_net_dict.get(network_name):
                network_id = self.name_mapper.network(context,
                                                      context.current['id'])
                self.apic_manager.delete_external_routed_network(network_id)

    @sync_init
    def create_subnet_postcommit(self, context):
        info = self._get_subnet_info(context, context.current)
        if info:
            tenant_id, network_id, gateway_ip = info
            # Create subnet on BD
            self.apic_manager.ensure_subnet_created_on_apic(
                tenant_id, network_id, gateway_ip)

    @sync_init
    def update_subnet_postcommit(self, context):
        if context.current['gateway_ip'] != context.original['gateway_ip']:
            with self.apic_manager.apic.transaction() as trs:
                info = self._get_subnet_info(context, context.original)
                if info:
                    tenant_id, network_id, gateway_ip = info
                    # Delete subnet
                    self.apic_manager.ensure_subnet_deleted_on_apic(
                        tenant_id, network_id, gateway_ip, transaction=trs)
                info = self._get_subnet_info(context, context.current)
                if info:
                    tenant_id, network_id, gateway_ip = info
                    # Create subnet
                    self.apic_manager.ensure_subnet_created_on_apic(
                        tenant_id, network_id, gateway_ip, transaction=trs)

    def delete_subnet_postcommit(self, context):
        info = self._get_subnet_info(context, context.current)
        if info:
            tenant_id, network_id, gateway_ip = info
            self.apic_manager.ensure_subnet_deleted_on_apic(
                tenant_id, network_id, gateway_ip)
<commit_msg>Non compute port binding fix<commit_after># Copyright (c) 2014 Cisco Systems Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#
# @author: Arvind Somya (asomya@cisco.com), Cisco Systems Inc.

from apicapi import apic_manager
from apicapi import exceptions as exc
from keystoneclient.v2_0 import client as keyclient
import netaddr
from oslo.config import cfg

from neutron.common import constants as n_constants
from neutron.extensions import portbindings
from neutron.openstack.common import lockutils
from neutron.openstack.common import log
from neutron.plugins.common import constants
from neutron.plugins.ml2 import driver_api as api
from neutron.plugins.ml2 import models

from apic_ml2.neutron.plugins.ml2.drivers.cisco.apic import apic_model
from apic_ml2.neutron.plugins.ml2.drivers.cisco.apic import apic_sync
from apic_ml2.neutron.plugins.ml2.drivers.cisco.apic import config

LOG = log.getLogger(__name__)

class APICMechanismDriver(api.MechanismDriver):

    @staticmethod
    def get_apic_manager(client=True):
        apic_config = cfg.CONF.ml2_cisco_apic
        network_config = {
            'vlan_ranges': cfg.CONF.ml2_type_vlan.network_vlan_ranges,
            'switch_dict': config.create_switch_dictionary(),
            'vpc_dict': config.create_vpc_dictionary(),
            'external_network_dict':
            config.create_external_network_dictionary(),
        }
        apic_system_id = cfg.CONF.apic_system_id
        keyclient_param = keyclient if client else None
        keystone_authtoken = cfg.CONF.keystone_authtoken if client else None
        return apic_manager.APICManager(apic_model.ApicDbModel(), log,
                                        network_config, apic_config,
                                        keyclient_param, keystone_authtoken,
                                        apic_system_id)

    @staticmethod
    def get_base_synchronizer(inst):
        apic_config = cfg.CONF.ml2_cisco_apic
        return apic_sync.ApicBaseSynchronizer(inst,
                                              apic_config.apic_sync_interval)

    @staticmethod
    def get_router_synchronizer(inst):
        apic_config = cfg.CONF.ml2_cisco_apic
        return apic_sync.ApicRouterSynchronizer(inst,
                                                apic_config.apic_sync_interval)

    def initialize(self):
        # initialize apic
        self.apic_manager = APICMechanismDriver.get_apic_manager()
        self.name_mapper = self.apic_manager.apic_mapper
        self.synchronizer = None
        self.apic_manager.ensure_infra_created_on_apic()
        self.apic_manager.ensure_bgp_pod_policy_created_on_apic()

    def sync_init(f):
        def inner(inst, *args, **kwargs):
            if not inst.synchronizer:
                inst.synchronizer = (
                    APICMechanismDriver.get_base_synchronizer(inst))
                inst.synchronizer.sync_base()
            return f(inst, *args, **kwargs)
        return inner

    @lockutils.synchronized('apic-portlock')
    def _perform_path_port_operations(self, context, port):
        # Get network
        network_id = context.network.current['id']
        anetwork_id = self.name_mapper.network(context, network_id)
        # Get tenant details from port context
        tenant_id = context.current['tenant_id']
        tenant_id = self.name_mapper.tenant(context, tenant_id)

        # Get segmentation id
        if not context.bound_segment:
            LOG.debug("Port %s is not bound to a segment", port)
            return
        seg = None
        if (context.bound_segment.get(api.NETWORK_TYPE)
                in [constants.TYPE_VLAN]):
            seg = context.bound_segment.get(api.SEGMENTATION_ID)
        # hosts on which this vlan is provisioned
        host = port.get(portbindings.HOST_ID)
        # Create a static path attachment for the host/epg/switchport combo
        with self.apic_manager.apic.transaction() as trs:
            self.apic_manager.ensure_path_created_for_port(
                tenant_id, anetwork_id, host, seg, transaction=trs)

    def _perform_gw_port_operations(self, context, port):
        router_id = port.get('device_id')
        network = context.network.current
        anetwork_id = self.name_mapper.network(context, network['id'])
        router_info = self.apic_manager.ext_net_dict.get(network['name'])

        if router_id and router_info:
            address = router_info['cidr_exposed']
            next_hop = router_info['gateway_ip']
            encap = router_info.get('encap')  # No encap if None
            switch = router_info['switch']
            module, sport = router_info['port'].split('/')
            with self.apic_manager.apic.transaction() as trs:
                # Get/Create contract
                arouter_id = self.name_mapper.router(context, router_id)
                cid = self.apic_manager.get_router_contract(arouter_id)
                # Ensure that the external ctx exists
                self.apic_manager.ensure_context_enforced()
                # Create External Routed Network and configure it
                self.apic_manager.ensure_external_routed_network_created(
                    anetwork_id, transaction=trs)
                self.apic_manager.ensure_logical_node_profile_created(
                    anetwork_id, switch, module, sport, encap,
                    address, transaction=trs)
                self.apic_manager.ensure_static_route_created(
                    anetwork_id, switch, next_hop, transaction=trs)
                self.apic_manager.ensure_external_epg_created(
                    anetwork_id, transaction=trs)
                self.apic_manager.ensure_external_epg_consumed_contract(
                    anetwork_id, cid, transaction=trs)
                self.apic_manager.ensure_external_epg_provided_contract(
                    anetwork_id, cid, transaction=trs)

    def _perform_port_operations(self, context):
        # Get port
        port = context.current
        # Check if a compute port
        if port.get(portbindings.HOST_ID):
            self._perform_path_port_operations(context, port)
        if port.get('device_owner') == n_constants.DEVICE_OWNER_ROUTER_GW:
            self._perform_gw_port_operations(context, port)

    def _delete_contract(self, context):
        port = context.current
        network_id = self.name_mapper.network(
            context, context.network.current['id'])
        arouter_id = self.name_mapper.router(context,
                                             port.get('device_id'))
        self.apic_manager.delete_external_epg_contract(arouter_id,
                                                       network_id)

    def _get_active_path_count(self, context):
        return (context._plugin_context.session.query(models.PortBinding).
                filter_by(host=context.current.get(portbindings.HOST_ID),
                          segment=context._binding.segment).count())

    @lockutils.synchronized('apic-portlock')
    def _delete_port_path(self, context, atenant_id, anetwork_id):
        if not self._get_active_path_count(context):
            self.apic_manager.ensure_path_deleted_for_port(
                atenant_id, anetwork_id,
                context.current.get(portbindings.HOST_ID))

    def _delete_path_if_last(self, context):
        if not self._get_active_path_count(context):
            tenant_id = context.current['tenant_id']
            atenant_id = self.name_mapper.tenant(context, tenant_id)
            network_id = context.network.current['id']
            anetwork_id = self.name_mapper.network(context, network_id)
            self._delete_port_path(context, atenant_id, anetwork_id)

    def _get_subnet_info(self, context, subnet):
        tenant_id = subnet['tenant_id']
        network_id = subnet['network_id']
        network = context._plugin.get_network(context._plugin_context,
                                              network_id)
        if not network.get('router:external'):
            cidr = netaddr.IPNetwork(subnet['cidr'])
            gateway_ip = '%s/%s' % (subnet['gateway_ip'], str(cidr.prefixlen))

            # Convert to APIC IDs
            tenant_id = self.name_mapper.tenant(context, tenant_id)
            network_id = self.name_mapper.network(context, network_id)
            return tenant_id, network_id, gateway_ip

    @sync_init
    def create_port_postcommit(self, context):
        self._perform_port_operations(context)

    @sync_init
    def update_port_postcommit(self, context):
        self._perform_port_operations(context)

    def delete_port_postcommit(self, context):
        port = context.current
        if port.get(portbindings.HOST_ID):
            self._delete_path_if_last(context)
        elif port.get('device_owner') == n_constants.DEVICE_OWNER_ROUTER_GW:
            self._delete_contract(context)

    @sync_init
    def create_network_postcommit(self, context):
        if not context.current.get('router:external'):
            tenant_id = context.current['tenant_id']
            network_id = context.current['id']

            # Convert to APIC IDs
            tenant_id = self.name_mapper.tenant(context, tenant_id)
            network_id = self.name_mapper.network(context, network_id)

            # Create BD and EPG for this network
            with self.apic_manager.apic.transaction() as trs:
                self.apic_manager.ensure_bd_created_on_apic(tenant_id,
                                                            network_id,
                                                            transaction=trs)
                self.apic_manager.ensure_epg_created(
                    tenant_id, network_id, transaction=trs)

    @sync_init
    def update_network_postcommit(self, context):
        super(APICMechanismDriver, self).update_network_postcommit(context)

    def delete_network_postcommit(self, context):
        if not context.current.get('router:external'):
            tenant_id = context.current['tenant_id']
            network_id = context.current['id']

            # Convert to APIC IDs
            tenant_id = self.name_mapper.tenant(context, tenant_id)
            network_id = self.name_mapper.network(context, network_id)

            # Delete BD and EPG for this network
            with self.apic_manager.apic.transaction() as trs:
                self.apic_manager.delete_epg_for_network(tenant_id, network_id,
                                                         transaction=trs)
                self.apic_manager.delete_bd_on_apic(tenant_id, network_id,
                                                    transaction=trs)
        else:
            network_name = context.current['name']
            if self.apic_manager.ext_net_dict.get(network_name):
                network_id = self.name_mapper.network(context,
                                                      context.current['id'])
                self.apic_manager.delete_external_routed_network(network_id)

    @sync_init
    def create_subnet_postcommit(self, context):
        info = self._get_subnet_info(context, context.current)
        if info:
            tenant_id, network_id, gateway_ip = info
            # Create subnet on BD
            self.apic_manager.ensure_subnet_created_on_apic(
                tenant_id, network_id, gateway_ip)

    @sync_init
    def update_subnet_postcommit(self, context):
        if context.current['gateway_ip'] != context.original['gateway_ip']:
            with self.apic_manager.apic.transaction() as trs:
                info = self._get_subnet_info(context, context.original)
                if info:
                    tenant_id, network_id, gateway_ip = info
                    # Delete subnet
                    self.apic_manager.ensure_subnet_deleted_on_apic(
                        tenant_id, network_id, gateway_ip, transaction=trs)
                info = self._get_subnet_info(context, context.current)
                if info:
                    tenant_id, network_id, gateway_ip = info
                    # Create subnet
                    self.apic_manager.ensure_subnet_created_on_apic(
                        tenant_id, network_id, gateway_ip, transaction=trs)

    def delete_subnet_postcommit(self, context):
        info = self._get_subnet_info(context, context.current)
        if info:
            tenant_id, network_id, gateway_ip = info
            self.apic_manager.ensure_subnet_deleted_on_apic(
                tenant_id, network_id, gateway_ip)
<|endoftext|>


<commit_before># Copyright (C) 2010 Linaro Limited
#
# Author: Zygmunt Krynicki <zygmunt.krynicki@linaro.org>
#
# This file is part of Launch Control.
#
# Launch Control is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License version 3
# as published by the Free Software Foundation
#
# Launch Control is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Launch Control.  If not, see <http://www.gnu.org/licenses/>.

"""
Unit tests for dashboard_app.views.bundle_stream_list
"""

from django.contrib.auth.models import User
from django.core.urlresolvers import reverse

from dashboard_app.tests import fixtures
from dashboard_app.tests.utils import (
    DashboardViewsTestCase,
    TestClient,
)

class BundleStreamListViewAnonymousTest(DashboardViewsTestCase):

    scenarios = [
        ('empty', {
            'bundle_streams': [],
        }),
        ('public_streams', {
            'bundle_streams': [
                '/anonymous/',
                '/anonymous/name/',
                '/public/personal/user/',
                '/public/personal/user/name/',
                '/public/team/group/',
                '/public/team/group/name/',
            ]
        }),
        ('private_streams', {
            'bundle_streams': [
                '/private/personal/user/',
                '/private/personal/user/name/',
                '/private/team/group/',
                '/private/team/group/name/',
            ]
        }),
    ]

    def setUp(self):
        super(BundleStreamListViewAnonymousTest, self).setUp()
        self.user = None

    def test_status_code(self):
        response = self.client.get(reverse("dashboard_app.views.bundle_stream_list"))
        self.assertEqual(response.status_code, 200)

    def test_template_used(self):
        response = self.client.get(reverse("dashboard_app.views.bundle_stream_list"))
        self.assertTemplateUsed(response,
                "dashboard_app/bundle_stream_list.html")

    def test_listed_bundles_are_the_ones_we_should_see(self):
        with fixtures.created_bundle_streams(self.bundle_streams) as bundle_streams:
            response = self.client.get(reverse("dashboard_app.views.bundle_stream_list"))
            expected_bsl = sorted(
                    [bundle_stream.pk for bundle_stream in
                        bundle_streams if
                        bundle_stream.is_accessible_by(self.user)])
            effective_bsl = sorted(
                    [bundle_stream.pk for bundle_stream in
                        response.context['bundle_stream_list']])
            self.assertEqual(effective_bsl, expected_bsl)

class BundleStreamListViewAuthorizedTest(BundleStreamListViewAnonymousTest):

    def setUp(self):
        super(BundleStreamListViewAuthorizedTest, self).setUp()
        self.client = TestClient()
        self.user = User.objects.create(username='user')
        self.user.groups.create(name='group')
        self.client.login_user(self.user)
<commit_msg>Less code duplication<commit_after># Copyright (C) 2010 Linaro Limited
#
# Author: Zygmunt Krynicki <zygmunt.krynicki@linaro.org>
#
# This file is part of Launch Control.
#
# Launch Control is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License version 3
# as published by the Free Software Foundation
#
# Launch Control is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Launch Control.  If not, see <http://www.gnu.org/licenses/>.

"""
Unit tests for dashboard_app.views.bundle_stream_list
"""

from django.contrib.auth.models import User
from django.core.urlresolvers import reverse

from dashboard_app.tests import fixtures
from dashboard_app.tests.utils import (
    DashboardViewsTestCase,
    TestClient,
)

class BundleStreamListViewAnonymousTest(DashboardViewsTestCase):

    scenarios = [
        ('empty', {
            'bundle_streams': [],
        }),
        ('public_streams', {
            'bundle_streams': [
                '/anonymous/',
                '/anonymous/name/',
                '/public/personal/user/',
                '/public/personal/user/name/',
                '/public/team/group/',
                '/public/team/group/name/',
            ]
        }),
        ('private_streams', {
            'bundle_streams': [
                '/private/personal/user/',
                '/private/personal/user/name/',
                '/private/team/group/',
                '/private/team/group/name/',
            ]
        }),
    ]

    def setUp(self):
        super(BundleStreamListViewAnonymousTest, self).setUp()
        self.url = reverse("dashboard_app.views.bundle_stream_list")
        self.user = None

    def test_status_code(self):
        response = self.client.get(self.url)
        self.assertEqual(response.status_code, 200)

    def test_template_used(self):
        response = self.client.get(self.url)
        self.assertTemplateUsed(response,
                "dashboard_app/bundle_stream_list.html")

    def test_listed_bundles_are_the_ones_we_should_see(self):
        with fixtures.created_bundle_streams(self.bundle_streams) as bundle_streams:
            response = self.client.get(self.url)
            expected_bsl = sorted(
                    [bundle_stream.pk for bundle_stream in
                        bundle_streams if
                        bundle_stream.is_accessible_by(self.user)])
            effective_bsl = sorted(
                    [bundle_stream.pk for bundle_stream in
                        response.context['bundle_stream_list']])
            self.assertEqual(effective_bsl, expected_bsl)

class BundleStreamListViewAuthorizedTest(BundleStreamListViewAnonymousTest):

    def setUp(self):
        super(BundleStreamListViewAuthorizedTest, self).setUp()
        self.client = TestClient()
        self.user = User.objects.create(username='user')
        self.user.groups.create(name='group')
        self.client.login_user(self.user)
<|endoftext|>


<commit_before># Copyright (c) 2012, 2017-2018 ARM Limited
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
# not be construed as granting a license to any other intellectual
# property including but not limited to intellectual property relating
# to a hardware implementation of the functionality of the software
# licensed hereunder.  You may use the software subject to the license
# terms below provided that you ensure that this notice is replicated
# unmodified and in its entirety in all distributions of the software,
# modified or unmodified, in source code or in binary form.
#
# Copyright (c) 2006-2007 The Regents of The University of Michigan
# Copyright (c) 2009 Advanced Micro Devices, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import math
import m5
from m5.objects import *
from m5.defines import buildEnv
from m5.util import addToPath, fatal

addToPath('../')

from common import ObjectList
from common import MemConfig
from common import FileSystemConfig

from topologies import *
from network import Network

def define_options(parser):
    # By default, ruby uses the simple timing cpu
    parser.set_defaults(cpu_type="TimingSimpleCPU")

    parser.add_option("--ruby-clock", action="store", type="string",
                      default='2GHz',
                      help="Clock for blocks running at Ruby system's speed")

    parser.add_option("--access-backing-store", action="store_true", default=False,
                      help="Should ruby maintain a second copy of memory")

    # Options related to cache structure
    parser.add_option("--ports", action="store", type="int", default=4,
                      help="used of transitions per cycle which is a proxy \
                            for the number of ports.")

    # network options are in network/Network.py

    # ruby mapping options
    parser.add_option("--numa-high-bit", type="int", default=0,
                      help="high order address bit to use for numa mapping. " \
                           "0 = highest bit, not specified = lowest bit")
    parser.add_option("--interleaving-bits", type="int", default=0,
                      help="number of bits to specify interleaving " \
                           "in directory, memory controllers and caches. "
                           "0 = not specified")
    parser.add_option("--xor-low-bit", type="int", default=20,
                      help="hashing bit for channel selection" \
                           "see MemConfig for explanation of the default"\
                           "parameter. If set to 0, xor_high_bit is also"\
                           "set to 0.")

    parser.add_option("--recycle-latency", type="int", default=10,
                      help="Recycle latency for ruby controller input buffers")

    protocol = buildEnv['PROTOCOL']
    exec("from . import %s" % protocol)
    eval("%s.define_options(parser)" % protocol)
    Network.define_options(parser)

def setup_memory_controllers(system, ruby, dir_cntrls, options):
    if (options.numa_high_bit):
        block_size_bits = options.numa_high_bit + 1 - \
                          int(math.log(options.num_dirs, 2))
        ruby.block_size_bytes = 2 ** (block_size_bits)
    else:
        ruby.block_size_bytes = options.cacheline_size

    ruby.memory_size_bits = 48

    index = 0
    mem_ctrls = []
    crossbars = []

    if options.numa_high_bit:
        dir_bits = int(math.log(options.num_dirs, 2))
        intlv_size = 2 ** (options.numa_high_bit - dir_bits + 1)
    else:
        # if the numa_bit is not specified, set the directory bits as the
        # lowest bits above the block offset bits
        intlv_size = options.cacheline_size

    # Sets bits to be used for interleaving.  Creates memory controllers
    # attached to a directory controller.  A separate controller is created
    # for each address range as the abstract memory can handle only one
    # contiguous address range as of now.
    for dir_cntrl in dir_cntrls:
        crossbar = None
        if len(system.mem_ranges) > 1:
            crossbar = IOXBar()
            crossbars.append(crossbar)
            dir_cntrl.memory = crossbar.slave

        dir_ranges = []
        for r in system.mem_ranges:
            mem_type = ObjectList.mem_list.get(options.mem_type)
            dram_intf = MemConfig.create_mem_intf(mem_type, r, index,
                options.num_dirs, int(math.log(options.num_dirs, 2)),
                intlv_size, options.xor_low_bit)
            mem_ctrl = m5.objects.MemCtrl(dram = dram_intf)

            if options.access_backing_store:
                dram_intf.kvm_map=False

            mem_ctrls.append(mem_ctrl)
            dir_ranges.append(mem_ctrl.dram.range)

            if crossbar != None:
                mem_ctrl.port = crossbar.master
            else:
                mem_ctrl.port = dir_cntrl.memory

            # Enable low-power DRAM states if option is set
            if issubclass(mem_type, DRAMInterface):
                mem_ctrl.dram.enable_dram_powerdown = \
                        options.enable_dram_powerdown

        index += 1
        dir_cntrl.addr_ranges = dir_ranges

    system.mem_ctrls = mem_ctrls

    if len(crossbars) > 0:
        ruby.crossbars = crossbars

def create_topology(controllers, options):
    """ Called from create_system in configs/ruby/<protocol>.py
        Must return an object which is a subclass of BaseTopology
        found in configs/topologies/BaseTopology.py
        This is a wrapper for the legacy topologies.
    """
    exec("import topologies.%s as Topo" % options.topology)
    topology = eval("Topo.%s(controllers)" % options.topology)
    return topology

def create_system(options, full_system, system, piobus = None, dma_ports = [],
                  bootmem=None):

    system.ruby = RubySystem()
    ruby = system.ruby

    # Generate pseudo filesystem
    FileSystemConfig.config_filesystem(system, options)

    # Create the network object
    (network, IntLinkClass, ExtLinkClass, RouterClass, InterfaceClass) = \
        Network.create_network(options, ruby)
    ruby.network = network

    protocol = buildEnv['PROTOCOL']
    exec("from . import %s" % protocol)
    try:
        (cpu_sequencers, dir_cntrls, topology) = \
             eval("%s.create_system(options, full_system, system, dma_ports,\
                                    bootmem, ruby)"
                  % protocol)
    except:
        print("Error: could not create sytem for ruby protocol %s" % protocol)
        raise

    # Create the network topology
    topology.makeTopology(options, network, IntLinkClass, ExtLinkClass,
            RouterClass)

    # Register the topology elements with faux filesystem (SE mode only)
    if not full_system:
        topology.registerTopology(options)

    # Initialize network based on topology
    Network.init_network(options, network, InterfaceClass)

    # Create a port proxy for connecting the system port. This is
    # independent of the protocol and kept in the protocol-agnostic
    # part (i.e. here).
    sys_port_proxy = RubyPortProxy(ruby_system = ruby)
    if piobus is not None:
        sys_port_proxy.pio_master_port = piobus.slave

    # Give the system port proxy a SimObject parent without creating a
    # full-fledged controller
    system.sys_port_proxy = sys_port_proxy

    # Connect the system port for loading of binaries etc
    system.system_port = system.sys_port_proxy.slave

    setup_memory_controllers(system, ruby, dir_cntrls, options)

    # Connect the cpu sequencers and the piobus
    if piobus != None:
        for cpu_seq in cpu_sequencers:
            cpu_seq.connectIOPorts(piobus)

    ruby.number_of_virtual_networks = ruby.network.number_of_virtual_networks
    ruby._cpu_ports = cpu_sequencers
    ruby.num_of_sequencers = len(cpu_sequencers)

    # Create a backing copy of physical memory in case required
    if options.access_backing_store:
        ruby.access_backing_store = True
        ruby.phys_mem = SimpleMemory(range=system.mem_ranges[0],
                                     in_addr_map=False)

def create_directories(options, bootmem, ruby_system, system):
    dir_cntrl_nodes = []
    for i in range(options.num_dirs):
        dir_cntrl = Directory_Controller()
        dir_cntrl.version = i
        dir_cntrl.directory = RubyDirectoryMemory()
        dir_cntrl.ruby_system = ruby_system

        exec("ruby_system.dir_cntrl%d = dir_cntrl" % i)
        dir_cntrl_nodes.append(dir_cntrl)

    if bootmem is not None:
        rom_dir_cntrl = Directory_Controller()
        rom_dir_cntrl.directory = RubyDirectoryMemory()
        rom_dir_cntrl.ruby_system = ruby_system
        rom_dir_cntrl.version = i + 1
        rom_dir_cntrl.memory = bootmem.port
        rom_dir_cntrl.addr_ranges = bootmem.range
        return (dir_cntrl_nodes, rom_dir_cntrl)

    return (dir_cntrl_nodes, None)

def send_evicts(options):
    # currently, 2 scenarios warrant forwarding evictions to the CPU:
    # 1. The O3 model must keep the LSQ coherent with the caches
    # 2. The x86 mwait instruction is built on top of coherence invalidations
    # 3. The local exclusive monitor in ARM systems
    if options.cpu_type == "DerivO3CPU" or \
       buildEnv['TARGET_ISA'] in ('x86', 'arm'):
        return True
    return False
<commit_msg>configs: Ruby fixes for SimpleMemory<commit_after># Copyright (c) 2012, 2017-2018, 2021 ARM Limited
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
# not be construed as granting a license to any other intellectual
# property including but not limited to intellectual property relating
# to a hardware implementation of the functionality of the software
# licensed hereunder.  You may use the software subject to the license
# terms below provided that you ensure that this notice is replicated
# unmodified and in its entirety in all distributions of the software,
# modified or unmodified, in source code or in binary form.
#
# Copyright (c) 2006-2007 The Regents of The University of Michigan
# Copyright (c) 2009 Advanced Micro Devices, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import math
import m5
from m5.objects import *
from m5.defines import buildEnv
from m5.util import addToPath, fatal

addToPath('../')

from common import ObjectList
from common import MemConfig
from common import FileSystemConfig

from topologies import *
from network import Network

def define_options(parser):
    # By default, ruby uses the simple timing cpu
    parser.set_defaults(cpu_type="TimingSimpleCPU")

    parser.add_option("--ruby-clock", action="store", type="string",
                      default='2GHz',
                      help="Clock for blocks running at Ruby system's speed")

    parser.add_option("--access-backing-store", action="store_true", default=False,
                      help="Should ruby maintain a second copy of memory")

    # Options related to cache structure
    parser.add_option("--ports", action="store", type="int", default=4,
                      help="used of transitions per cycle which is a proxy \
                            for the number of ports.")

    # network options are in network/Network.py

    # ruby mapping options
    parser.add_option("--numa-high-bit", type="int", default=0,
                      help="high order address bit to use for numa mapping. " \
                           "0 = highest bit, not specified = lowest bit")
    parser.add_option("--interleaving-bits", type="int", default=0,
                      help="number of bits to specify interleaving " \
                           "in directory, memory controllers and caches. "
                           "0 = not specified")
    parser.add_option("--xor-low-bit", type="int", default=20,
                      help="hashing bit for channel selection" \
                           "see MemConfig for explanation of the default"\
                           "parameter. If set to 0, xor_high_bit is also"\
                           "set to 0.")

    parser.add_option("--recycle-latency", type="int", default=10,
                      help="Recycle latency for ruby controller input buffers")

    protocol = buildEnv['PROTOCOL']
    exec("from . import %s" % protocol)
    eval("%s.define_options(parser)" % protocol)
    Network.define_options(parser)

def setup_memory_controllers(system, ruby, dir_cntrls, options):
    if (options.numa_high_bit):
        block_size_bits = options.numa_high_bit + 1 - \
                          int(math.log(options.num_dirs, 2))
        ruby.block_size_bytes = 2 ** (block_size_bits)
    else:
        ruby.block_size_bytes = options.cacheline_size

    ruby.memory_size_bits = 48

    index = 0
    mem_ctrls = []
    crossbars = []

    if options.numa_high_bit:
        dir_bits = int(math.log(options.num_dirs, 2))
        intlv_size = 2 ** (options.numa_high_bit - dir_bits + 1)
    else:
        # if the numa_bit is not specified, set the directory bits as the
        # lowest bits above the block offset bits
        intlv_size = options.cacheline_size

    # Sets bits to be used for interleaving.  Creates memory controllers
    # attached to a directory controller.  A separate controller is created
    # for each address range as the abstract memory can handle only one
    # contiguous address range as of now.
    for dir_cntrl in dir_cntrls:
        crossbar = None
        if len(system.mem_ranges) > 1:
            crossbar = IOXBar()
            crossbars.append(crossbar)
            dir_cntrl.memory = crossbar.slave

        dir_ranges = []
        for r in system.mem_ranges:
            mem_type = ObjectList.mem_list.get(options.mem_type)
            dram_intf = MemConfig.create_mem_intf(mem_type, r, index,
                options.num_dirs, int(math.log(options.num_dirs, 2)),
                intlv_size, options.xor_low_bit)
            if issubclass(mem_type, DRAMInterface):
                mem_ctrl = m5.objects.MemCtrl(dram = dram_intf)
            else:
                mem_ctrl = dram_intf

            if options.access_backing_store:
                dram_intf.kvm_map=False

            mem_ctrls.append(mem_ctrl)
            dir_ranges.append(dram_intf.range)

            if crossbar != None:
                mem_ctrl.port = crossbar.master
            else:
                mem_ctrl.port = dir_cntrl.memory

            # Enable low-power DRAM states if option is set
            if issubclass(mem_type, DRAMInterface):
                mem_ctrl.dram.enable_dram_powerdown = \
                        options.enable_dram_powerdown

        index += 1
        dir_cntrl.addr_ranges = dir_ranges

    system.mem_ctrls = mem_ctrls

    if len(crossbars) > 0:
        ruby.crossbars = crossbars

def create_topology(controllers, options):
    """ Called from create_system in configs/ruby/<protocol>.py
        Must return an object which is a subclass of BaseTopology
        found in configs/topologies/BaseTopology.py
        This is a wrapper for the legacy topologies.
    """
    exec("import topologies.%s as Topo" % options.topology)
    topology = eval("Topo.%s(controllers)" % options.topology)
    return topology

def create_system(options, full_system, system, piobus = None, dma_ports = [],
                  bootmem=None):

    system.ruby = RubySystem()
    ruby = system.ruby

    # Generate pseudo filesystem
    FileSystemConfig.config_filesystem(system, options)

    # Create the network object
    (network, IntLinkClass, ExtLinkClass, RouterClass, InterfaceClass) = \
        Network.create_network(options, ruby)
    ruby.network = network

    protocol = buildEnv['PROTOCOL']
    exec("from . import %s" % protocol)
    try:
        (cpu_sequencers, dir_cntrls, topology) = \
             eval("%s.create_system(options, full_system, system, dma_ports,\
                                    bootmem, ruby)"
                  % protocol)
    except:
        print("Error: could not create sytem for ruby protocol %s" % protocol)
        raise

    # Create the network topology
    topology.makeTopology(options, network, IntLinkClass, ExtLinkClass,
            RouterClass)

    # Register the topology elements with faux filesystem (SE mode only)
    if not full_system:
        topology.registerTopology(options)

    # Initialize network based on topology
    Network.init_network(options, network, InterfaceClass)

    # Create a port proxy for connecting the system port. This is
    # independent of the protocol and kept in the protocol-agnostic
    # part (i.e. here).
    sys_port_proxy = RubyPortProxy(ruby_system = ruby)
    if piobus is not None:
        sys_port_proxy.pio_master_port = piobus.slave

    # Give the system port proxy a SimObject parent without creating a
    # full-fledged controller
    system.sys_port_proxy = sys_port_proxy

    # Connect the system port for loading of binaries etc
    system.system_port = system.sys_port_proxy.slave

    setup_memory_controllers(system, ruby, dir_cntrls, options)

    # Connect the cpu sequencers and the piobus
    if piobus != None:
        for cpu_seq in cpu_sequencers:
            cpu_seq.connectIOPorts(piobus)

    ruby.number_of_virtual_networks = ruby.network.number_of_virtual_networks
    ruby._cpu_ports = cpu_sequencers
    ruby.num_of_sequencers = len(cpu_sequencers)

    # Create a backing copy of physical memory in case required
    if options.access_backing_store:
        ruby.access_backing_store = True
        ruby.phys_mem = SimpleMemory(range=system.mem_ranges[0],
                                     in_addr_map=False)

def create_directories(options, bootmem, ruby_system, system):
    dir_cntrl_nodes = []
    for i in range(options.num_dirs):
        dir_cntrl = Directory_Controller()
        dir_cntrl.version = i
        dir_cntrl.directory = RubyDirectoryMemory()
        dir_cntrl.ruby_system = ruby_system

        exec("ruby_system.dir_cntrl%d = dir_cntrl" % i)
        dir_cntrl_nodes.append(dir_cntrl)

    if bootmem is not None:
        rom_dir_cntrl = Directory_Controller()
        rom_dir_cntrl.directory = RubyDirectoryMemory()
        rom_dir_cntrl.ruby_system = ruby_system
        rom_dir_cntrl.version = i + 1
        rom_dir_cntrl.memory = bootmem.port
        rom_dir_cntrl.addr_ranges = bootmem.range
        return (dir_cntrl_nodes, rom_dir_cntrl)

    return (dir_cntrl_nodes, None)

def send_evicts(options):
    # currently, 2 scenarios warrant forwarding evictions to the CPU:
    # 1. The O3 model must keep the LSQ coherent with the caches
    # 2. The x86 mwait instruction is built on top of coherence invalidations
    # 3. The local exclusive monitor in ARM systems
    if options.cpu_type == "DerivO3CPU" or \
       buildEnv['TARGET_ISA'] in ('x86', 'arm'):
        return True
    return False
<|endoftext|>


<commit_before>from .formatter import *

class FormatterRegistry():
    def __init__(self):
        self.__formatters = []

    def populate(self):
        self.__formatters = [
            ClangFormat(), ElmFormat(), GoFormat(), JavaScriptFormat(),
            JsonFormat(), PythonFormat(), RustFormat(), TerraformFormat()
        ]

    @property
    def all(self):
        return self.__formatters

    @property
    def enabled(self):
        return [x for x in self.all if x.format_on_save]

    def find(self, predicate, default=None):
        return next((x for x in self.all if predicate(x)), default)

    def by_view(self, view):
        source = view.scope_name(0).split(' ')[0].split('.')[1]
        return self.find(lambda x: source in x.sources)

    def by_name(self, name):
        return self.find(lambda x: x.name == name)
<commit_msg>Speed up lookup of formatter by source type<commit_after>from .formatter import *

class FormatterRegistry():
    def __init__(self):
        self.__formatters = []
        self.__formatter_source_map = {}

    def populate(self):
        self.__formatters = [
            ClangFormat(), ElmFormat(), GoFormat(), JavaScriptFormat(),
            JsonFormat(), PythonFormat(), RustFormat(), TerraformFormat()
        ]
        self.__formatter_source_map = dict((source, formatter)
                                           for formatter in self.__formatters
                                           for source in formatter.sources)

    @property
    def all(self):
        return self.__formatters

    @property
    def enabled(self):
        return [x for x in self.all if x.format_on_save]

    def by_view(self, view):
        source = view.scope_name(0).split(' ')[0].split('.')[1]
        return self.__formatter_source_map.get(source)

    def by_name(self, name):
        return next((x for x in self.all if x.name == name))
<|endoftext|>


<commit_before>#!/usr/bin/env python

from fs import _iteratepath, pathsplit

class _ObjectDict(dict):
    pass

class ObjectTree(object):

    """A class to facilitate the creation of tree structures."""

    def __init__(self):
        self.root = _ObjectDict()

    def _locate(self, path):
        current = self.root
        for path_component in path.split('/'):
            if type(current) is not _ObjectDict:
                return None
            node = current.get(path_component, None)
            if node is None:
                return None
            current = node
        return node

    def __setitem__(self, path, object):
        if not path:
            raise IndexError("No path supplied")
        current = self.root
        path, name = path.rsplit('/', 1)
        for path_component in path.split('/'):
            node = current.get(path_component, None)
            if type(node) is not _ObjectDict:
                new_dict = _ObjectDict()
                current[path_component] = new_dict
                current = new_dict
            else:
                current = node
        current[name] = object

    def __getitem__(self, path):
        node = self._locate(path)
        if node is None:
            raise IndexError("Path does not exist")
        return node

    def get(self, path, default):
        node = self._locate(path)
        if node is None:
            return default
        return node

    def isobject(self, path):
        node = self._locate(path)
        return type(node) is not _ObjectDict

    def __contains__(self, path):
        node = self._locate(path)
        return node is not None

    def __iter__(self):
        return iter(self.root)

    def keys(self):
        return self.root.keys()

    def iterkeys(self):
        return self.root.keys()

    def items(self):
        return self.root.items()

    def iteritems(self):
        return self.root.iteritems()

if __name__ == "__main__":

    ot = ObjectTree()
    ot['a/b/c'] = "Hai!"

    print ot['a/b/c']

    ot['a/b/c/d'] = "?"

    print ot['a/b/c'].keys()<commit_msg>Added deleting of items to object tree<commit_after>#!/usr/bin/env python

from fs import _iteratepath, pathsplit

class _ObjectDict(dict):
    pass

class ObjectTree(object):

    """A class to facilitate the creation of tree structures."""

    def __init__(self):
        self.root = _ObjectDict()

    def _locate(self, path):
        current = self.root
        for path_component in path.split('/'):
            if type(current) is not _ObjectDict:
                return None
            node = current.get(path_component, None)
            if node is None:
                return None
            current = node
        return node

    def __setitem__(self, path, object):
        if not path:
            raise IndexError("No path supplied")
        current = self.root
        path, name = path.rsplit('/', 1)
        for path_component in path.split('/'):
            node = current.get(path_component, None)
            if type(node) is not _ObjectDict:
                new_dict = _ObjectDict()
                current[path_component] = new_dict
                current = new_dict
            else:
                current = node
        current[name] = object

    def __getitem__(self, path):
        node = self._locate(path)
        if node is None:
            raise IndexError("Path does not exist")
        return node

    def __delitem__(self, path):
        path, name = path.rsplit('/', 1)
        node = self._locate(path)
        if node is None or type(node) is not _ObjectDict:
            raise IndexError("Path does not exist")
        del node[name]

    def get(self, path, default):
        node = self._locate(path)
        if node is None:
            return default
        return node

    def isobject(self, path):
        node = self._locate(path)
        return type(node) is not _ObjectDict

    def __contains__(self, path):
        node = self._locate(path)
        return node is not None

    def __iter__(self):
        return iter(self.root)

    def keys(self):
        return self.root.keys()

    def iterkeys(self):
        return self.root.keys()

    def items(self):
        return self.root.items()

    def iteritems(self):
        return self.root.iteritems()

if __name__ == "__main__":

    ot = ObjectTree()
    ot['a/b/c'] = "Hai!"

    print ot['a/b/c']

    ot['a/b/c/d'] = "?"

    print ot['a/b/c'].keys()<|endoftext|>


<commit_before>
def calc_SNP_freqs(pops):

	def freq(value,total_count):
		"""Avoid divide by zero errors"""
		if total_count == 0.0:
			return value
		else:
			return value/total_count

	frequencies = []
	coverage = []
	for pop in pops:
		counts =  [float(value) for value in pop.split(":")]
		total_count = float(sum(counts))
		coverage.append(total_count)
		freqs = [freq(value, total_count) for value in counts]
		frequencies.append(freqs)

	return (frequencies, coverage)

def harmonic_mean(values):
	"""calculates harmonic mean from list of integers"""

	fractional_counts = sum([1.0/v for v in values ])
	harmonic_mean = float(len(values))/fractional_counts

	return harmonic_mean

def harmonic_mean_chao(values):
	"""Calculates the harmonic mean following the method suggested by
	Anne Chao. The formula is: 1/[(1/A)+var(D)(1/A)**3]. Used for
	calculating multilocus Dest."""

	count = float(len(values))
	if count == 0.0:
		return 0.0
	elif sum(values)/count == 0:
		return 0.0
	else:
		A = sum(values)/count
		varD = sum([(v-A)**2 for v in values])/count
		harmonic_mean_chao = 1/((1/A)+(varD)*pow((1/A),3))
		return harmonic_mean_chao

def Hs_prime_est(allele_freqs, n):
	"""Calculate corrected Hs: the mean within-subpopulation
	heterozygosity (Nei and Chesser 1983)."""

	n = float(n)
	Hj = [1.0-sum([freq**2 for freq in pop[:4]]) for pop in allele_freqs]
	Hs_prime_est = 1/n * sum(Hj)
	return Hs_prime_est

def Hs_est(Hs_prime_est,harm_mean):
	""" Basic Equation: ((2*N_harmonic)/(2*N_harmonic-1))*Hs"""

	Hs = Hs_prime_est
	Hs_est = ((2.0*harm_mean)/(2.0*harm_mean-1.0))*Hs
	return Hs_est

def Ht_prime_est(allele_freqs,n):
	"""Calculate corrected Ht: the heterozygosity of the pooled
	subpopulations (Nei and Chesser 1983)"""

	n = float(n)
	inner = [(1/n*sum(allele_list))**2 for allele_list in zip(*allele_freqs)[:4]]
	Ht_prime_est = 1.0-sum(inner)
	return Ht_prime_est

def Ht_est(Ht_p_est,Hs_est,harm_mean,n):
	"""Basic Equation: Ht+Hs_est/(2*N_harmonic*n)"""

	n = float(n)
	Ht = Ht_p_est
	Ht_est = Ht+Hs_est/(2.0*harm_mean*n)
	return Ht_est

def Gst_est(Ht_est, Hs_est):
	"""Gst = (Ht-Hs)/Ht"""

	if Ht_est == 0.0: return 0.0
	else: Gst_est = (Ht_est-Hs_est)/Ht_est
	return Gst_est

def G_prime_st_est(Ht_est, Hs_est, Gst_est, n):

	n = float(n)
	if (n-1.0)*(1.0-Hs_est) == 0: return 0.0
	else:  G_prime_st = (Gst_est*(n-1.0+Hs_est))/((n-1.0)*(1.0-Hs_est))
	return G_prime_st

def G_double_prime_st_est(Ht_est, Hs_est, n):
	"""G''st = k*(HT-HS)/((k*HT-HS)*(1-HS)"""

	n = float(n)
	if (n*Ht_est-Hs_est)*(1-Hs_est) == 0.0: return 0.0
	else:  G_double_prime_st_est = n*(Ht_est-Hs_est)/((n*Ht_est-Hs_est)*(1-Hs_est))
	return  G_double_prime_st_est

def D_est(Ht_est, Hs_est, n):
	"""((Ht-Hs)/(1.0-Hs))*(n/(n-1))"""
	n = float(n)
	if ((1.0-Hs_est))*(n/(n-1)) == 0.0: return 0.0
	else: D_est = ((Ht_est-Hs_est)/(1.0-Hs_est))*(n/(n-1))
	return D_est

# MULTILOCUS FUNCTIONS
def multilocus_Gst_est(Ht_est, Hs_est):
	"""Averages across loci before calculating Gst."""

	Ht_est = sum(Ht_est)/float(len(Ht_est))
	Hs_est = sum(Hs_est)/float(len(Hs_est))
	if Ht_est == 0.0: return 0.0
	else: Gst_est = (Ht_est-Hs_est)/Ht_est
	return Gst_est

def multilocus_G_prime_st_est(Ht_est, Hs_est,n):
	"""Averages across loci before calculating G'st."""

	Ht_est_ = sum(Ht_est)/float(len(Ht_est))
	Hs_est_ = sum(Hs_est)/float(len(Hs_est))
	G_est = multilocus_Gst_est(Ht_est,Hs_est)
	if (n-1.0)*(1.0-Hs_est_) == 0: return 0.0
	else: G_prime_st = (G_est*(n-1.0+Hs_est_))/((n-1.0)*(1.0-Hs_est_))
	return G_prime_st

def multilocus_G_double_prime_st_est(Ht_est, Hs_est, n):
	"""Averages across loci before calculating G''st."""

	Ht_est = sum(Ht_est)/float(len(Ht_est))
	Hs_est = sum(Hs_est)/float(len(Hs_est))
	# G_est = multilocus_Gst_est(Ht_est,Hs_est)
	if (n*Ht_est-Hs_est)*(1.0-Hs_est) == 0: return 0.0

	else: G_double_prime_st_est = n*(Ht_est-Hs_est)/((n*Ht_est-Hs_est)*(1.0-Hs_est))
	return G_double_prime_st_est

def multilocus_D_est(Ht_est, Hs_est, n):
	"""Calculate Anne Chao's harmonic mean all Dest values."""

	pairs = zip(Ht_est, Hs_est)
	Dest_values = [D_est(pair[0], pair[1], 2) for pair in pairs]
	D_est_ = harmonic_mean_chao(Dest_values)
	return D_est

<commit_msg>Fixed missing underscore<commit_after>
def calc_SNP_freqs(pops):

	def freq(value,total_count):
		"""Avoid divide by zero errors"""
		if total_count == 0.0:
			return value
		else:
			return value/total_count

	frequencies = []
	coverage = []
	for pop in pops:
		counts =  [float(value) for value in pop.split(":")]
		total_count = float(sum(counts))
		coverage.append(total_count)
		freqs = [freq(value, total_count) for value in counts]
		frequencies.append(freqs)

	return (frequencies, coverage)

def harmonic_mean(values):
	"""calculates harmonic mean from list of integers"""

	fractional_counts = sum([1.0/v for v in values ])
	harmonic_mean = float(len(values))/fractional_counts

	return harmonic_mean

def harmonic_mean_chao(values):
	"""Calculates the harmonic mean following the method suggested by
	Anne Chao. The formula is: 1/[(1/A)+var(D)(1/A)**3]. Used for
	calculating multilocus Dest."""

	count = float(len(values))
	if count == 0.0:
		return 0.0
	elif sum(values)/count == 0:
		return 0.0
	else:
		A = sum(values)/count
		varD = sum([(v-A)**2 for v in values])/count
		harmonic_mean_chao = 1/((1/A)+(varD)*pow((1/A),3))
		return harmonic_mean_chao

def Hs_prime_est(allele_freqs, n):
	"""Calculate corrected Hs: the mean within-subpopulation
	heterozygosity (Nei and Chesser 1983)."""

	n = float(n)
	Hj = [1.0-sum([freq**2 for freq in pop[:4]]) for pop in allele_freqs]
	Hs_prime_est = 1/n * sum(Hj)
	return Hs_prime_est

def Hs_est(Hs_prime_est,harm_mean):
	""" Basic Equation: ((2*N_harmonic)/(2*N_harmonic-1))*Hs"""

	Hs = Hs_prime_est
	Hs_est = ((2.0*harm_mean)/(2.0*harm_mean-1.0))*Hs
	return Hs_est

def Ht_prime_est(allele_freqs,n):
	"""Calculate corrected Ht: the heterozygosity of the pooled
	subpopulations (Nei and Chesser 1983)"""

	n = float(n)
	inner = [(1/n*sum(allele_list))**2 for allele_list in zip(*allele_freqs)[:4]]
	Ht_prime_est = 1.0-sum(inner)
	return Ht_prime_est

def Ht_est(Ht_p_est,Hs_est,harm_mean,n):
	"""Basic Equation: Ht+Hs_est/(2*N_harmonic*n)"""

	n = float(n)
	Ht = Ht_p_est
	Ht_est = Ht+Hs_est/(2.0*harm_mean*n)
	return Ht_est

def Gst_est(Ht_est, Hs_est):
	"""Gst = (Ht-Hs)/Ht"""

	if Ht_est == 0.0: return 0.0
	else: Gst_est = (Ht_est-Hs_est)/Ht_est
	return Gst_est

def G_prime_st_est(Ht_est, Hs_est, Gst_est, n):

	n = float(n)
	if (n-1.0)*(1.0-Hs_est) == 0: return 0.0
	else:  G_prime_st = (Gst_est*(n-1.0+Hs_est))/((n-1.0)*(1.0-Hs_est))
	return G_prime_st

def G_double_prime_st_est(Ht_est, Hs_est, n):
	"""G''st = k*(HT-HS)/((k*HT-HS)*(1-HS)"""

	n = float(n)
	if (n*Ht_est-Hs_est)*(1-Hs_est) == 0.0: return 0.0
	else:  G_double_prime_st_est = n*(Ht_est-Hs_est)/((n*Ht_est-Hs_est)*(1-Hs_est))
	return  G_double_prime_st_est

def D_est(Ht_est, Hs_est, n):
	"""((Ht-Hs)/(1.0-Hs))*(n/(n-1))"""
	n = float(n)
	if ((1.0-Hs_est))*(n/(n-1)) == 0.0: return 0.0
	else: D_est = ((Ht_est-Hs_est)/(1.0-Hs_est))*(n/(n-1))
	return D_est

# MULTILOCUS FUNCTIONS
def multilocus_Gst_est(Ht_est, Hs_est):
	"""Averages across loci before calculating Gst."""

	Ht_est = sum(Ht_est)/float(len(Ht_est))
	Hs_est = sum(Hs_est)/float(len(Hs_est))
	if Ht_est == 0.0: return 0.0
	else: Gst_est = (Ht_est-Hs_est)/Ht_est
	return Gst_est

def multilocus_G_prime_st_est(Ht_est, Hs_est,n):
	"""Averages across loci before calculating G'st."""

	Ht_est_ = sum(Ht_est)/float(len(Ht_est))
	Hs_est_ = sum(Hs_est)/float(len(Hs_est))
	G_est = multilocus_Gst_est(Ht_est,Hs_est)
	if (n-1.0)*(1.0-Hs_est_) == 0: return 0.0
	else: G_prime_st = (G_est*(n-1.0+Hs_est_))/((n-1.0)*(1.0-Hs_est_))
	return G_prime_st

def multilocus_G_double_prime_st_est(Ht_est, Hs_est, n):
	"""Averages across loci before calculating G''st."""

	Ht_est = sum(Ht_est)/float(len(Ht_est))
	Hs_est = sum(Hs_est)/float(len(Hs_est))
	# G_est = multilocus_Gst_est(Ht_est,Hs_est)
	if (n*Ht_est-Hs_est)*(1.0-Hs_est) == 0: return 0.0

	else: G_double_prime_st_est = n*(Ht_est-Hs_est)/((n*Ht_est-Hs_est)*(1.0-Hs_est))
	return G_double_prime_st_est

def multilocus_D_est(Ht_est, Hs_est, n):
	"""Calculate Anne Chao's harmonic mean all Dest values."""

	pairs = zip(Ht_est, Hs_est)
	Dest_values = [D_est(pair[0], pair[1], 2) for pair in pairs]
	D_est_ = harmonic_mean_chao(Dest_values)
	return D_est_

<|endoftext|>


<commit_before>"""Handles secret getters and setters for Scale"""

import ast
import jwt
import json
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning

from django.conf import settings
from vault.exceptions import InvalidSecretsAuthorization, InvalidSecretsRequest, InvalidSecretsToken, \
    InvalidSecretsValue

class SecretsHandler(object):
    """Represents a secrets handler for setting and retrieving secrets
    """

    def __init__(self):
        """Creates a secrets handler object.  The backend is initially tested to ensure it exists and Scale can
        authenticate properly with it.
        """

        self.secrets_error_codes = {
            301: 'Status Code 301 - Too many "/" separators in URL.',
            400: 'Status Code 400 - Invalid request, missing or invalid data.',
            403: 'Status Code 403 - Forbidden, authentication details are incorrect or no access to feature.',
            404: 'Status Code 404 - Invalid path, secret or backend not found.',
            405: 'Status Code 405 - Unsupported operation.',
            409: 'Status Code 409 - Secret or secret store already exists.',
            429: 'Status Code 429 - Rate limit exceeded.',
            500: 'Status Code 500 - Internal server error.',
            503: 'Status Code 503 - Secrets store is down for maintenance or is currently sealed.',
        }

        self.secrets_url = settings.SECRETS_URL
        self.secrets_token = settings.SECRETS_TOKEN
        self.service_account = settings.DCOS_SERVICE_ACCOUNT
        self.raise_ssl_warnings = settings.SECRETS_SSL_WARNINGS

        if not self.raise_ssl_warnings:
            requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

        if self.service_account:
            self.dcos_token = self._dcos_authenticate()
        else:
            self.dcos_token = None
            self._vault_authenticate()

        self._check_secrets_backend()

    def get_job_type_secrets(self, job_name):
        """Retrieves the secrets located at the job_name within the backend

        :param job_name: path within the secrets backend that the secret is stored
        :type job_name: str
        :return: secret_values
        :rtype: str
        """

        url = self.secrets_url

        if self.dcos_token:
            url = ''.join([url, '/secret/default/scale/job-type/', job_name])
            headers = {
                'Content-Type': 'application/json',
                'Authorization': self.dcos_token
            }
            get_secret = self._make_request('GET', url, headers=headers)

            try:
                response = get_secret.json()
                secret_values = ast.literal_eval(response['value'])
            except SyntaxError:
                raise InvalidSecretsValue('The DCOS secrets value could not be converted to JSON.')

        else:
            url = url + '/secret/scale/job-type/' + job_name
            headers = {
                'Content-Type': 'application/json',
                'X-Vault-Token': self.secrets_token
            }
            get_secret = self._make_request('GET', url, headers=headers)

            response = get_secret.json()
            secret_values = response['data']

        return secret_values

    def list_job_types(self):
        """Gets the names of all job types that have secrets

        :return: all_job_types
        :rtype: [string]
        """

        url = self.secrets_url

        if self.dcos_token:
            url += '/secret/default/scale/job-type?list=true'
            headers = {
                'Content-Type': 'application/json',
                'Authorization': self.dcos_token
            }
            list_jobs = self._make_request('GET', url, headers=headers)
            response = list_jobs.json()
            all_job_types = response['array']

        else:
            url += '/secret/scale/job-type'
            headers = {
                'Content-Type': 'application/json',
                'X-Vault-Token': self.secrets_token
            }
            list_jobs = self._make_request('LIST', url, headers)
            response = list_jobs.json()
            all_job_types = response['data']['keys']

        return all_job_types

    def set_job_type_secrets(self, job_name, secret_json):
        """write job-type secrets to the secrets backend

        :param job_name: name of the job that the screts belong to.
        :type job_name: str
        :param secret_json: dict with name:value pairs for all secrets associated with the job
        :type secret_json: dict
        :return:
        """

        url = self.secrets_url
        secret_values = json.dumps(secret_json)

        if self.dcos_token:
            url = ''.join([url, '/secret/default/scale/job-type/', job_name])
            headers = {
                'Content-Type': 'application/json',
                'Authorization': self.dcos_token
            }
            data = json.dumps({
                'author': 'scale',
                'description': 'Secrets for Scale job ' + job_name,
                'value': secret_values
            })
            self._make_request('PUT', url, headers, data)

        else:
            url = ''.join([url, '/secret/scale/job-type/', job_name])
            headers = {
                'Content-Type': 'application/json',
                'X-Vault-Token': self.secrets_token
            }
            data = secret_values
            self._make_request('PUT', url, headers, data)

    def _check_secrets_backend(self):
        """Validates that Scale can transact with the secrets backend properly.
        """

        url = self.secrets_url

        if self.dcos_token:
            url += '/secret/default/scale?list=true'
            headers = {
                'Content-Type': 'application/json',
                'Authorization': self.dcos_token
            }
            self._make_request('GET', url, headers=headers)

        else:
            url += '/sys/mounts'
            headers = {
                "Content-Type": "application/json",
                "X-Vault-Token": self.secrets_token
            }
            check_mount = self._make_request('GET', url, headers=headers)

            if 'scale/' not in json.loads(check_mount.content).keys():
                self._create_vault_store()

    def _create_vault_store(self):
        """Create a new store within the Vault backend that will be used for all Scale secrets.
        """

        url = self.secrets_url + '/sys/mounts/scale'
        headers = {
            "Content-Type": "application/json",
            "X-Vault-Token": self.secrets_token
        }
        data = json.dumps({
            'type': 'generic',
            'description': 'Secrets store for all secrets used by Scale'
        })
        self._make_request('POST', url, headers=headers, data=data)

        # A store needs at least one secret in it for other functions to work...
        backends = ['/job-type/placeholder', '/internal/placeholder', '/storage/placeholder']
        data = json.dumps({'empty': 'secret'})

        for folder in backends:
            url = self.secrets_url + '/secret/scale' + folder
            self._make_request('PUT', url, headers, data)

    def _dcos_authenticate(self):
        """Authenticate with DC/OS Vault backend and expect a status code 200 returned.
        """

        try:
            token = jwt.encode({'uid': self.service_account}, self.secrets_token, algorithm='RS256')
        except ValueError:
            raise InvalidSecretsToken('The provided token could not be encoded: ')

        url = self.secrets_url + '/acs/api/v1/auth/login'
        data = json.dumps({
            'uid': self.service_account, 'token':token
        })
        request_auth = self._make_request('POST', url, data=data)
        self.secrets_url += '/secrets/v1'
        access_token = [k + '=' + v for k, v in request_auth.json().items()][0]

        return access_token

    def _make_request(self, method, url, headers=None, data=None):
        """Make a request to the secrets backend with the provided variables

        :param method: string that determines GET or POST request type
        :type method: str
        :param url: string containing the url for the request
        :type url: str
        :param headers: headers to attach to the request
        :type headers: dict
        :param data: data to attach to the request
        :type data: json

        :return: an object containing information from the request
        :rtype: requests.request
        """

        if not headers:
            headers = {
                'Content-Type': 'application/json'
            }

        if not data:
            data = {}

        r = requests.request(method=method, url=url, headers=headers, data=data, verify=self.raise_ssl_warnings)

        if r.status_code in self.secrets_error_codes:
            if r.status_code == 403:
                raise InvalidSecretsAuthorization('Permission was denied when getting a secret')
            else:
                raise InvalidSecretsRequest('Invalid request return: ' + self.secrets_error_codes[r.status_code])

        return r

    def _vault_authenticate(self):
        """Authenticate with Vault and expect a status code 200 returned.
        """

        url = self.secrets_url + '/v1/sys/health'
        self._make_request('GET', url)
        self.secrets_url += '/v1'
<commit_msg>Refactored set_secrets for uniformity<commit_after>"""Handles secret getters and setters for Scale"""

import ast
import jwt
import json
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning

from django.conf import settings
from vault.exceptions import InvalidSecretsAuthorization, InvalidSecretsRequest, InvalidSecretsToken, \
    InvalidSecretsValue

class SecretsHandler(object):
    """Represents a secrets handler for setting and retrieving secrets
    """

    def __init__(self):
        """Creates a secrets handler object.  The backend is initially tested to ensure it exists and Scale can
        authenticate properly with it.
        """

        self.secrets_error_codes = {
            301: 'Status Code 301 - Too many "/" separators in URL.',
            400: 'Status Code 400 - Invalid request, missing or invalid data.',
            403: 'Status Code 403 - Forbidden, authentication details are incorrect or no access to feature.',
            404: 'Status Code 404 - Invalid path, secret or backend not found.',
            405: 'Status Code 405 - Unsupported operation.',
            409: 'Status Code 409 - Secret or secret store already exists.',
            429: 'Status Code 429 - Rate limit exceeded.',
            500: 'Status Code 500 - Internal server error.',
            503: 'Status Code 503 - Secrets store is down for maintenance or is currently sealed.',
        }

        self.secrets_url = settings.SECRETS_URL
        self.secrets_token = settings.SECRETS_TOKEN
        self.service_account = settings.DCOS_SERVICE_ACCOUNT
        self.raise_ssl_warnings = settings.SECRETS_SSL_WARNINGS

        if not self.raise_ssl_warnings:
            requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

        if self.service_account:
            self.dcos_token = self._dcos_authenticate()
        else:
            self.dcos_token = None
            self._vault_authenticate()

        self._check_secrets_backend()

    def get_job_type_secrets(self, job_name):
        """Retrieves the secrets located at the job_name within the backend

        :param job_name: path within the secrets backend that the secret is stored
        :type job_name: str
        :return: secret_values
        :rtype: str
        """

        url = self.secrets_url

        if self.dcos_token:
            url = ''.join([url, '/secret/default/scale/job-type/', job_name])
            headers = {
                'Content-Type': 'application/json',
                'Authorization': self.dcos_token
            }
            get_secret = self._make_request('GET', url, headers=headers)

            try:
                response = get_secret.json()
                secret_values = ast.literal_eval(response['value'])
            except SyntaxError:
                raise InvalidSecretsValue('The DCOS secrets value could not be converted to JSON.')

        else:
            url = url + '/secret/scale/job-type/' + job_name
            headers = {
                'Content-Type': 'application/json',
                'X-Vault-Token': self.secrets_token
            }
            get_secret = self._make_request('GET', url, headers=headers)

            response = get_secret.json()
            secret_values = response['data']

        return secret_values

    def list_job_types(self):
        """Gets the names of all job types that have secrets

        :return: all_job_types
        :rtype: [string]
        """

        url = self.secrets_url

        if self.dcos_token:
            url += '/secret/default/scale/job-type?list=true'
            headers = {
                'Content-Type': 'application/json',
                'Authorization': self.dcos_token
            }
            list_jobs = self._make_request('GET', url, headers=headers)
            response = list_jobs.json()
            all_job_types = response['array']

        else:
            url += '/secret/scale/job-type'
            headers = {
                'Content-Type': 'application/json',
                'X-Vault-Token': self.secrets_token
            }
            list_jobs = self._make_request('LIST', url, headers)
            response = list_jobs.json()
            all_job_types = response['data']['keys']

        return all_job_types

    def set_job_type_secrets(self, job_name, secrets):
        """write job-type secrets to the secrets backend

        :param job_name: name of the job that the secrets belong to.
                         Format: [job_name]-[job_version]
        :type job_name: str
        :param secrets: dict with name:value pairs for all secrets associated with the job
        :type secrets: dict
        :return:
        """

        url = self.secrets_url
        secret_values = json.dumps(secrets)

        if self.dcos_token:
            url = ''.join([url, '/secret/default/scale/job-type/', job_name])
            headers = {
                'Content-Type': 'application/json',
                'Authorization': self.dcos_token
            }
            data = json.dumps({
                'author': 'scale',
                'description': 'Secrets for Scale job ' + job_name,
                'value': secret_values
            })
            self._make_request('PUT', url, headers, data)

        else:
            url = ''.join([url, '/secret/scale/job-type/', job_name])
            headers = {
                'Content-Type': 'application/json',
                'X-Vault-Token': self.secrets_token
            }
            data = secret_values
            self._make_request('PUT', url, headers, data)

    def _check_secrets_backend(self):
        """Validates that Scale can transact with the secrets backend properly.
        """

        url = self.secrets_url

        if self.dcos_token:
            url += '/secret/default/scale?list=true'
            headers = {
                'Content-Type': 'application/json',
                'Authorization': self.dcos_token
            }
            self._make_request('GET', url, headers=headers)

        else:
            url += '/sys/mounts'
            headers = {
                "Content-Type": "application/json",
                "X-Vault-Token": self.secrets_token
            }
            check_mount = self._make_request('GET', url, headers=headers)

            if 'scale/' not in json.loads(check_mount.content).keys():
                self._create_vault_store()

    def _create_vault_store(self):
        """Create a new store within the Vault backend that will be used for all Scale secrets.
        """

        url = self.secrets_url + '/sys/mounts/scale'
        headers = {
            "Content-Type": "application/json",
            "X-Vault-Token": self.secrets_token
        }
        data = json.dumps({
            'type': 'generic',
            'description': 'Secrets store for all secrets used by Scale'
        })
        self._make_request('POST', url, headers=headers, data=data)

        # A store needs at least one secret in it for other functions to work...
        backends = ['/job-type/placeholder', '/internal/placeholder', '/storage/placeholder']
        data = json.dumps({'empty': 'secret'})

        for folder in backends:
            url = self.secrets_url + '/secret/scale' + folder
            self._make_request('PUT', url, headers, data)

    def _dcos_authenticate(self):
        """Authenticate with DC/OS Vault backend and expect a status code 200 returned.
        """

        try:
            token = jwt.encode({'uid': self.service_account}, self.secrets_token, algorithm='RS256')
        except ValueError:
            raise InvalidSecretsToken('The provided token could not be encoded: ')

        url = self.secrets_url + '/acs/api/v1/auth/login'
        data = json.dumps({
            'uid': self.service_account, 'token':token
        })
        request_auth = self._make_request('POST', url, data=data)
        self.secrets_url += '/secrets/v1'
        access_token = [k + '=' + v for k, v in request_auth.json().items()][0]

        return access_token

    def _make_request(self, method, url, headers=None, data=None):
        """Make a request to the secrets backend with the provided variables

        :param method: string that determines GET or POST request type
        :type method: str
        :param url: string containing the url for the request
        :type url: str
        :param headers: headers to attach to the request
        :type headers: dict
        :param data: data to attach to the request
        :type data: json

        :return: an object containing information from the request
        :rtype: requests.request
        """

        if not headers:
            headers = {
                'Content-Type': 'application/json'
            }

        if not data:
            data = {}

        r = requests.request(method=method, url=url, headers=headers, data=data, verify=self.raise_ssl_warnings)

        if r.status_code in self.secrets_error_codes:
            if r.status_code == 403:
                raise InvalidSecretsAuthorization('Permission was denied when getting a secret')
            else:
                raise InvalidSecretsRequest('Invalid request return: ' + self.secrets_error_codes[r.status_code])

        return r

    def _vault_authenticate(self):
        """Authenticate with Vault and expect a status code 200 returned.
        """

        url = self.secrets_url + '/v1/sys/health'
        self._make_request('GET', url)
        self.secrets_url += '/v1'
<|endoftext|>


<commit_before>from ModernEnigma import ModernEnigma
from CharIndexMap import CharIndexMap
from Wiring import Wiring
from Rotor import Rotor
from Reflector import Reflector
from Plugboard import PlugBoard
import random
class EnigmaDynamicFactory(object):

    def __init__(self):
        pass

    def createEnigmaMachineFromModel(self,modelNo):
        cfg=self.createMachineConfig(modelNo)
        mc=self.createEnigmaMachineFromConfig(cfg)
        return mc

    def createMachineConfig(self,modelNo):
        seedStr=modelNo
        random.seed(seedStr)
        machineConfig={"ROTORS":[]}
        rotorCount=self.nextInt(CharIndexMap.getRangeSize()//5,CharIndexMap.getRangeSize()//2)
        for i in range(rotorCount):
            rotorConfig=self.createRotorConfig(i)
            machineConfig["ROTORS"].append(rotorConfig)
        machineConfig["REFLECTOR"]=self.createReflectorConfig(self.nextInt())
        machineConfig["PLUGBOARD"]=self.createPlugboardConfig(self.nextInt())
        return machineConfig

    def createEnigmaMachineFromConfig(self,config):
        rotorCfgList=config["ROTORS"]
        rotorList=[]
        for r in rotorCfgList:
            rotorList.append(Rotor(r["ID"],Wiring(r["wiring"]),r["noch"]))
        plugboard=PlugBoard(Wiring(config["PLUGBOARD"]["wiring"]))
        reflector=Reflector(Wiring(config["REFLECTOR"]["wiring"]))
        mc=ModernEnigma(rotorList,reflector,plugboard)
        return mc

    def createReflectorConfig(self,id):
        reflectorConfig={"ID":id}
        reflectorConfig["wiring"]=self.seqToStr(self.getShuffledSequence())
        return reflectorConfig

    def createRotorConfig(self,id):
        rotorConfig={"ID":id}
        rotorConfig["wiring"]=self.seqToStr(self.getShuffledSequence())
        rotorConfig["noch"]=self.seqToStr(self.getSampleSeq())
        return rotorConfig

    def createPlugboardConfig(self,id):
        plugboardConfig={"ID":id}
        plugboardConfig["wiring"]=self.seqToStr(CharIndexMap.getRange())
        return plugboardConfig

    def seqToStr(self,seq):
        result=""
        for item in seq:
            result+=item
        return result

    def nextInt(self,a=0,b=CharIndexMap.getRangeSize()):
        return random.randint(a,b)
    def getShuffledSequence(self,seq=CharIndexMap.getRange()):
        result=seq
        times=self.nextInt()
        for t in range(times):
            #to keep the original and copy it
            result= sorted(result, key=lambda k: random.random())
        return result
    def getSampleSeq(self,seq=CharIndexMap.getRange()):
        result=random.sample(seq,self.nextInt(0,CharIndexMap.getRangeSize()//2))
        return result

mcCfg=EnigmaDynamicFactory().createMachineConfig("1")
mc1=EnigmaDynamicFactory().createEnigmaMachineFromConfig(mcCfg)
msg="AAAAAAA"
print("MSG:"+msg)
encMsg=""
for c in msg:
    encMsg+=mc1.processKeyPress(c)
print ("Encrpyted:"+encMsg)

mc2=EnigmaDynamicFactory().createEnigmaMachineFromConfig(mcCfg)
decMsg=""
for c in encMsg:
    decMsg+=mc2.processKeyPress(c)

print("Decrypted:"+decMsg)

<commit_msg>fixed reflector generation<commit_after>from ModernEnigma import ModernEnigma
from CharIndexMap import CharIndexMap
from Wiring import Wiring
from Rotor import Rotor
from Reflector import Reflector
from Plugboard import PlugBoard
import random
class EnigmaDynamicFactory(object):

    def __init__(self):
        pass

    def createEnigmaMachineFromModel(self,modelNo):
        cfg=self.createMachineConfig(modelNo)
        mc=self.createEnigmaMachineFromConfig(cfg)
        return mc

    def createMachineConfig(self,modelNo):
        seedStr=modelNo
        random.seed(seedStr)
        machineConfig={"ROTORS":[]}
        rotorCount=self.nextInt(CharIndexMap.getRangeSize()//5,CharIndexMap.getRangeSize()//2)
        for i in range(rotorCount):
            rotorConfig=self.createRotorConfig(i)
            machineConfig["ROTORS"].append(rotorConfig)
        machineConfig["REFLECTOR"]=self.createReflectorConfig(self.nextInt())
        machineConfig["PLUGBOARD"]=self.createPlugboardConfig(self.nextInt())
        return machineConfig

    def createEnigmaMachineFromConfig(self,config):
        rotorCfgList=config["ROTORS"]
        rotorList=[]
        for r in rotorCfgList:
            rotorList.append(Rotor(r["ID"],Wiring(r["wiring"]),r["noch"]))
        plugboard=PlugBoard(Wiring(config["PLUGBOARD"]["wiring"]))
        reflector=Reflector(Wiring(config["REFLECTOR"]["wiring"]))
        mc=ModernEnigma(rotorList,reflector,plugboard)
        return mc

    def createReflectorConfig(self,id):
        reflectorConfig={"ID":id}
        reflectorConfig["wiring"]=self.seqToStr(self.getValidReflectorShuffledSequence())
        return reflectorConfig

    def getValidReflectorShuffledSequence(self):
        result=[]
        #init result before mapping
        for i in range(CharIndexMap.getRangeSize()):
            result.append("")

        seq= self.getShuffledSequence()
        while len(seq)>0:
            selectedPair=random.sample(seq,2)
            result[CharIndexMap.charToIndex(selectedPair[0])]=selectedPair[1]
            result[CharIndexMap.charToIndex(selectedPair[1])]=selectedPair[0]
            seq.remove(selectedPair[0])
            seq.remove(selectedPair[1])

        return result

    def createRotorConfig(self,id):
        rotorConfig={"ID":id}
        rotorConfig["wiring"]=self.seqToStr(self.getShuffledSequence())
        rotorConfig["noch"]=self.seqToStr(self.getSampleNotchSeq())
        return rotorConfig

    def createPlugboardConfig(self,id):
        plugboardConfig={"ID":id}
        plugboardConfig["wiring"]=self.seqToStr(CharIndexMap.getRange())
        return plugboardConfig

    def seqToStr(self,seq):
        result=""
        for item in seq:
            result+=item
        return result

    def nextInt(self,a=0,b=CharIndexMap.getRangeSize()):
        return random.randint(a,b)
    def getShuffledSequence(self,seq=CharIndexMap.getRange()):
        result=seq
        times=self.nextInt()
        for t in range(times):
            #to keep the original and copy it
            result= sorted(result, key=lambda k: random.random())
        return result
    def getSampleNotchSeq(self,seq=CharIndexMap.getRange()):
        result=random.sample(seq,self.nextInt(0,CharIndexMap.getRangeSize()//2))
        return result

mcCfg=EnigmaDynamicFactory().createMachineConfig("1")
mc1=EnigmaDynamicFactory().createEnigmaMachineFromConfig(mcCfg)
msg="AAAAAAA"
print("MSG:"+msg)
encMsg=""
for c in msg:
    encMsg+=mc1.processKeyPress(c)
print ("Encrpyted:"+encMsg)

mc2=EnigmaDynamicFactory().createEnigmaMachineFromConfig(mcCfg)
decMsg=""
for c in encMsg:
    decMsg+=mc2.processKeyPress(c)

print("Decrypted:"+decMsg)

<|endoftext|>