<commit_before># Copyright 2015 Red Hat, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import functools

from oslo_utils import uuidutils

from neutron.agent.l3 import agent as l3_agent
from neutron.agent.l3 import namespaces
from neutron.agent.linux import ip_lib
from neutron.agent.linux import utils
from neutron.tests.fullstack import base
from neutron.tests.fullstack.resources import environment

class TestLegacyL3Agent(base.BaseFullStackTestCase):

    def setUp(self):
        host_descriptions = [environment.HostDescription(l3_agent=True)]
        env = environment.Environment(
            environment.EnvironmentDescription(
                network_type='vlan', l2_pop=False),
            host_descriptions)
        super(TestLegacyL3Agent, self).setUp(env)

    def _get_namespace(self, router_id):
        return namespaces.build_ns_name(l3_agent.NS_PREFIX, router_id)

    def _assert_namespace_exists(self, ns_name):
        ip = ip_lib.IPWrapper(ns_name)
        utils.wait_until_true(lambda: ip.netns.exists(ns_name))

    def test_namespace_exists(self):
        tenant_id = uuidutils.generate_uuid()

        router = self.safe_client.create_router(tenant_id)
        network = self.safe_client.create_network(tenant_id)
        subnet = self.safe_client.create_subnet(
            tenant_id, network['id'], '20.0.0.0/24', gateway_ip='20.0.0.1')
        self.safe_client.add_router_interface(router['id'], subnet['id'])

        namespace = "%s@%s" % (
            self._get_namespace(router['id']),
            self.environment.hosts[0].l3_agent.get_namespace_suffix(), )
        self._assert_namespace_exists(namespace)

class TestHAL3Agent(base.BaseFullStackTestCase):

    def setUp(self):
        host_descriptions = [
            environment.HostDescription(l3_agent=True) for _ in range(2)]
        env = environment.Environment(
            environment.EnvironmentDescription(
                network_type='vxlan', l2_pop=True),
            host_descriptions)
        super(TestHAL3Agent, self).setUp(env)

    def _is_ha_router_active_on_one_agent(self, router_id):
        agents = self.client.list_l3_agent_hosting_routers(router_id)
        return (
            agents['agents'][0]['ha_state'] != agents['agents'][1]['ha_state'])

    def test_ha_router(self):
        # TODO(amuller): Test external connectivity before and after a
        # failover, see: https://review.openstack.org/#/c/196393/

        tenant_id = uuidutils.generate_uuid()
        router = self.safe_client.create_router(tenant_id, ha=True)
        agents = self.client.list_l3_agent_hosting_routers(router['id'])
        self.assertEqual(2, len(agents['agents']),
                         'HA router must be scheduled to both nodes')

        utils.wait_until_true(
            functools.partial(
                self._is_ha_router_active_on_one_agent,
                router['id']),
            timeout=90)
<commit_msg>Skip fullstack L3 HA test<commit_after># Copyright 2015 Red Hat, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import functools

from oslo_utils import uuidutils
import testtools

from neutron.agent.l3 import agent as l3_agent
from neutron.agent.l3 import namespaces
from neutron.agent.linux import ip_lib
from neutron.agent.linux import utils
from neutron.tests.fullstack import base
from neutron.tests.fullstack.resources import environment

class TestLegacyL3Agent(base.BaseFullStackTestCase):

    def setUp(self):
        host_descriptions = [environment.HostDescription(l3_agent=True)]
        env = environment.Environment(
            environment.EnvironmentDescription(
                network_type='vlan', l2_pop=False),
            host_descriptions)
        super(TestLegacyL3Agent, self).setUp(env)

    def _get_namespace(self, router_id):
        return namespaces.build_ns_name(l3_agent.NS_PREFIX, router_id)

    def _assert_namespace_exists(self, ns_name):
        ip = ip_lib.IPWrapper(ns_name)
        utils.wait_until_true(lambda: ip.netns.exists(ns_name))

    def test_namespace_exists(self):
        tenant_id = uuidutils.generate_uuid()

        router = self.safe_client.create_router(tenant_id)
        network = self.safe_client.create_network(tenant_id)
        subnet = self.safe_client.create_subnet(
            tenant_id, network['id'], '20.0.0.0/24', gateway_ip='20.0.0.1')
        self.safe_client.add_router_interface(router['id'], subnet['id'])

        namespace = "%s@%s" % (
            self._get_namespace(router['id']),
            self.environment.hosts[0].l3_agent.get_namespace_suffix(), )
        self._assert_namespace_exists(namespace)

class TestHAL3Agent(base.BaseFullStackTestCase):

    def setUp(self):
        host_descriptions = [
            environment.HostDescription(l3_agent=True) for _ in range(2)]
        env = environment.Environment(
            environment.EnvironmentDescription(
                network_type='vxlan', l2_pop=True),
            host_descriptions)
        super(TestHAL3Agent, self).setUp(env)

    def _is_ha_router_active_on_one_agent(self, router_id):
        agents = self.client.list_l3_agent_hosting_routers(router_id)
        return (
            agents['agents'][0]['ha_state'] != agents['agents'][1]['ha_state'])

    @testtools.skip('bug/1550886')
    def test_ha_router(self):
        # TODO(amuller): Test external connectivity before and after a
        # failover, see: https://review.openstack.org/#/c/196393/

        tenant_id = uuidutils.generate_uuid()
        router = self.safe_client.create_router(tenant_id, ha=True)
        agents = self.client.list_l3_agent_hosting_routers(router['id'])
        self.assertEqual(2, len(agents['agents']),
                         'HA router must be scheduled to both nodes')

        utils.wait_until_true(
            functools.partial(
                self._is_ha_router_active_on_one_agent,
                router['id']),
            timeout=90)
<|endoftext|>


<commit_before>#  File:       grade_school.py
#  Purpose:    Write a small archiving program that stores students' names along with the grade that they are in.
#  Programmer: Amal Shehu
#  Course:     Exercism
#  Date:       Monday 12th September 2016, 11:00 PM

from collections import defaultdict

class School(object):
    """docstring for School."""
    def __init__(self, name):
        self.name = name
        self.data = defaultdict(set)

    def add(self, student, grade):
        self.data[grade].add(student)
    def grade(self, stage):
        return self.data[stage]

<commit_msg>commit<commit_after>#  File:       grade_school.py
#  Purpose:    Write a small archiving program that stores students' names along with the grade that they are in.
#  Programmer: Amal Shehu
#  Course:     Exercism
#  Date:       Monday 12th September 2016, 11:00 PM

from collections import defaultdict

class School(object):
    """docstring for School."""
    def __init__(self, name):
        self.name = name
        self.data = defaultdict(set)

    def add(self, student, grade):
        self.data[grade].add(student)
    def grade(self, stage):
        return self.data[stage]

    def sort(self):
        return sorted((grade, tuple(sorted(students)))
                      for grade, students in self.data.items())
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (c) 2013 dotCloud, Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
A Docker Hypervisor which allows running Linux Containers instead of VMs.
"""

import os
import random
import socket
import time

from oslo.config import cfg

from nova.compute import power_state
from nova.compute import task_states
from nova import exception
from nova.image import glance
from nova.openstack.common.gettextutils import _
from nova.openstack.common import log
from nova import utils
import nova.virt.docker.client
from nova.virt.docker import hostinfo
from nova.virt import driver

docker_opts = [
    cfg.StrOpt('docker_registry_default_port',
           default=5042,
           help='Default TCP port to find the docker-registry container')
    ]

CONF = cfg.CONF
CONF.register_opts(docker_opts)
CONF.import_opt('my_ip', 'nova.netconf')

LOG = log.getLogger(__name__)

class DockerDriver(driver.ComputeDriver):
    """Docker hypervisor driver."""

    capabilities = {
        'has_imagecache': True,
        'supports_recreate': True,
    }

    def __init__(self, virtapi):
        super(DockerDriver, self).__init__(virtapi)
        self._docker = None

    @property
    def docker(self):
        if self._docker is None:
            self._docker = nova.virt.docker.client.DockerHTTPClient()
        return self._docker

    def init_host(self, host):
        if self.is_daemon_running() is False:
            raise exception.NovaException(_("Docker daemon is not running or "
                "is not reachable (check the rights on /var/run/docker.sock)"))

    def is_daemon_running(self):
        try:
            self.docker.list_containers()
            return True
        except socket.error:
            return False

    def list_instances(self, inspect=False):
        res = []
        for container in self.docker.list_containers():
            info = self.docker.inspect_container(container['id'])
            if inspect:
                res.append(info)
            else:
                res.append(info['Config'].get('Hostname'))
        return res

    def plug_vifs(self, instance, network_info):
        """Plug VIFs into networks."""
        pass

    def unplug_vifs(self, instance, network_info):
        """Unplug VIFs from networks."""
        pass

    def find_container_by_name(self, name):
        for info in self.list_instances(inspect=True):
            if info['Config'].get('Hostname') == name:
                return info
        return {}

    def get_info(self, instance):
        container = self.find_container_by_name(instance['name'])
        if not container:
            raise exception.InstanceNotFound(instance_id=instance['name'])
        running = container['State'].get('Running')
        info = {
            'max_mem': 0,
            'mem': 0,
            'num_cpu': 1,
            'cpu_time': 0
        }
        info['state'] = power_state.RUNNING if running \
            else power_state.SHUTDOWN
        return info

    def get_host_stats(self, refresh=False):
        hostname = socket.gethostname()
        memory = hostinfo.get_memory_usage()
        disk = hostinfo.get_disk_usage()
        stats = self.get_available_resource(hostname)
        stats['hypervisor_hostname'] = hostname
        stats['host_hostname'] = hostname
        stats['host_name_label'] = hostname
        return stats

    def get_available_resource(self, nodename):
        memory = hostinfo.get_memory_usage()
        disk = hostinfo.get_disk_usage()
        stats = {
            'vcpus': 1,
            'vcpus_used': 0,
            'memory_mb': memory['total'] / (1024 ** 2),
            'memory_mb_used': memory['used'] / (1024 ** 2),
            'local_gb': disk['total'] / (1024 ** 3),
            'local_gb_used': disk['used'] / (1024 ** 3),
            'disk_available_least': disk['available'] / (1024 ** 3),
            'hypervisor_type': 'docker',
            'hypervisor_version': '1.0',
            'hypervisor_hostname': nodename,
            'cpu_info': '?'
        }
        return stats

    def _find_cgroup_devices_path(self):
        for ln in open('/proc/mounts'):
            if ln.startswith('cgroup ') and 'devices' in ln:
                return ln.split(' ')[1]

    def _find_container_pid(self, container_id):
        cgroup_path = self._find_cgroup_devices_path()
        lxc_path = os.path.join(cgroup_path, 'lxc')
        tasks_path = os.path.join(lxc_path, container_id, 'tasks')
        n = 0
        while True:
            if n > 20:
                return
            try:
                with open(tasks_path) as f:
                    pids = f.readlines()
                    if pids:
                        return int(pids[0].strip())
            except IOError:
                pass
            time.sleep(0.5)
            n += 1

    def _find_fixed_ip(self, subnets):
        for subnet in subnets:
            for ip in subnet['ips']:
                if ip['type'] == 'fixed' and ip['address']:
                    return ip['address']

    def _setup_network(self, instance, network_info):
        if not network_info:
            return
        container_id = self.find_container_by_name(instance['name']).get('id')
        if not container_id:
            return
        network_info = network_info[0]['network']
        netns_path = '/var/run/netns'
        if not os.path.exists(netns_path):
            utils.execute(
                'mkdir', '-p', netns_path, run_as_root=True)
        nspid = self._find_container_pid(container_id)
        if not nspid:
            raise RuntimeError(_(
                'Cannot find any PID under '
                'container "{0}"'.format(container_id)))
        netns_path = os.path.join(netns_path, container_id)
        utils.execute(
            'ln', '-sf', '/proc/{0}/ns/net'.format(nspid),
            '/var/run/netns/{0}'.format(container_id),
            run_as_root=True)
        rand = random.randint(0, 100000)
        if_local_name = 'pvnetl{0}'.format(rand)
        if_remote_name = 'pvnetr{0}'.format(rand)
        bridge = network_info['bridge']
        ip = self._find_fixed_ip(network_info['subnets'])
        if not ip:
            raise RuntimeError(_('Cannot set fixed ip'))
        utils.execute(
            'ip', 'link', 'add', 'name', if_local_name, 'type',
            'veth', 'peer', 'name', if_remote_name,
            run_as_root=True)
        utils.execute(
            'brctl', 'addif', bridge, if_local_name,
            run_as_root=True)
        utils.execute(
            'ip', 'link', 'set', if_local_name, 'up',
            run_as_root=True)
        utils.execute(
            'ip', 'link', 'set', if_remote_name, 'netns', nspid,
            run_as_root=True)
        utils.execute(
            'ip', 'netns', 'exec', container_id, 'ifconfig',
            if_remote_name, ip,
            run_as_root=True)

    def _get_memory_limit_bytes(self, instance):
        for metadata in instance.get('system_metadata', []):
            if metadata['deleted']:
                continue
            if metadata['key'] == 'instance_type_memory_mb':
                return int(metadata['value']) * 1024 * 1024
        return 0

    def _get_image_name(self, context, instance, image):
        fmt = image['container_format']
        if fmt != 'docker':
            raise exception.InstanceDeployFailure(_(
                'Image container format not supported ({0})'.format(fmt)),
                instance_id=instance['name'])
        registry_port = self._get_registry_port()
        return '{0}:{1}/{2}'.format(CONF.my_ip,
                                    registry_port,
                                    image['name'])

    def _get_default_cmd(self, image_name):
        default_cmd = ['sh']
        info = self.docker.inspect_image(image_name)
        if not info:
            return default_cmd
        if not info['container_config']['Cmd']:
            return default_cmd

    def spawn(self, context, instance, image_meta, injected_files,
              admin_password, network_info=None, block_device_info=None):
        image_name = self._get_image_name(context, instance, image_meta)
        args = {
            'Hostname': instance['name'],
            'Image': image_name,
            'Memory': self._get_memory_limit_bytes(instance)
        }
        default_cmd = self._get_default_cmd(image_name)
        if default_cmd:
            args['Cmd'] = default_cmd
        container_id = self.docker.create_container(args)
        if not container_id:
            LOG.info(_('Image name "{0}" does not exist, '
                'fetching it...'.format(image_name)))
            res = self.docker.pull_repository(image_name)
            if res is False:
                raise exception.InstanceDeployFailure(
                    _('Cannot pull missing image'),
                    instance_id=instance['name'])
            container_id = self.docker.create_container(args)
            if not container_id:
                raise exception.InstanceDeployFailure(
                    _('Cannot create container'),
                    instance_id=instance['name'])
        self.docker.start_container(container_id)
        try:
            self._setup_network(instance, network_info)
        except Exception as e:
            raise exception.InstanceDeployFailure(
                _('Cannot setup network: {0}'.format(e)),
                instance_id=instance['name'])

    def destroy(self, instance, network_info, block_device_info=None,
                destroy_disks=True):
        container_id = self.find_container_by_name(instance['name']).get('id')
        if not container_id:
            return
        self.docker.stop_container(container_id)
        self.docker.destroy_container(container_id)

    def reboot(self, context, instance, network_info, reboot_type,
               block_device_info=None, bad_volumes_callback=None):
        container_id = self.find_container_by_name(instance['name']).get('id')
        if not container_id:
            return
        self.docker.stop_container(container_id)
        self.docker.start_container(container_id)

    def power_on(self, context, instance, network_info, block_device_info):
        container_id = self.find_container_by_name(instance['name']).get('id')
        if not container_id:
            return
        self.docker.start_container(container_id)

    def power_off(self, instance):
        container_id = self.find_container_by_name(instance['name']).get('id')
        if not container_id:
            return
        self.docker.stop_container(container_id)

    def get_console_output(self, instance):
        container_id = self.find_container_by_name(instance['name']).get('id')
        if not container_id:
            return
        return self.docker.get_container_logs(container_id)

    def _get_registry_port(self):
        default_port = CONF.docker_registry_default_port
        registry = None
        for container in self.docker.list_containers(_all=False):
            container = self.docker.inspect_container(container['id'])
            if 'docker-registry' in container['Path']:
                registry = container
                break
        if not registry:
            return default_port
        # NOTE(samalba): The registry service always binds on port 5000 in the
        # container
        try:
            return container['NetworkSettings']['PortMapping']['Tcp']['5000']
        except (KeyError, TypeError):
            # NOTE(samalba): Falling back to a default port allows more
            # flexibility (run docker-registry outside a container)
            return default_port

    def snapshot(self, context, instance, image_href, update_task_state):
        container_id = self.find_container_by_name(instance['name']).get('id')
        if not container_id:
            raise exception.InstanceNotRunning(instance_id=instance['uuid'])
        update_task_state(task_state=task_states.IMAGE_PENDING_UPLOAD)
        (image_service, image_id) = glance.get_remote_image_service(
            context, image_href)
        image = image_service.show(context, image_id)
        registry_port = self._get_registry_port()
        name = image['name']
        default_tag = (':' not in name)
        name = '{0}:{1}/{2}'.format(CONF.my_ip,
                                    registry_port,
                                    name)
        commit_name = name if not default_tag else name + ':latest'
        self.docker.commit_container(container_id, commit_name)
        update_task_state(task_state=task_states.IMAGE_UPLOADING,
                          expected_state=task_states.IMAGE_PENDING_UPLOAD)
        headers = {'X-Meta-Glance-Image-Id': image_href}
        self.docker.push_repository(name, headers=headers)
<commit_msg>Added support for supported_instances arg<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (c) 2013 dotCloud, Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
A Docker Hypervisor which allows running Linux Containers instead of VMs.
"""

import os
import random
import socket
import time

from oslo.config import cfg

from nova.compute import power_state
from nova.compute import task_states
from nova import exception
from nova.image import glance
from nova.openstack.common.gettextutils import _
from nova.openstack.common import jsonutils
from nova.openstack.common import log
from nova import utils
import nova.virt.docker.client
from nova.virt.docker import hostinfo
from nova.virt import driver

docker_opts = [
    cfg.StrOpt('docker_registry_default_port',
           default=5042,
           help='Default TCP port to find the docker-registry container')
    ]

CONF = cfg.CONF
CONF.register_opts(docker_opts)
CONF.import_opt('my_ip', 'nova.netconf')

LOG = log.getLogger(__name__)

class DockerDriver(driver.ComputeDriver):
    """Docker hypervisor driver."""

    capabilities = {
        'has_imagecache': True,
        'supports_recreate': True,
    }

    def __init__(self, virtapi):
        super(DockerDriver, self).__init__(virtapi)
        self._docker = None

    @property
    def docker(self):
        if self._docker is None:
            self._docker = nova.virt.docker.client.DockerHTTPClient()
        return self._docker

    def init_host(self, host):
        if self.is_daemon_running() is False:
            raise exception.NovaException(_("Docker daemon is not running or "
                "is not reachable (check the rights on /var/run/docker.sock)"))

    def is_daemon_running(self):
        try:
            self.docker.list_containers()
            return True
        except socket.error:
            return False

    def list_instances(self, inspect=False):
        res = []
        for container in self.docker.list_containers():
            info = self.docker.inspect_container(container['id'])
            if inspect:
                res.append(info)
            else:
                res.append(info['Config'].get('Hostname'))
        return res

    def plug_vifs(self, instance, network_info):
        """Plug VIFs into networks."""
        pass

    def unplug_vifs(self, instance, network_info):
        """Unplug VIFs from networks."""
        pass

    def find_container_by_name(self, name):
        for info in self.list_instances(inspect=True):
            if info['Config'].get('Hostname') == name:
                return info
        return {}

    def get_info(self, instance):
        container = self.find_container_by_name(instance['name'])
        if not container:
            raise exception.InstanceNotFound(instance_id=instance['name'])
        running = container['State'].get('Running')
        info = {
            'max_mem': 0,
            'mem': 0,
            'num_cpu': 1,
            'cpu_time': 0
        }
        info['state'] = power_state.RUNNING if running \
            else power_state.SHUTDOWN
        return info

    def get_host_stats(self, refresh=False):
        hostname = socket.gethostname()
        memory = hostinfo.get_memory_usage()
        disk = hostinfo.get_disk_usage()
        stats = self.get_available_resource(hostname)
        stats['hypervisor_hostname'] = hostname
        stats['host_hostname'] = hostname
        stats['host_name_label'] = hostname
        return stats

    def get_available_resource(self, nodename):
        memory = hostinfo.get_memory_usage()
        disk = hostinfo.get_disk_usage()
        stats = {
            'vcpus': 1,
            'vcpus_used': 0,
            'memory_mb': memory['total'] / (1024 ** 2),
            'memory_mb_used': memory['used'] / (1024 ** 2),
            'local_gb': disk['total'] / (1024 ** 3),
            'local_gb_used': disk['used'] / (1024 ** 3),
            'disk_available_least': disk['available'] / (1024 ** 3),
            'hypervisor_type': 'docker',
            'hypervisor_version': '1.0',
            'hypervisor_hostname': nodename,
            'cpu_info': '?',
            'supported_instances': jsonutils([
                    ('i686', 'docker', 'lxc'),
                    ('x86_64', 'docker', 'lxc')
                ])
        }
        return stats

    def _find_cgroup_devices_path(self):
        for ln in open('/proc/mounts'):
            if ln.startswith('cgroup ') and 'devices' in ln:
                return ln.split(' ')[1]

    def _find_container_pid(self, container_id):
        cgroup_path = self._find_cgroup_devices_path()
        lxc_path = os.path.join(cgroup_path, 'lxc')
        tasks_path = os.path.join(lxc_path, container_id, 'tasks')
        n = 0
        while True:
            if n > 20:
                return
            try:
                with open(tasks_path) as f:
                    pids = f.readlines()
                    if pids:
                        return int(pids[0].strip())
            except IOError:
                pass
            time.sleep(0.5)
            n += 1

    def _find_fixed_ip(self, subnets):
        for subnet in subnets:
            for ip in subnet['ips']:
                if ip['type'] == 'fixed' and ip['address']:
                    return ip['address']

    def _setup_network(self, instance, network_info):
        if not network_info:
            return
        container_id = self.find_container_by_name(instance['name']).get('id')
        if not container_id:
            return
        network_info = network_info[0]['network']
        netns_path = '/var/run/netns'
        if not os.path.exists(netns_path):
            utils.execute(
                'mkdir', '-p', netns_path, run_as_root=True)
        nspid = self._find_container_pid(container_id)
        if not nspid:
            raise RuntimeError(_(
                'Cannot find any PID under '
                'container "{0}"'.format(container_id)))
        netns_path = os.path.join(netns_path, container_id)
        utils.execute(
            'ln', '-sf', '/proc/{0}/ns/net'.format(nspid),
            '/var/run/netns/{0}'.format(container_id),
            run_as_root=True)
        rand = random.randint(0, 100000)
        if_local_name = 'pvnetl{0}'.format(rand)
        if_remote_name = 'pvnetr{0}'.format(rand)
        bridge = network_info['bridge']
        ip = self._find_fixed_ip(network_info['subnets'])
        if not ip:
            raise RuntimeError(_('Cannot set fixed ip'))
        utils.execute(
            'ip', 'link', 'add', 'name', if_local_name, 'type',
            'veth', 'peer', 'name', if_remote_name,
            run_as_root=True)
        utils.execute(
            'brctl', 'addif', bridge, if_local_name,
            run_as_root=True)
        utils.execute(
            'ip', 'link', 'set', if_local_name, 'up',
            run_as_root=True)
        utils.execute(
            'ip', 'link', 'set', if_remote_name, 'netns', nspid,
            run_as_root=True)
        utils.execute(
            'ip', 'netns', 'exec', container_id, 'ifconfig',
            if_remote_name, ip,
            run_as_root=True)

    def _get_memory_limit_bytes(self, instance):
        for metadata in instance.get('system_metadata', []):
            if metadata['deleted']:
                continue
            if metadata['key'] == 'instance_type_memory_mb':
                return int(metadata['value']) * 1024 * 1024
        return 0

    def _get_image_name(self, context, instance, image):
        fmt = image['container_format']
        if fmt != 'docker':
            raise exception.InstanceDeployFailure(_(
                'Image container format not supported ({0})'.format(fmt)),
                instance_id=instance['name'])
        registry_port = self._get_registry_port()
        return '{0}:{1}/{2}'.format(CONF.my_ip,
                                    registry_port,
                                    image['name'])

    def _get_default_cmd(self, image_name):
        default_cmd = ['sh']
        info = self.docker.inspect_image(image_name)
        if not info:
            return default_cmd
        if not info['container_config']['Cmd']:
            return default_cmd

    def spawn(self, context, instance, image_meta, injected_files,
              admin_password, network_info=None, block_device_info=None):
        image_name = self._get_image_name(context, instance, image_meta)
        args = {
            'Hostname': instance['name'],
            'Image': image_name,
            'Memory': self._get_memory_limit_bytes(instance)
        }
        default_cmd = self._get_default_cmd(image_name)
        if default_cmd:
            args['Cmd'] = default_cmd
        container_id = self.docker.create_container(args)
        if not container_id:
            LOG.info(_('Image name "{0}" does not exist, '
                'fetching it...'.format(image_name)))
            res = self.docker.pull_repository(image_name)
            if res is False:
                raise exception.InstanceDeployFailure(
                    _('Cannot pull missing image'),
                    instance_id=instance['name'])
            container_id = self.docker.create_container(args)
            if not container_id:
                raise exception.InstanceDeployFailure(
                    _('Cannot create container'),
                    instance_id=instance['name'])
        self.docker.start_container(container_id)
        try:
            self._setup_network(instance, network_info)
        except Exception as e:
            raise exception.InstanceDeployFailure(
                _('Cannot setup network: {0}'.format(e)),
                instance_id=instance['name'])

    def destroy(self, instance, network_info, block_device_info=None,
                destroy_disks=True):
        container_id = self.find_container_by_name(instance['name']).get('id')
        if not container_id:
            return
        self.docker.stop_container(container_id)
        self.docker.destroy_container(container_id)

    def reboot(self, context, instance, network_info, reboot_type,
               block_device_info=None, bad_volumes_callback=None):
        container_id = self.find_container_by_name(instance['name']).get('id')
        if not container_id:
            return
        self.docker.stop_container(container_id)
        self.docker.start_container(container_id)

    def power_on(self, context, instance, network_info, block_device_info):
        container_id = self.find_container_by_name(instance['name']).get('id')
        if not container_id:
            return
        self.docker.start_container(container_id)

    def power_off(self, instance):
        container_id = self.find_container_by_name(instance['name']).get('id')
        if not container_id:
            return
        self.docker.stop_container(container_id)

    def get_console_output(self, instance):
        container_id = self.find_container_by_name(instance['name']).get('id')
        if not container_id:
            return
        return self.docker.get_container_logs(container_id)

    def _get_registry_port(self):
        default_port = CONF.docker_registry_default_port
        registry = None
        for container in self.docker.list_containers(_all=False):
            container = self.docker.inspect_container(container['id'])
            if 'docker-registry' in container['Path']:
                registry = container
                break
        if not registry:
            return default_port
        # NOTE(samalba): The registry service always binds on port 5000 in the
        # container
        try:
            return container['NetworkSettings']['PortMapping']['Tcp']['5000']
        except (KeyError, TypeError):
            # NOTE(samalba): Falling back to a default port allows more
            # flexibility (run docker-registry outside a container)
            return default_port

    def snapshot(self, context, instance, image_href, update_task_state):
        container_id = self.find_container_by_name(instance['name']).get('id')
        if not container_id:
            raise exception.InstanceNotRunning(instance_id=instance['uuid'])
        update_task_state(task_state=task_states.IMAGE_PENDING_UPLOAD)
        (image_service, image_id) = glance.get_remote_image_service(
            context, image_href)
        image = image_service.show(context, image_id)
        registry_port = self._get_registry_port()
        name = image['name']
        default_tag = (':' not in name)
        name = '{0}:{1}/{2}'.format(CONF.my_ip,
                                    registry_port,
                                    name)
        commit_name = name if not default_tag else name + ':latest'
        self.docker.commit_container(container_id, commit_name)
        update_task_state(task_state=task_states.IMAGE_UPLOADING,
                          expected_state=task_states.IMAGE_PENDING_UPLOAD)
        headers = {'X-Meta-Glance-Image-Id': image_href}
        self.docker.push_repository(name, headers=headers)
<|endoftext|>


<commit_before># Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_log import log as logging
from oslo_log import versionutils

LOG = logging.getLogger(__name__)

class CloudController(object):
    def __init__(self):
        versionutils.report_deprecated_feature(
            LOG,
            'The in tree EC2 API has been removed in Mitaka. '
            'Please remove entries from api-paste.ini and use '
            'the OpenStack ec2-api project '
            'http://opendev.org/openstack/ec2-api/'
        )
<commit_msg>ec2: Remove ec2.CloudController<commit_after><|endoftext|>


<commit_before>from bluesky.callbacks.broker import LiveTiffExporter

class SubtractedTiffExporter(LiveTiffExporter):
    "Intercept images before saving and subtract dark image"

    def start(self, doc):
        # The metadata refers to the scan uid of the dark scan.
        if 'dark_frame' not in doc:
            raise ValueError("No dark_frame was recorded.")
        uid = doc['dark_frame']
        dark_header = db[uid]
        self.dark_img, = get_images(db[uid], 'pe1_image')
        super().start(doc)

    def event(self, doc):
        img = doc['data'][self.field]
        subtracted_img = img - self.dark_img
        doc['data'][self.field] = subtracted_img
        super().event(doc)

template = "/home/xf28id1/xpdUser/tiff_base/LaB6_EPICS/{start.sa_name}_{start.scan_id}_step{event.seq_num}.tif"
exporter = SubtractedTiffExporter('pe1_image', template)
<commit_msg>MNT: switch from get_images -> get_data<commit_after>from bluesky.callbacks.broker import LiveTiffExporter

class SubtractedTiffExporter(LiveTiffExporter):
    "Intercept images before saving and subtract dark image"

    def start(self, doc):
        # The metadata refers to the scan uid of the dark scan.
        if 'dark_frame' not in doc:
            raise ValueError("No dark_frame was recorded.")
        uid = doc['dark_frame']
        h = db[uid]
        self.dark_img, = h.get_data('pe1_image')
        super().start(doc)

    def event(self, doc):
        img = doc['data'][self.field]
        subtracted_img = img - self.dark_img
        doc['data'][self.field] = subtracted_img
        super().event(doc)

template = "/home/xf28id1/xpdUser/tiff_base/LaB6_EPICS/{start.sa_name}_{start.scan_id}_step{event.seq_num}.tif"
exporter = SubtractedTiffExporter('pe1_image', template)
<|endoftext|>


<commit_before>import numpy as np
import pysam
import re
from collections import defaultdict
from Bio import SeqIO
import time

class State:
	on_on = 0
	on_off = 1
	off_on = 2
	off_off = 3

def make_freq_csv(freq_dict, read_max, ref_max, out):
	#print(freq_dict)
	freq_cnt = np.zeros(shape=(ref_max + 1, read_max + 1), dtype=float)
	total = 0
	for key, val in freq_dict.items():
		ref_len = key[0]
		read_len = key[1]
		letter = key[2]
		freq_cnt[ref_len][read_len] += val
		total += val
	print(total)
	freq_cnt = np.divide(freq_cnt, float(total))
	with open(out,'w') as f:
		np.savetxt(f, freq_cnt.astype(float), fmt='%2.5f', delimiter=',')

def make_test_csv(regions_dict, read_max, out):
	size = len(regions_dict)
	freq_cnt = np.zeros(shape=(size, read_max + 4), dtype=int)
	cnt = 0
	for key, val in regions_dict.items():
		if(cnt >= size):
			break
		length = key[1] - key[0]
		for i in val:
			freq_cnt[cnt][i] += 1
		freq_cnt[cnt][read_max + 1] = length
		freq_cnt[cnt][read_max + 2] = key[0]
		freq_cnt[cnt][read_max + 3] = key[1]
		cnt += 1
	with open(out,'w') as f:
		np.savetxt(f, freq_cnt.astype(int), fmt='%i', delimiter=',')

def genome_preprocessing(reference_file):

	with open(reference_file) as f:
		content = f.readlines()
	content = [x for x in content if x[0] != '>']
	content = [x.strip() for x in content]
	genome=''.join(content)
	return genome

def get_cigar_string(cigar_string):

	cigar_components = re.findall(r'(\d+)([A-Z]{1})', cigar_string)
	cigar = ''
	for comp in cigar_components:
		number = comp[0]
		letter = comp[1]
		for i in range(int(number)):
			cigar+=letter
	return cigar

def readerrors(ref, reads):

	freq_dict = {}
	regions_dict = defaultdict(list)

	#track max homopolymer size
	ref_max = 0
	read_max = 0

	fasta_sequences = SeqIO.parse(open(ref),'fasta')

	indent = 0 # globalni pomak po referenci

	for seq in fasta_sequences:

		seq_id, sequence = seq.id, str(seq.seq) + 'X'
		content = pysam.AlignmentFile(reads)

		for r in content.fetch(until_eof=True):

			#ako se read nije namapirao na referencu
			if r.cigarstring is None:
	 			continue

	 		# ako se read nije namapirao na ovaj kromosom
			if(content.getrname(r.reference_id) != seq_id):
				continue

	 		cigar = get_cigar_string(r.cigarstring)

			#set up necessary variables
			#current pointers, they show us which positions we are looking at
			sequence_pointer = r.pos
			reference_pointer = r.pos + indent
			read_pointer = 0
			cigar_pointer = 0

			#length of homopolymers
			read_homopolymer = 0
			ref_homopolymer = 0

			#coordinates of homopolymers
			ref_begin = 0
			ref_end = 0

			#homopolymer letters. if they exist, they should never be different (except if one of them is empty)
			read_letter = ''
			ref_letter = ''

			#read sequence
			read = r.seq + 'X'

			#starting state
			state = State.off_off

			while(cigar_pointer < len(cigar)):

				if(cigar[cigar_pointer] == 'M'):

				#not a read nor a reference has a detected homopolymer
					if(state == State.off_off):
						if(read[read_pointer] == read[read_pointer + 1]):
							read_letter = read[read_pointer]
							read_homopolymer += 1
							state = State.on_off

							if(sequence[sequence_pointer] == read_letter):
								ref_homopolymer += 1
								ref_letter = read_letter
								state = State.on_on
								ref_begin = reference_pointer

						elif(sequence[sequence_pointer] == sequence[sequence_pointer + 1]):
							ref_homopolymer += 1
							ref_letter = sequence[sequence_pointer]
							state = State.off_on
							ref_begin = reference_pointer

							if(read[read_pointer] == ref_letter):
								read_letter = ref_letter
								read_homopolymer += 1
								state = State.on_on

						reference_pointer += 1
						read_pointer += 1
						cigar_pointer += 1
						sequence_pointer += 1

					# we have a homopolymer in the read, but not in the reference(or ref homopolymer ended)
					elif(state == State.on_off):

						#if homopolymer in read continues
						if(read[read_pointer] == read_letter):
							read_homopolymer += 1
							#if we didn't found a homopolymer in the reference by now, check
							if(ref_homopolymer == 0 and sequence[sequence_pointer] == read_letter):
								ref_letter = read_letter
								ref_homopolymer += 1
								state = State.on_on
								ref_begin = reference_pointer

							reference_pointer += 1
							read_pointer += 1
							cigar_pointer += 1
							sequence_pointer += 1

						else:
							#here we don't update pointers so we can check them in off_off state
							state = State.off_off

							data = (ref_homopolymer, read_homopolymer, read_letter)
							if(freq_dict.has_key(data)):
								freq_dict[data] += 1
							else:
								freq_dict[data] = 1

							ref_coordinates = (ref_begin, ref_end)
							regions_dict[ref_coordinates].append(read_homopolymer)

							ref_max = max(ref_max, ref_homopolymer)
							read_max = max(read_max, read_homopolymer)

							ref_begin = 0
							ref_end = 0
							read_homopolymer = 0
							ref_homopolymer = 0
							read_letter = ''
							ref_letter = ''

					# we have a homopolymer in the reference, but not in the read (or read homopolymer ended)
					elif(state == State.off_on):

						#if homopolymer in reference continues
						if(sequence[sequence_pointer] == ref_letter):
							ref_homopolymer += 1
							if(read_homopolymer == 0 and read[read_pointer] == ref_letter):
								read_letter = ref_letter
								read_homopolymer += 1
								state = State.on_on

							reference_pointer += 1
							read_pointer += 1
							cigar_pointer += 1
							sequence_pointer += 1

						else:
							#here we don't update pointers so we can check them in off_off state
							state = State.off_off
							ref_end = reference_pointer

							data = (ref_homopolymer, read_homopolymer, ref_letter)
							if(freq_dict.has_key(data)):
								freq_dict[data] += 1
							else:
								freq_dict[data] = 1

							ref_coordinates = (ref_begin, ref_end)
							regions_dict[ref_coordinates].append(read_homopolymer)

							ref_max = max(ref_max, ref_homopolymer)
							read_max = max(read_max, read_homopolymer)

							ref_begin = 0
							ref_end = 0
							read_homopolymer = 0
							ref_homopolymer = 0
							read_letter = ''
							ref_letter = ''

					elif(state == State.on_on):
						#print("ON_ON")
						# if read homopolymer continues
						if(read[read_pointer] == read_letter and sequence[sequence_pointer] == ref_letter):
							read_homopolymer += 1
							ref_homopolymer += 1

							reference_pointer += 1
							read_pointer += 1
							cigar_pointer += 1
							sequence_pointer += 1

						elif(read[read_pointer] == read_letter):
							read_homopolymer += 1
							state = State.on_off

							ref_end = reference_pointer

							reference_pointer += 1
							read_pointer += 1
							cigar_pointer += 1
							sequence_pointer += 1

						elif(sequence[sequence_pointer] == ref_letter):
							ref_homopolymer += 1
							state = State.off_on

							reference_pointer += 1
							read_pointer += 1
							cigar_pointer += 1
							sequence_pointer += 1

						else:
							state = State.off_off

							ref_end = reference_pointer

							data = (ref_homopolymer, read_homopolymer, read_letter)
							if(freq_dict.has_key(data)):
								freq_dict[data] += 1
							else:
								freq_dict[data] = 1

							ref_coordinates = (ref_begin, ref_end)
							regions_dict[ref_coordinates].append(read_homopolymer)

							ref_max = max(ref_max, ref_homopolymer)
							read_max = max(read_max, read_homopolymer)

							ref_begin = 0
							ref_end = 0

							read_homopolymer = 0
							ref_homopolymer = 0
							read_letter = ''
							ref_letter = ''

				elif(cigar[cigar_pointer] == 'I'):

					if(state == State.off_off):
						#print("OFF_OFF")
						if(read[read_pointer] == read[read_pointer + 1]):
							read_letter = read[read_pointer]
							read_homopolymer += 1
							state = State.on_off

						read_pointer += 1
						cigar_pointer += 1
						# we have a homopolymer in the read, but not in the reference(or ref homopolymer ended)
					elif(state == State.on_off):
						#print("ON_OFF")
						if(read[read_pointer] == read_letter):
							read_homopolymer += 1
							read_pointer += 1
							cigar_pointer += 1
						else:
							state = State.off_off

							data = (ref_homopolymer, read_homopolymer, read_letter)
							if(freq_dict.has_key(data)):
								freq_dict[data] += 1
							else:
								freq_dict[data] = 1

							ref_coordinates = (ref_begin, ref_end)
							regions_dict[ref_coordinates].append(read_homopolymer)

							ref_max = max(ref_max, ref_homopolymer)
							read_max = max(read_max, read_homopolymer)

							ref_begin = 0
							ref_end = 0

							read_homopolymer = 0
							ref_homopolymer = 0
							read_letter = ''
							ref_letter = ''

					elif(state == State.off_on):
						#print("OFF_ON")
						if(read_homopolymer == 0 and read[read_pointer] == ref_letter):
							read_letter = ref_letter
							read_homopolymer += 1
							state = State.on_on
						read_pointer += 1
						cigar_pointer += 1

					elif(state == State.on_on):
						#print("ON_ON")
						if(read[read_pointer] == read_letter):
							read_homopolymer += 1
						else:
							state = State.off_on
						read_pointer += 1
						cigar_pointer += 1

				elif(cigar[cigar_pointer] == 'D'):
					if(state == State.off_off):
						#print("OFF_OFF")
						if(sequence[sequence_pointer] == sequence[sequence_pointer + 1]):
							ref_homopolymer += 1
							ref_letter = sequence[sequence_pointer]
							ref_begin = reference_pointer
							state = State.off_on
						reference_pointer += 1
						cigar_pointer += 1
						sequence_pointer += 1
						# we have a homopolymer in the read, but not in the reference(or ref homopolymer ended)
					elif(state == State.on_off):
						#print("ON_OFF")
						if(ref_homopolymer == 0 and sequence[sequence_pointer] == read_letter):
							ref_letter = read_letter
							ref_homopolymer += 1
							state = State.on_on
							ref_begin = reference_pointer
						reference_pointer += 1
						cigar_pointer += 1
						sequence_pointer += 1

					elif(state == State.off_on):
						#rint("OFF_ON")
						if(sequence[sequence_pointer] == ref_letter):
							ref_homopolymer += 1
							reference_pointer += 1
							cigar_pointer += 1
							sequence_pointer += 1
						else:
							state = State.off_off

							ref_end = reference_pointer

							data = (ref_homopolymer, read_homopolymer, ref_letter)
							if(freq_dict.has_key(data)):
								freq_dict[data] += 1
							else:
								freq_dict[data] = 1

							ref_coordinates = (ref_begin, ref_end)
							regions_dict[ref_coordinates].append(read_homopolymer)

							ref_max = max(ref_max, ref_homopolymer)
							read_max = max(read_max, read_homopolymer)

							ref_begin = 0
							ref_end = 0

							read_homopolymer = 0
							ref_homopolymer = 0
							read_letter = ''
							ref_letter = ''

					elif(state == State.on_on):
						#print("ON_ON")
						if(sequence[sequence_pointer] == ref_letter):
							ref_homopolymer += 1
						else:
							state = State.on_off
							ref_end = reference_pointer
						cigar_pointer += 1
						reference_pointer += 1
						sequence_pointer += 1
				else:
					# S
					read_pointer += 1
					cigar_pointer += 1

		indent += len(sequence)

	return read_max, regions_dict<commit_msg>feature number added<commit_after>import numpy as np
import pysam
import re
from collections import defaultdict
from Bio import SeqIO
import time

class State:
	on_on = 0
	on_off = 1
	off_on = 2
	off_off = 3

def make_freq_csv(freq_dict, read_max, ref_max, out):
	#print(freq_dict)
	freq_cnt = np.zeros(shape=(ref_max + 1, read_max + 1), dtype=float)
	total = 0
	for key, val in freq_dict.items():
		ref_len = key[0]
		read_len = key[1]
		letter = key[2]
		freq_cnt[ref_len][read_len] += val
		total += val
	print(total)
	freq_cnt = np.divide(freq_cnt, float(total))
	with open(out,'w') as f:
		np.savetxt(f, freq_cnt.astype(float), fmt='%2.5f', delimiter=',')

def make_test_csv(regions_dict, read_max, out):
	size = len(regions_dict)
	freq_cnt = np.zeros(shape=(size, read_max + 4), dtype=int)
	cnt = 0
	for key, val in regions_dict.items():
		if(cnt >= size):
			break
		length = key[1] - key[0]
		for i in val:
			if(i < read_max):
				freq_cnt[cnt][i] += 1
		freq_cnt[cnt][read_max + 1] = length
		freq_cnt[cnt][read_max + 2] = key[0]
		freq_cnt[cnt][read_max + 3] = key[1]
		cnt += 1
	with open(out,'w') as f:
		np.savetxt(f, freq_cnt.astype(int), fmt='%i', delimiter=',')

def genome_preprocessing(reference_file):

	with open(reference_file) as f:
		content = f.readlines()
	content = [x for x in content if x[0] != '>']
	content = [x.strip() for x in content]
	genome=''.join(content)
	return genome

def get_cigar_string(cigar_string):

	cigar_components = re.findall(r'(\d+)([A-Z]{1})', cigar_string)
	cigar = ''
	for comp in cigar_components:
		number = comp[0]
		letter = comp[1]
		for i in range(int(number)):
			cigar+=letter
	return cigar

def readerrors(ref, reads):

	freq_dict = {}
	regions_dict = defaultdict(list)

	#track max homopolymer size
	ref_max = 0
	read_max = 0

	fasta_sequences = SeqIO.parse(open(ref),'fasta')

	indent = 0 # globalni pomak po referenci

	for seq in fasta_sequences:

		seq_id, sequence = seq.id, str(seq.seq) + 'X'
		content = pysam.AlignmentFile(reads)

		for r in content.fetch(until_eof=True):

			#ako se read nije namapirao na referencu
			if r.cigarstring is None:
	 			continue

	 		# ako se read nije namapirao na ovaj kromosom
			if(content.getrname(r.reference_id) != seq_id):
				continue

	 		cigar = get_cigar_string(r.cigarstring)

			#set up necessary variables
			#current pointers, they show us which positions we are looking at
			sequence_pointer = r.pos
			reference_pointer = r.pos + indent
			read_pointer = 0
			cigar_pointer = 0

			#length of homopolymers
			read_homopolymer = 0
			ref_homopolymer = 0

			#coordinates of homopolymers
			ref_begin = 0
			ref_end = 0

			#homopolymer letters. if they exist, they should never be different (except if one of them is empty)
			read_letter = ''
			ref_letter = ''

			#read sequence
			read = r.seq + 'X'

			#starting state
			state = State.off_off

			while(cigar_pointer < len(cigar)):

				if(cigar[cigar_pointer] == 'M'):

				#not a read nor a reference has a detected homopolymer
					if(state == State.off_off):
						if(read[read_pointer] == read[read_pointer + 1]):
							read_letter = read[read_pointer]
							read_homopolymer += 1
							state = State.on_off

							if(sequence[sequence_pointer] == read_letter):
								ref_homopolymer += 1
								ref_letter = read_letter
								state = State.on_on
								ref_begin = reference_pointer

						elif(sequence[sequence_pointer] == sequence[sequence_pointer + 1]):
							ref_homopolymer += 1
							ref_letter = sequence[sequence_pointer]
							state = State.off_on
							ref_begin = reference_pointer

							if(read[read_pointer] == ref_letter):
								read_letter = ref_letter
								read_homopolymer += 1
								state = State.on_on

						reference_pointer += 1
						read_pointer += 1
						cigar_pointer += 1
						sequence_pointer += 1

					# we have a homopolymer in the read, but not in the reference(or ref homopolymer ended)
					elif(state == State.on_off):

						#if homopolymer in read continues
						if(read[read_pointer] == read_letter):
							read_homopolymer += 1
							#if we didn't found a homopolymer in the reference by now, check
							if(ref_homopolymer == 0 and sequence[sequence_pointer] == read_letter):
								ref_letter = read_letter
								ref_homopolymer += 1
								state = State.on_on
								ref_begin = reference_pointer

							reference_pointer += 1
							read_pointer += 1
							cigar_pointer += 1
							sequence_pointer += 1

						else:
							#here we don't update pointers so we can check them in off_off state
							state = State.off_off

							data = (ref_homopolymer, read_homopolymer, read_letter)
							if(freq_dict.has_key(data)):
								freq_dict[data] += 1
							else:
								freq_dict[data] = 1

							ref_coordinates = (ref_begin, ref_end)
							regions_dict[ref_coordinates].append(read_homopolymer)

							ref_max = max(ref_max, ref_homopolymer)
							read_max = max(read_max, read_homopolymer)

							ref_begin = 0
							ref_end = 0
							read_homopolymer = 0
							ref_homopolymer = 0
							read_letter = ''
							ref_letter = ''

					# we have a homopolymer in the reference, but not in the read (or read homopolymer ended)
					elif(state == State.off_on):

						#if homopolymer in reference continues
						if(sequence[sequence_pointer] == ref_letter):
							ref_homopolymer += 1
							if(read_homopolymer == 0 and read[read_pointer] == ref_letter):
								read_letter = ref_letter
								read_homopolymer += 1
								state = State.on_on

							reference_pointer += 1
							read_pointer += 1
							cigar_pointer += 1
							sequence_pointer += 1

						else:
							#here we don't update pointers so we can check them in off_off state
							state = State.off_off
							ref_end = reference_pointer

							data = (ref_homopolymer, read_homopolymer, ref_letter)
							if(freq_dict.has_key(data)):
								freq_dict[data] += 1
							else:
								freq_dict[data] = 1

							ref_coordinates = (ref_begin, ref_end)
							regions_dict[ref_coordinates].append(read_homopolymer)

							ref_max = max(ref_max, ref_homopolymer)
							read_max = max(read_max, read_homopolymer)

							ref_begin = 0
							ref_end = 0
							read_homopolymer = 0
							ref_homopolymer = 0
							read_letter = ''
							ref_letter = ''

					elif(state == State.on_on):
						#print("ON_ON")
						# if read homopolymer continues
						if(read[read_pointer] == read_letter and sequence[sequence_pointer] == ref_letter):
							read_homopolymer += 1
							ref_homopolymer += 1

							reference_pointer += 1
							read_pointer += 1
							cigar_pointer += 1
							sequence_pointer += 1

						elif(read[read_pointer] == read_letter):
							read_homopolymer += 1
							state = State.on_off

							ref_end = reference_pointer

							reference_pointer += 1
							read_pointer += 1
							cigar_pointer += 1
							sequence_pointer += 1

						elif(sequence[sequence_pointer] == ref_letter):
							ref_homopolymer += 1
							state = State.off_on

							reference_pointer += 1
							read_pointer += 1
							cigar_pointer += 1
							sequence_pointer += 1

						else:
							state = State.off_off

							ref_end = reference_pointer

							data = (ref_homopolymer, read_homopolymer, read_letter)
							if(freq_dict.has_key(data)):
								freq_dict[data] += 1
							else:
								freq_dict[data] = 1

							ref_coordinates = (ref_begin, ref_end)
							regions_dict[ref_coordinates].append(read_homopolymer)

							ref_max = max(ref_max, ref_homopolymer)
							read_max = max(read_max, read_homopolymer)

							ref_begin = 0
							ref_end = 0

							read_homopolymer = 0
							ref_homopolymer = 0
							read_letter = ''
							ref_letter = ''

				elif(cigar[cigar_pointer] == 'I'):

					if(state == State.off_off):
						#print("OFF_OFF")
						if(read[read_pointer] == read[read_pointer + 1]):
							read_letter = read[read_pointer]
							read_homopolymer += 1
							state = State.on_off

						read_pointer += 1
						cigar_pointer += 1
						# we have a homopolymer in the read, but not in the reference(or ref homopolymer ended)
					elif(state == State.on_off):
						#print("ON_OFF")
						if(read[read_pointer] == read_letter):
							read_homopolymer += 1
							read_pointer += 1
							cigar_pointer += 1
						else:
							state = State.off_off

							data = (ref_homopolymer, read_homopolymer, read_letter)
							if(freq_dict.has_key(data)):
								freq_dict[data] += 1
							else:
								freq_dict[data] = 1

							ref_coordinates = (ref_begin, ref_end)
							regions_dict[ref_coordinates].append(read_homopolymer)

							ref_max = max(ref_max, ref_homopolymer)
							read_max = max(read_max, read_homopolymer)

							ref_begin = 0
							ref_end = 0

							read_homopolymer = 0
							ref_homopolymer = 0
							read_letter = ''
							ref_letter = ''

					elif(state == State.off_on):
						#print("OFF_ON")
						if(read_homopolymer == 0 and read[read_pointer] == ref_letter):
							read_letter = ref_letter
							read_homopolymer += 1
							state = State.on_on
						read_pointer += 1
						cigar_pointer += 1

					elif(state == State.on_on):
						#print("ON_ON")
						if(read[read_pointer] == read_letter):
							read_homopolymer += 1
						else:
							state = State.off_on
						read_pointer += 1
						cigar_pointer += 1

				elif(cigar[cigar_pointer] == 'D'):
					if(state == State.off_off):
						#print("OFF_OFF")
						if(sequence[sequence_pointer] == sequence[sequence_pointer + 1]):
							ref_homopolymer += 1
							ref_letter = sequence[sequence_pointer]
							ref_begin = reference_pointer
							state = State.off_on
						reference_pointer += 1
						cigar_pointer += 1
						sequence_pointer += 1
						# we have a homopolymer in the read, but not in the reference(or ref homopolymer ended)
					elif(state == State.on_off):
						#print("ON_OFF")
						if(ref_homopolymer == 0 and sequence[sequence_pointer] == read_letter):
							ref_letter = read_letter
							ref_homopolymer += 1
							state = State.on_on
							ref_begin = reference_pointer
						reference_pointer += 1
						cigar_pointer += 1
						sequence_pointer += 1

					elif(state == State.off_on):
						#rint("OFF_ON")
						if(sequence[sequence_pointer] == ref_letter):
							ref_homopolymer += 1
							reference_pointer += 1
							cigar_pointer += 1
							sequence_pointer += 1
						else:
							state = State.off_off

							ref_end = reference_pointer

							data = (ref_homopolymer, read_homopolymer, ref_letter)
							if(freq_dict.has_key(data)):
								freq_dict[data] += 1
							else:
								freq_dict[data] = 1

							ref_coordinates = (ref_begin, ref_end)
							regions_dict[ref_coordinates].append(read_homopolymer)

							ref_max = max(ref_max, ref_homopolymer)
							read_max = max(read_max, read_homopolymer)

							ref_begin = 0
							ref_end = 0

							read_homopolymer = 0
							ref_homopolymer = 0
							read_letter = ''
							ref_letter = ''

					elif(state == State.on_on):
						#print("ON_ON")
						if(sequence[sequence_pointer] == ref_letter):
							ref_homopolymer += 1
						else:
							state = State.on_off
							ref_end = reference_pointer
						cigar_pointer += 1
						reference_pointer += 1
						sequence_pointer += 1
				else:
					# S
					read_pointer += 1
					cigar_pointer += 1

		indent += len(sequence)

	return read_max, regions_dict<|endoftext|>


<commit_before>##############################################################################
# PyCharm Community Edition
# -*- coding: utf-8 -*-
"""
Orthologs-Project
Directory_management updated on 11/15/2016 at 11:30 AM
##############################################################################

    Input:  A string that represents a path for the main file systems

    Output:  Custom class variables for import in the project file template.

    Description:  This file manges the directories for projects created in PyCharm.
    Each new project gets it's own function.

##############################################################################
@author: rgilmore
"""
##############################################################################
# Libraries:

import json
import os
import shutil
import time
from pathlib import Path
from cookiecutter.main import cookiecutter
from Manager.utils.treelib2.treelib2.tree import Tree
import ete3
from Manager.utils.json_to_newick import _parse_json
# TODO-ROB once this is a pypi package all of these will be unnecessary
import Cookies
import Manager
import Orthologs
import Tools
from Manager.logit.logit import LogIt

#from project_mana import project_mana  # //TODO-ROB: Add a configure function to proj_mana to get the root directory using the project name
##############################################################################
# Directory Initializations:

class Mana(object):
    """
    This is the directory management base class.  It
    maps the directories in the PyPi package using the pathlib module and
    turns the names of each important directory into a pathlike object.  The
    base class gives the option of creating a new repository with cookiecutter.

    This is also the home for many of the utility functions for manipulating
    directories or paths.
    """

# //TODO-ROB Add JSON loading of the different directory variables
# //TODO-Rob change project to projects and add another variable called project_type

    def __init__(self, repo=None, home=os.getcwd(), new_repo=False):
        """
        :param home(path or path-like): The home of the file calling this name.  When creating a new
            repository it is best to explicitly name the home path.
        :param repo(string): The name of the new repository to be created.
        :param new_repo(bool): Triggers cookiecutter to create a new repository.
        """
        # config = tablib.Dataset().load(open('config.yaml').read())
        self.file_home = Path(home)  # Home of the file calling this class
        # TODO-ROB:  SOme of these directories don't need to be accessed directly
        # Below are the PyPi path strings
        #    The first group is to access the cookiecutter templates
        self.Cookies = Path(Cookies.__path__._path[0])
        self.repo_cookie = self.Cookies / Path('new_repository')
        self.user_cookie = self.Cookies / Path('new_user')
        self.project_cookie = self.Cookies / Path('new_project')
        self.research_cookie = self.Cookies / Path('new_research')
        self.app_cookie = self.Cookies / Path('new_app')
        #    The second group is for the Manager module
        self.Manager = Path(Manager.__path__._path[0])
        self.index = self.Manager / Path('index')
        self.logit = self.Manager / Path('logit')
        self.utils = self.Manager / Path('utils')
        self.shiny = self.Manager / Path('shiny')
        #    The third group is for the Orthologs module
        self.Orthologs = Path(Orthologs.__path__._path[0])
        self.biosql = Path(self.Orthologs) / Path('biosql')
        self.blast = Path(self.Orthologs) / Path('blast')
        self.comp_gen = Path(self.Orthologs) / Path('comparative_genetics')
        self.genbank = Path(self.Orthologs) / Path('genbank')
        self.manager = Path(self.Orthologs) / Path('manager')
        self.phylogenetics = Path(self.Orthologs) / Path('phylogenetics')
        #    The fourth group is for the Tools module
        self.Tools = Path(Tools.__path__._path[0])
        self.ftp = Path(self.Tools) / Path('ftp')
        self.multiprocessing = Path(self.Tools) / Path('multiprocessing')
        self.pandoc = Path(self.Tools) / Path('pandoc')
        self.pybasher = Path(self.Tools) / Path('pybasher')
        self.qsub = Path(self.Tools) / Path('qsub')

        if repo:
            self.repo_path = self.file_home / Path(self.repo)
            self.repo = repo
        if new_repo is True:
            self.create_repo()

        # Create a directory management logger
        # TODO-ROB figure out where to put this based on user stuff
        #log = LogIt('user/path/userfile.log', 'Directory Management')
        #self.dm_log = log.basic

    def create_repo(self):
        """This function creates a new repository.  If a repository name
        is given to the class then it is given a name.  If not, cookiecutters
        takes input from the user.

        The base class will be the only class that allows cookiecutters parameter
        no_input to be False.
        """
        if self.repo:
            no_input = True
            e_c = {
                "repository_name": self.repo
            }
        else:
            no_input = False
            e_c = None
        cookiecutter(self.repo_cookie, no_input=no_input, extra_context=e_c, output_dir=self.file_home)

    # def git_ignore(self, path):
    #     """Get the ignored file patterns from the .gitignore file in the repo."""
    #     with open(Path(path) / Path('.gitignore'), 'r', newline='') as ignore:
    #         ignored = ignore.read().splitlines()
    #     return ignored
    #
    # # Map the main project directory.
    # def get_dir_map(self, top, gitignore=None):
    #     # TODO-ROB:  Change ignore to a .gitignore filename
    #     default_ignore = self.git_ignore(top)
    #     if gitignore is not None:
    #         gitignore += default_ignore
    #     else:
    #         gitignore = default_ignore
    #     # Treelib will help to map everything and create a json at the same time
    #     tree = Tree()
    #     tree.create_node('.', top)
    #     # Walk through the directory of choice (top)
    #     # Topdown is true so that directories can be modified in place
    #     for root, dirs, files in os.walk(top, topdown=True):
    #         # Only remove directories from the top
    #         if root == top:
    #             print(root)
    #             try:
    #                 dirs[:] = set(dirs) - set(gitignore)  # Remove directories from os.walk()
    #                 print(dirs)
    #             except ValueError:
    #                 pass
    #         for d in dirs:
    #             rd = str(Path(root) / Path(d))
    #             tree.create_node(d, identifier=rd, parent=root)
    #         for f in files:
    #             tree.create_node(f, parent=root)
    #     return tree

    def get_newick_dir_map(self, top, ignore=None):
        """Takes a treelib tree created by get_dir_map and returns
        a tree a dir_map in newick format.  This will be useful for Bio.Phylo
        applications."""

        tree = Tree()
        t = tree.get_dir_map(top, ignore)
        Ntree = tree.parse_newick_json()
        return Ntree

    def get_ete3_tree(self, top, tree=None):
        if not tree:
            tree = self.get_newick_dir_map(top)
        t = ete3.Tree(tree, format=8)
        return t

    # DEPRECATED Change this IN OTHER CLASSES
    def path_list_make(self, path, o_path=None):
        # Takes a path and reduces it to a list of directories within the project
        # An optional attribute (o_path) is give so that a deeper path within the project can be used
        home = str(self.__project_home).split('/')
        path_list = str(path).split('/')
        for item in home:
            if item in path_list:
                path_list.remove(item)
        # path_list = set(p) - set(home)
        if o_path is not None:
            o_path = str(o_path).split('/')
            for item in o_path:
                if item in path_list:
                    path_list.remove(item)
            # path_list = set(path_list) - set(o_path)
        return path_list

    # //TODO-ROB utilize Path.mkdir(parents=TRUE) instead
        # DEPRECATED Change this IN OTHER CLASSES
    def dir_make(self, path, path_list):
        # Takes a path list which is a list of folder names
        # path_list created by str(path).split('/')
        # The path_list appends to path, which is already an established directory inside the project
        t = None
        for item in path_list:

            if os.path.isdir(path + '/' + item): # If for some reason the directory already exists...
                path += '/' + item  # Append a directory
                continue
            path += '/' + item  # Append a directory
            os.mkdir(path)
        if len(os.listdir(path)) > 0:
            path, t = self.dir_archive(path, path_list='')
        return path, t

    # # //TODO-ROB Change to using a compression module https://pymotw.com/2/compression.html
        # DEPRECATED Change this IN OTHER CLASSES
    # def dir_archive(self, path, path_list):
    #     # Use the path that you want to update/add to
    #     # Returns path and the time stamp (could be None)
    #     unique_dir = False
    #     archive_path = path
    #     for item in path_list:
    #
    #         path += '/' + item  # Append a directory
    #         if os.path.isdir(path):  # If the child directory exists
    #             archive_path = path  # Then update the dir_archive path and continue
    #             continue
    #         else:                     # If the child directory doesnt exist
    #             unique_dir = True     # Then raise the flag
    #             os.mkdir(path)        # And make a directory
    #
    #     if unique_dir is False:  # Only dir_archive if the final child directory is not unique (via unique_dir = False)
    #         t = time.strftime("%m%d%Y-%I%M%S")
    #         new_archive = self.Archive + '/' + t  # Creates a time stamped directory
    #
    #         os.mkdir(new_archive)
    #         for item in os.listdir(archive_path):
    #             if os.path.isfile(archive_path + '/' + item):  # Only dir_archive the FILES
    #                 shutil.move(archive_path + '/' + item, new_archive)
    #         return path, t
    #     else:
    #         return path, None

'(.gitignore,GetBlastDB.py,README.md,ftp2db.py,ncbiftp.cfg,(blastftp.txt,ftpdownloadtest.py)tests);'

<commit_msg>Added website cookie variable.<commit_after>##############################################################################
# PyCharm Community Edition
# -*- coding: utf-8 -*-
"""
Orthologs-Project
Directory_management updated on 11/15/2016 at 11:30 AM
##############################################################################

    Input:  A string that represents a path for the main file systems

    Output:  Custom class variables for import in the project file template.

    Description:  This file manges the directories for projects created in PyCharm.
    Each new project gets it's own function.

##############################################################################
@author: rgilmore
"""
##############################################################################
# Libraries:

import json
import os
import shutil
import time
from pathlib import Path
from cookiecutter.main import cookiecutter
from Manager.utils.treelib2.treelib2.tree import Tree
import ete3
from Manager.utils.json_to_newick import _parse_json
# TODO-ROB once this is a pypi package all of these will be unnecessary
import Cookies
import Manager
import Orthologs
import Tools
from Manager.logit.logit import LogIt

#from project_mana import project_mana  # //TODO-ROB: Add a configure function to proj_mana to get the root directory using the project name
##############################################################################
# Directory Initializations:

class Mana(object):
    """
    This is the directory management base class.  It
    maps the directories in the PyPi package using the pathlib module and
    turns the names of each important directory into a pathlike object.  The
    base class gives the option of creating a new repository with cookiecutter.

    This is also the home for many of the utility functions for manipulating
    directories or paths.
    """

# //TODO-ROB Add JSON loading of the different directory variables
# //TODO-Rob change project to projects and add another variable called project_type

    def __init__(self, repo=None, home=os.getcwd(), new_repo=False):
        """
        :param home(path or path-like): The home of the file calling this name.  When creating a new
            repository it is best to explicitly name the home path.
        :param repo(string): The name of the new repository to be created.
        :param new_repo(bool): Triggers cookiecutter to create a new repository.
        """
        # config = tablib.Dataset().load(open('config.yaml').read())
        self.file_home = Path(home)  # Home of the file calling this class
        # TODO-ROB:  SOme of these directories don't need to be accessed directly
        # Below are the PyPi path strings
        #    The first group is to access the cookiecutter templates
        self.Cookies = Path(Cookies.__path__._path[0])
        self.repo_cookie = self.Cookies / Path('new_repository')
        self.user_cookie = self.Cookies / Path('new_user')
        self.project_cookie = self.Cookies / Path('new_project')
        self.research_cookie = self.Cookies / Path('new_research')
        self.app_cookie = self.Cookies / Path('new_app')
        self.website_cookie = self.Cookies / Path('new_website')
        #    The second group is for the Manager module
        self.Manager = Path(Manager.__path__._path[0])
        self.index = self.Manager / Path('index')
        self.logit = self.Manager / Path('logit')
        self.utils = self.Manager / Path('utils')
        self.shiny = self.Manager / Path('shiny')
        #    The third group is for the Orthologs module
        self.Orthologs = Path(Orthologs.__path__._path[0])
        self.biosql = Path(self.Orthologs) / Path('biosql')
        self.blast = Path(self.Orthologs) / Path('blast')
        self.comp_gen = Path(self.Orthologs) / Path('comparative_genetics')
        self.genbank = Path(self.Orthologs) / Path('genbank')
        self.manager = Path(self.Orthologs) / Path('manager')
        self.phylogenetics = Path(self.Orthologs) / Path('phylogenetics')
        #    The fourth group is for the Tools module
        self.Tools = Path(Tools.__path__._path[0])
        self.ftp = Path(self.Tools) / Path('ftp')
        self.multiprocessing = Path(self.Tools) / Path('multiprocessing')
        self.pandoc = Path(self.Tools) / Path('pandoc')
        self.pybasher = Path(self.Tools) / Path('pybasher')
        self.qsub = Path(self.Tools) / Path('qsub')

        if repo:
            self.repo_path = self.file_home / Path(self.repo)
            self.repo = repo
        if new_repo is True:
            self.create_repo()

        # Create a directory management logger
        # TODO-ROB figure out where to put this based on user stuff
        #log = LogIt('user/path/userfile.log', 'Directory Management')
        #self.dm_log = log.basic

    def create_repo(self):
        """This function creates a new repository.  If a repository name
        is given to the class then it is given a name.  If not, cookiecutters
        takes input from the user.

        The base class will be the only class that allows cookiecutters parameter
        no_input to be False.
        """
        if self.repo:
            no_input = True
            e_c = {
                "repository_name": self.repo
            }
        else:
            no_input = False
            e_c = None
        cookiecutter(self.repo_cookie, no_input=no_input, extra_context=e_c, output_dir=self.file_home)

    # def git_ignore(self, path):
    #     """Get the ignored file patterns from the .gitignore file in the repo."""
    #     with open(Path(path) / Path('.gitignore'), 'r', newline='') as ignore:
    #         ignored = ignore.read().splitlines()
    #     return ignored
    #
    # # Map the main project directory.
    # def get_dir_map(self, top, gitignore=None):
    #     # TODO-ROB:  Change ignore to a .gitignore filename
    #     default_ignore = self.git_ignore(top)
    #     if gitignore is not None:
    #         gitignore += default_ignore
    #     else:
    #         gitignore = default_ignore
    #     # Treelib will help to map everything and create a json at the same time
    #     tree = Tree()
    #     tree.create_node('.', top)
    #     # Walk through the directory of choice (top)
    #     # Topdown is true so that directories can be modified in place
    #     for root, dirs, files in os.walk(top, topdown=True):
    #         # Only remove directories from the top
    #         if root == top:
    #             print(root)
    #             try:
    #                 dirs[:] = set(dirs) - set(gitignore)  # Remove directories from os.walk()
    #                 print(dirs)
    #             except ValueError:
    #                 pass
    #         for d in dirs:
    #             rd = str(Path(root) / Path(d))
    #             tree.create_node(d, identifier=rd, parent=root)
    #         for f in files:
    #             tree.create_node(f, parent=root)
    #     return tree

    def get_newick_dir_map(self, top, ignore=None):
        """Takes a treelib tree created by get_dir_map and returns
        a tree a dir_map in newick format.  This will be useful for Bio.Phylo
        applications."""

        tree = Tree()
        t = tree.get_dir_map(top, ignore)
        Ntree = tree.parse_newick_json()
        return Ntree

    def get_ete3_tree(self, top, tree=None):
        if not tree:
            tree = self.get_newick_dir_map(top)
        t = ete3.Tree(tree, format=8)
        return t

    # DEPRECATED Change this IN OTHER CLASSES
    def path_list_make(self, path, o_path=None):
        # Takes a path and reduces it to a list of directories within the project
        # An optional attribute (o_path) is give so that a deeper path within the project can be used
        home = str(self.__project_home).split('/')
        path_list = str(path).split('/')
        for item in home:
            if item in path_list:
                path_list.remove(item)
        # path_list = set(p) - set(home)
        if o_path is not None:
            o_path = str(o_path).split('/')
            for item in o_path:
                if item in path_list:
                    path_list.remove(item)
            # path_list = set(path_list) - set(o_path)
        return path_list

    # //TODO-ROB utilize Path.mkdir(parents=TRUE) instead
        # DEPRECATED Change this IN OTHER CLASSES
    def dir_make(self, path, path_list):
        # Takes a path list which is a list of folder names
        # path_list created by str(path).split('/')
        # The path_list appends to path, which is already an established directory inside the project
        t = None
        for item in path_list:

            if os.path.isdir(path + '/' + item): # If for some reason the directory already exists...
                path += '/' + item  # Append a directory
                continue
            path += '/' + item  # Append a directory
            os.mkdir(path)
        if len(os.listdir(path)) > 0:
            path, t = self.dir_archive(path, path_list='')
        return path, t

    # # //TODO-ROB Change to using a compression module https://pymotw.com/2/compression.html
        # DEPRECATED Change this IN OTHER CLASSES
    # def dir_archive(self, path, path_list):
    #     # Use the path that you want to update/add to
    #     # Returns path and the time stamp (could be None)
    #     unique_dir = False
    #     archive_path = path
    #     for item in path_list:
    #
    #         path += '/' + item  # Append a directory
    #         if os.path.isdir(path):  # If the child directory exists
    #             archive_path = path  # Then update the dir_archive path and continue
    #             continue
    #         else:                     # If the child directory doesnt exist
    #             unique_dir = True     # Then raise the flag
    #             os.mkdir(path)        # And make a directory
    #
    #     if unique_dir is False:  # Only dir_archive if the final child directory is not unique (via unique_dir = False)
    #         t = time.strftime("%m%d%Y-%I%M%S")
    #         new_archive = self.Archive + '/' + t  # Creates a time stamped directory
    #
    #         os.mkdir(new_archive)
    #         for item in os.listdir(archive_path):
    #             if os.path.isfile(archive_path + '/' + item):  # Only dir_archive the FILES
    #                 shutil.move(archive_path + '/' + item, new_archive)
    #         return path, t
    #     else:
    #         return path, None

'(.gitignore,GetBlastDB.py,README.md,ftp2db.py,ncbiftp.cfg,(blastftp.txt,ftpdownloadtest.py)tests);'

<|endoftext|>


<commit_before>#
# Copyright (c) 2004-2007 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
"""
Provides a file which can store multiple files inside of it.

A table of contents is utilized, and the caller can attach arbitrary
data to an entry in the table. No compression is used, but that could
be changed.

The data attached to table entries can be modified, but doing so isn't
particularly efficient (the entire table gets rewritten).

The last file added to the container can be erased from it. This is
to allow operations to be reverted.

The file format is::
  - magic
  - file format version
  - file table entry 1
  - file table entry 2
  - file N

The header and table entries are uncompressed.  The contents of the file
table are compressed, and each file is individually compressed. When files
are retrieved from the container, the returned file object automatically
uncompresses the file.

There are two formats for file table entrys. The original format is used
for all files less than 4GB in size::

  SUBFILE_MAGIC (2 bytes)
  length of file name (2 bytes)
  length of compressed file data(4 bytes)
  length of arbitrary data (2 bytes)
  file name
  arbitrary file table data
  file data

Files larger than 4GB are stored somewhat differently:

  LARGE_SUBFILE_MAGIC (2 bytes)
  length of file name + length of file table data + file length (8 bytes)
  file name
  arbitrary file table data
  file data
  length of file name (2 bytes)
  length of arbitrary data (2 bytes)

This somewhat tortuous format was designed to accomodate the addFile API not
knowing the total size being stored as well as the dynamic compression making
that size unknowable in advance. It does limit file storage size a bit, but
leaves us with well over 63 bits of length.

This code is careful not to depend on the file pointer at all for reading
(via pread). The file pointer is used while creating file containers.
"""

import gzip
import struct

import conary.errors
from conary.lib import util
from conary.repository import filecontents

FILE_CONTAINER_MAGIC = "\xEA\x3F\x81\xBB"
SUBFILE_MAGIC = 0x3FBB
# used for files whose contents are > 4gig
LARGE_SUBFILE_MAGIC = 0x40CD

FILE_CONTAINER_VERSION_FILEID_IDX   = 2007022001
FILE_CONTAINER_VERSION_WITH_REMOVES = 2006071301
FILE_CONTAINER_VERSION_NO_REMOVES   = 2005101901

READABLE_VERSIONS = [ FILE_CONTAINER_VERSION_FILEID_IDX,
                      FILE_CONTAINER_VERSION_WITH_REMOVES,
                      FILE_CONTAINER_VERSION_NO_REMOVES ]
SEEK_SET = 0
SEEK_CUR = 1
SEEK_END = 2

class FileContainer:

    bufSize = 128 * 1024

    def readHeader(self):
	magic = self.file.pread(4, 0)
	if len(magic) != 4 or magic != FILE_CONTAINER_MAGIC:
	    raise BadContainer, "bad magic"

	version = self.file.pread(4, 4)
	if len(version) != 4:
	    raise BadContainer, "invalid container version"
        self.version = struct.unpack("!I", version)[0]
        if self.version not in READABLE_VERSIONS:
            raise BadContainer, "unsupported file container version %d" % \
                        self.version

	self.contentsStart = 8
	self.next = self.contentsStart

    def close(self):
        self.file = None

    def addFile(self, fileName, contents, tableData, precompressed = False):
	assert(isinstance(contents, filecontents.FileContents))
	assert(self.mutable)

	fileObj = contents.get()
        headerOffset = self.file.tell()
        self.file.write(struct.pack("!HH", SUBFILE_MAGIC, len(fileName)))
	self.file.write(struct.pack("!IH", 0, len(tableData)))
	self.file.write(fileName)
	self.file.write(tableData)

        if precompressed:
            size = util.copyfileobj(fileObj, self.file)
        else:
            start = self.file.tell()
            gzFile = gzip.GzipFile('', "wb", 6, self.file)
            util.copyfileobj(fileObj, gzFile)
            gzFile.close()
            size = self.file.tell() - start

        if size < 0x100000000:
            self.file.seek(headerOffset + 4, SEEK_SET)
            self.file.write(struct.pack("!I", size))
            self.file.seek(0, SEEK_END)
        else:
            self.file.seek(headerOffset, SEEK_SET)
            totalSize = size + len(fileName) + len(tableData)
            self.file.write(struct.pack("!HII", LARGE_SUBFILE_MAGIC,
                                        totalSize >> 32,
                                        totalSize & 0xFFFFFFFF))
            self.file.seek(0, SEEK_END)
            self.file.write(struct.pack("!HH", len(fileName), len(tableData)))

    def getNextFile(self):
	assert(not self.mutable)

        name, tag, size, dataOffset, nextOffset = self._nextFile()

	if name is None:
	    return None

	fcf = util.SeekableNestedFile(self.file, size, start = dataOffset)

        self.next = nextOffset

	return (name, tag, fcf)

    def _nextFile(self):
        offset = self.next

	nameLen = self.file.pread(10, offset)
	if not len(nameLen):
            return (None, None, None, None, None)

        offset += 10

        subMagic = struct.unpack("!H", nameLen[0:2])[0]
        if subMagic == LARGE_SUBFILE_MAGIC:
            most, least = struct.unpack("!II", nameLen[2:])
            totalSize = (most << 32) + least

            otherLengths = self.file.pread(4, offset + totalSize)
            nameLen, tagLen  = struct.unpack("!HH", otherLengths)
            size = totalSize - nameLen - tagLen
            nextOffset = offset + totalSize + 4
        else:
            assert(subMagic == SUBFILE_MAGIC)
            nameLen, size, tagLen = struct.unpack("!HIH", nameLen[2:])
            nextOffset = offset + nameLen + tagLen + size

        name = self.file.pread(nameLen, offset)
        offset += nameLen
        tag = self.file.pread(tagLen, offset)
        offset += tagLen

        return (name, tag, size, offset, nextOffset)

    def dump(self, dumpString, dumpFile):
        def sizeCallback(dumpString, name, tag, size):
            hdr = struct.pack("!HHIH%ds%ds" % (len(name), len(tag)),
                        SUBFILE_MAGIC, len(name), size, len(tag), name, tag)
            dumpString(hdr)

	assert(not self.mutable)

        fileHeader = self.file.pread(8, 0)
        dumpString(fileHeader)

        next = self.getNextFile()
        while next is not None:
            (name, tag, fcf) = next
            size = fcf.size
            dumpFile(name, tag, size, fcf,
                     lambda size, newTag:
                        sizeCallback(dumpString, name, newTag, size))
            next = self.getNextFile()

    def reset(self):
        """
        Reset the current position in the filecontainer to the beginning.
        """
        assert(not self.mutable)
        self.next = self.contentsStart

    def __del__(self):
	if self.file:
	    self.close()

    def __init__(self, file, version = None):
        """
        Create a FileContainer object.

        @param file: an open python file object referencing the file
        container file on disk. If that file is empty (size 0) the
        file container is immediately initialized. A copy of the file
        is retained, so the caller may optionally close it.
        """

	# make our own copy of this file which nobody can close underneath us
	self.file = file

        if version is None:
            version = FILE_CONTAINER_VERSION_FILEID_IDX

	self.file.seek(0, SEEK_END)
	if not self.file.tell():
	    self.file.seek(SEEK_SET, 0)
	    self.file.truncate()
	    self.file.write(FILE_CONTAINER_MAGIC)
	    self.file.write(struct.pack("!I", version))

	    self.mutable = True
	else:
            # we don't need to put this file pointer back; we don't depend
            # on it here at all; everything is through pseek
	    try:
		self.readHeader()
	    except:
		self.file.close()
		self.file = None
		raise
	    self.mutable = False

class BadContainer(conary.errors.ConaryError):

    pass
<commit_msg>improve file entry docs<commit_after>#
# Copyright (c) 2004-2007 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
"""
Provides a file which can store multiple files inside of it.

A table of contents is utilized, and the caller can attach arbitrary
data to an entry in the table. No compression is used, but that could
be changed.

The data attached to table entries can be modified, but doing so isn't
particularly efficient (the entire table gets rewritten).

The last file added to the container can be erased from it. This is
to allow operations to be reverted.

The file format is::
  - magic
  - file format version
  - file table entry 1
  - file table entry 2
  - file N

The header and table entries are uncompressed.  The contents of the file
table are compressed, and each file is individually compressed. When files
are retrieved from the container, the returned file object automatically
uncompresses the file.

There are two formats for file table entries. The original format is used
for all files less than 4GB in size (after compression)::

  SUBFILE_MAGIC (2 bytes)
  length of file name (2 bytes)
  length of compressed file data(4 bytes)
  length of arbitrary data (2 bytes)
  file name
  arbitrary file table data
  file data

Files larger than 4GB are stored somewhat differently:

  LARGE_SUBFILE_MAGIC (2 bytes)
  length of file name + length of file table data + file length (8 bytes)
  file name
  arbitrary file table data
  file data
  length of file name (2 bytes)
  length of arbitrary data (2 bytes)

This somewhat tortuous format was designed to accomodate the addFile API not
knowing the total size being stored as well as the dynamic compression making
that size unknowable in advance. It does limit file storage size a bit, but
leaves us with well over 63 bits of length.

This code is careful not to depend on the file pointer at all for reading
(via pread). The file pointer is used while creating file containers.
"""

import gzip
import struct

import conary.errors
from conary.lib import util
from conary.repository import filecontents

FILE_CONTAINER_MAGIC = "\xEA\x3F\x81\xBB"
SUBFILE_MAGIC = 0x3FBB
# used for files whose contents are > 4gig
LARGE_SUBFILE_MAGIC = 0x40CD

FILE_CONTAINER_VERSION_FILEID_IDX   = 2007022001
FILE_CONTAINER_VERSION_WITH_REMOVES = 2006071301
FILE_CONTAINER_VERSION_NO_REMOVES   = 2005101901

READABLE_VERSIONS = [ FILE_CONTAINER_VERSION_FILEID_IDX,
                      FILE_CONTAINER_VERSION_WITH_REMOVES,
                      FILE_CONTAINER_VERSION_NO_REMOVES ]
SEEK_SET = 0
SEEK_CUR = 1
SEEK_END = 2

class FileContainer:

    bufSize = 128 * 1024

    def readHeader(self):
	magic = self.file.pread(4, 0)
	if len(magic) != 4 or magic != FILE_CONTAINER_MAGIC:
	    raise BadContainer, "bad magic"

	version = self.file.pread(4, 4)
	if len(version) != 4:
	    raise BadContainer, "invalid container version"
        self.version = struct.unpack("!I", version)[0]
        if self.version not in READABLE_VERSIONS:
            raise BadContainer, "unsupported file container version %d" % \
                        self.version

	self.contentsStart = 8
	self.next = self.contentsStart

    def close(self):
        self.file = None

    def addFile(self, fileName, contents, tableData, precompressed = False):
	assert(isinstance(contents, filecontents.FileContents))
	assert(self.mutable)

	fileObj = contents.get()
        headerOffset = self.file.tell()
        self.file.write(struct.pack("!HH", SUBFILE_MAGIC, len(fileName)))
	self.file.write(struct.pack("!IH", 0, len(tableData)))
	self.file.write(fileName)
	self.file.write(tableData)

        if precompressed:
            size = util.copyfileobj(fileObj, self.file)
        else:
            start = self.file.tell()
            gzFile = gzip.GzipFile('', "wb", 6, self.file)
            util.copyfileobj(fileObj, gzFile)
            gzFile.close()
            size = self.file.tell() - start

        if size < 0x100000000:
            self.file.seek(headerOffset + 4, SEEK_SET)
            self.file.write(struct.pack("!I", size))
            self.file.seek(0, SEEK_END)
        else:
            self.file.seek(headerOffset, SEEK_SET)
            totalSize = size + len(fileName) + len(tableData)
            self.file.write(struct.pack("!HII", LARGE_SUBFILE_MAGIC,
                                        totalSize >> 32,
                                        totalSize & 0xFFFFFFFF))
            self.file.seek(0, SEEK_END)
            self.file.write(struct.pack("!HH", len(fileName), len(tableData)))

    def getNextFile(self):
	assert(not self.mutable)

        name, tag, size, dataOffset, nextOffset = self._nextFile()

	if name is None:
	    return None

	fcf = util.SeekableNestedFile(self.file, size, start = dataOffset)

        self.next = nextOffset

	return (name, tag, fcf)

    def _nextFile(self):
        offset = self.next

	nameLen = self.file.pread(10, offset)
	if not len(nameLen):
            return (None, None, None, None, None)

        offset += 10

        subMagic = struct.unpack("!H", nameLen[0:2])[0]
        if subMagic == LARGE_SUBFILE_MAGIC:
            most, least = struct.unpack("!II", nameLen[2:])
            totalSize = (most << 32) + least

            otherLengths = self.file.pread(4, offset + totalSize)
            nameLen, tagLen  = struct.unpack("!HH", otherLengths)
            size = totalSize - nameLen - tagLen
            nextOffset = offset + totalSize + 4
        else:
            assert(subMagic == SUBFILE_MAGIC)
            nameLen, size, tagLen = struct.unpack("!HIH", nameLen[2:])
            nextOffset = offset + nameLen + tagLen + size

        name = self.file.pread(nameLen, offset)
        offset += nameLen
        tag = self.file.pread(tagLen, offset)
        offset += tagLen

        return (name, tag, size, offset, nextOffset)

    def dump(self, dumpString, dumpFile):
        def sizeCallback(dumpString, name, tag, size):
            hdr = struct.pack("!HHIH%ds%ds" % (len(name), len(tag)),
                        SUBFILE_MAGIC, len(name), size, len(tag), name, tag)
            dumpString(hdr)

	assert(not self.mutable)

        fileHeader = self.file.pread(8, 0)
        dumpString(fileHeader)

        next = self.getNextFile()
        while next is not None:
            (name, tag, fcf) = next
            size = fcf.size
            dumpFile(name, tag, size, fcf,
                     lambda size, newTag:
                        sizeCallback(dumpString, name, newTag, size))
            next = self.getNextFile()

    def reset(self):
        """
        Reset the current position in the filecontainer to the beginning.
        """
        assert(not self.mutable)
        self.next = self.contentsStart

    def __del__(self):
	if self.file:
	    self.close()

    def __init__(self, file, version = None):
        """
        Create a FileContainer object.

        @param file: an open python file object referencing the file
        container file on disk. If that file is empty (size 0) the
        file container is immediately initialized. A copy of the file
        is retained, so the caller may optionally close it.
        """

	# make our own copy of this file which nobody can close underneath us
	self.file = file

        if version is None:
            version = FILE_CONTAINER_VERSION_FILEID_IDX

	self.file.seek(0, SEEK_END)
	if not self.file.tell():
	    self.file.seek(SEEK_SET, 0)
	    self.file.truncate()
	    self.file.write(FILE_CONTAINER_MAGIC)
	    self.file.write(struct.pack("!I", version))

	    self.mutable = True
	else:
            # we don't need to put this file pointer back; we don't depend
            # on it here at all; everything is through pseek
	    try:
		self.readHeader()
	    except:
		self.file.close()
		self.file = None
		raise
	    self.mutable = False

class BadContainer(conary.errors.ConaryError):

    pass
<|endoftext|>