<commit_before>from __future__ import unicode_literals
from __future__ import absolute_import
from .packages.docker.client import APIError
import logging
import re
import os
import sys
from .container import Container

log = logging.getLogger(__name__)

class BuildError(Exception):
    pass

class CannotBeScaledError(Exception):
    pass

class Service(object):
    def __init__(self, name, client=None, project='default', links=[], **options):
        if not re.match('^[a-zA-Z0-9]+$', name):
            raise ValueError('Invalid name: %s' % name)
        if not re.match('^[a-zA-Z0-9]+$', project):
            raise ValueError('Invalid project: %s' % project)
        if 'image' in options and 'build' in options:
            raise ValueError('Service %s has both an image and build path specified. A service can either be built to image or use an existing image, not both.' % name)

        self.name = name
        self.client = client
        self.project = project
        self.links = links or []
        self.options = options

    def containers(self, stopped=False, one_off=False):
        l = []
        for container in self.client.containers(all=stopped):
            name = get_container_name(container)
            if not name or not is_valid_name(name, one_off):
                continue
            project, name, number = parse_name(name)
            if project == self.project and name == self.name:
                l.append(Container.from_ps(self.client, container))
        return l

    def start(self, **options):
        for c in self.containers(stopped=True):
            if not c.is_running:
                log.info("Starting %s..." % c.name)
                self.start_container(c, **options)

    def stop(self, **options):
        for c in self.containers():
            log.info("Stopping %s..." % c.name)
            c.stop(**options)

    def kill(self, **options):
        for c in self.containers():
            log.info("Killing %s..." % c.name)
            c.kill(**options)

    def scale(self, desired_num):
        if not self.can_be_scaled():
            raise CannotBeScaledError()

        # Create enough containers
        containers = self.containers(stopped=True)
        while len(containers) < desired_num:
            containers.append(self.create_container())

        running_containers = []
        stopped_containers = []
        for c in containers:
            if c.is_running:
                running_containers.append(c)
            else:
                stopped_containers.append(c)
        running_containers.sort(key=lambda c: c.number)
        stopped_containers.sort(key=lambda c: c.number)

        # Stop containers
        while len(running_containers) > desired_num:
            c = running_containers.pop()
            log.info("Stopping %s..." % c.name)
            c.stop(timeout=1)
            stopped_containers.append(c)

        # Start containers
        while len(running_containers) < desired_num:
            c = stopped_containers.pop(0)
            log.info("Starting %s..." % c.name)
            c.start()
            running_containers.append(c)

    def remove_stopped(self, **options):
        for c in self.containers(stopped=True):
            if not c.is_running:
                log.info("Removing %s..." % c.name)
                c.remove(**options)

    def create_container(self, one_off=False, **override_options):
        """
        Create a container for this service. If the image doesn't exist, attempt to pull
        it.
        """
        container_options = self._get_container_options(override_options, one_off=one_off)
        try:
            return Container.create(self.client, **container_options)
        except APIError as e:
            if e.response.status_code == 404 and e.explanation and 'No such image' in str(e.explanation):
                log.info('Pulling image %s...' % container_options['image'])
                self.client.pull(container_options['image'])
                return Container.create(self.client, **container_options)
            raise

    def recreate_containers(self, **override_options):
        """
        If a container for this service doesn't exist, create one. If there are
        any, stop them and create new ones. Does not remove the old containers.
        """
        containers = self.containers(stopped=True)

        if len(containers) == 0:
            log.info("Creating %s..." % self.next_container_name())
            return ([], [self.create_container(**override_options)])
        else:
            old_containers = []
            new_containers = []

            for c in containers:
                log.info("Recreating %s..." % c.name)
                (old_container, new_container) = self.recreate_container(c, **override_options)
                old_containers.append(old_container)
                new_containers.append(new_container)

            return (old_containers, new_containers)

    def recreate_container(self, container, **override_options):
        if container.is_running:
            container.stop(timeout=1)

        intermediate_container = Container.create(
            self.client,
            image='ubuntu',
            command='echo',
            volumes_from=container.id,
        )
        intermediate_container.start()
        intermediate_container.wait()
        container.remove()

        options = dict(override_options)
        options['volumes_from'] = intermediate_container.id
        new_container = self.create_container(**options)

        return (intermediate_container, new_container)

    def start_container(self, container=None, **override_options):
        if container is None:
            container = self.create_container(**override_options)

        options = self.options.copy()
        options.update(override_options)

        port_bindings = {}

        if options.get('ports', None) is not None:
            for port in options['ports']:
                port = str(port)
                if ':' in port:
                    external_port, internal_port = port.split(':', 1)
                    port_bindings[int(internal_port)] = int(external_port)
                else:
                    port_bindings[int(port)] = None

        volume_bindings = {}

        if options.get('volumes', None) is not None:
            for volume in options['volumes']:
                if ':' in volume:
                    external_dir, internal_dir = volume.split(':')
                    volume_bindings[os.path.abspath(external_dir)] = internal_dir

        container.start(
            links=self._get_links(),
            port_bindings=port_bindings,
            binds=volume_bindings,
        )
        return container

    def next_container_name(self, one_off=False):
        bits = [self.project, self.name]
        if one_off:
            bits.append('run')
        return '_'.join(bits + [str(self.next_container_number(one_off=one_off))])

    def next_container_number(self, one_off=False):
        numbers = [parse_name(c.name)[2] for c in self.containers(stopped=True, one_off=one_off)]

        if len(numbers) == 0:
            return 1
        else:
            return max(numbers) + 1

    def _get_links(self):
        links = {}
        for service in self.links:
            for container in service.containers():
                links[container.name] = container.name
        return links

    def _get_container_options(self, override_options, one_off=False):
        keys = ['image', 'command', 'hostname', 'user', 'detach', 'stdin_open', 'tty', 'mem_limit', 'ports', 'environment', 'dns', 'volumes', 'volumes_from']
        container_options = dict((k, self.options[k]) for k in keys if k in self.options)
        container_options.update(override_options)

        container_options['name'] = self.next_container_name(one_off)

        if 'ports' in container_options:
            ports = []
            for port in container_options['ports']:
                port = str(port)
                if ':' in port:
                    port = port.split(':')[-1]
                ports.append(port)
            container_options['ports'] = ports

        if 'volumes' in container_options:
            container_options['volumes'] = dict((split_volume(v)[1], {}) for v in container_options['volumes'])

        if self.can_be_built():
            if len(self.client.images(name=self._build_tag_name())) == 0:
                self.build()
            container_options['image'] = self._build_tag_name()

        return container_options

    def build(self):
        log.info('Building %s...' % self.name)

        build_output = self.client.build(
            self.options['build'],
            tag=self._build_tag_name(),
            stream=True
        )

        image_id = None

        for line in build_output:
            if line:
                match = re.search(r'Successfully built ([0-9a-f]+)', line)
                if match:
                    image_id = match.group(1)
            sys.stdout.write(line)

        if image_id is None:
            raise BuildError()

        return image_id

    def can_be_built(self):
        return 'build' in self.options

    def _build_tag_name(self):
        """
        The tag to give to images built for this service.
        """
        return '%s_%s' % (self.project, self.name)

    def can_be_scaled(self):
        for port in self.options.get('ports', []):
            if ':' in str(port):
                return False
        return True

NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$')

def is_valid_name(name, one_off=False):
    match = NAME_RE.match(name)
    if match is None:
        return False
    if one_off:
        return match.group(3) == 'run_'
    else:
        return match.group(3) is None

def parse_name(name, one_off=False):
    match = NAME_RE.match(name)
    (project, service_name, _, suffix) = match.groups()
    return (project, service_name, int(suffix))

def get_container_name(container):
    if not container.get('Name') and not container.get('Names'):
        return None
    # inspect
    if 'Name' in container:
        return container['Name']
    # ps
    for name in container['Names']:
        if len(name.split('/')) == 2:
            return name[1:]

def split_volume(v):
    """
    If v is of the format EXTERNAL:INTERNAL, returns (EXTERNAL, INTERNAL).
    If v is of the format INTERNAL, returns (None, INTERNAL).
    """
    if ':' in v:
        return v.split(':', 1)
    else:
        return (None, v)
<commit_msg>Fix external port config<commit_after>from __future__ import unicode_literals
from __future__ import absolute_import
from .packages.docker.client import APIError
import logging
import re
import os
import sys
from .container import Container

log = logging.getLogger(__name__)

class BuildError(Exception):
    pass

class CannotBeScaledError(Exception):
    pass

class Service(object):
    def __init__(self, name, client=None, project='default', links=[], **options):
        if not re.match('^[a-zA-Z0-9]+$', name):
            raise ValueError('Invalid name: %s' % name)
        if not re.match('^[a-zA-Z0-9]+$', project):
            raise ValueError('Invalid project: %s' % project)
        if 'image' in options and 'build' in options:
            raise ValueError('Service %s has both an image and build path specified. A service can either be built to image or use an existing image, not both.' % name)

        self.name = name
        self.client = client
        self.project = project
        self.links = links or []
        self.options = options

    def containers(self, stopped=False, one_off=False):
        l = []
        for container in self.client.containers(all=stopped):
            name = get_container_name(container)
            if not name or not is_valid_name(name, one_off):
                continue
            project, name, number = parse_name(name)
            if project == self.project and name == self.name:
                l.append(Container.from_ps(self.client, container))
        return l

    def start(self, **options):
        for c in self.containers(stopped=True):
            if not c.is_running:
                log.info("Starting %s..." % c.name)
                self.start_container(c, **options)

    def stop(self, **options):
        for c in self.containers():
            log.info("Stopping %s..." % c.name)
            c.stop(**options)

    def kill(self, **options):
        for c in self.containers():
            log.info("Killing %s..." % c.name)
            c.kill(**options)

    def scale(self, desired_num):
        if not self.can_be_scaled():
            raise CannotBeScaledError()

        # Create enough containers
        containers = self.containers(stopped=True)
        while len(containers) < desired_num:
            containers.append(self.create_container())

        running_containers = []
        stopped_containers = []
        for c in containers:
            if c.is_running:
                running_containers.append(c)
            else:
                stopped_containers.append(c)
        running_containers.sort(key=lambda c: c.number)
        stopped_containers.sort(key=lambda c: c.number)

        # Stop containers
        while len(running_containers) > desired_num:
            c = running_containers.pop()
            log.info("Stopping %s..." % c.name)
            c.stop(timeout=1)
            stopped_containers.append(c)

        # Start containers
        while len(running_containers) < desired_num:
            c = stopped_containers.pop(0)
            log.info("Starting %s..." % c.name)
            c.start()
            running_containers.append(c)

    def remove_stopped(self, **options):
        for c in self.containers(stopped=True):
            if not c.is_running:
                log.info("Removing %s..." % c.name)
                c.remove(**options)

    def create_container(self, one_off=False, **override_options):
        """
        Create a container for this service. If the image doesn't exist, attempt to pull
        it.
        """
        container_options = self._get_container_options(override_options, one_off=one_off)
        try:
            return Container.create(self.client, **container_options)
        except APIError as e:
            if e.response.status_code == 404 and e.explanation and 'No such image' in str(e.explanation):
                log.info('Pulling image %s...' % container_options['image'])
                self.client.pull(container_options['image'])
                return Container.create(self.client, **container_options)
            raise

    def recreate_containers(self, **override_options):
        """
        If a container for this service doesn't exist, create one. If there are
        any, stop them and create new ones. Does not remove the old containers.
        """
        containers = self.containers(stopped=True)

        if len(containers) == 0:
            log.info("Creating %s..." % self.next_container_name())
            return ([], [self.create_container(**override_options)])
        else:
            old_containers = []
            new_containers = []

            for c in containers:
                log.info("Recreating %s..." % c.name)
                (old_container, new_container) = self.recreate_container(c, **override_options)
                old_containers.append(old_container)
                new_containers.append(new_container)

            return (old_containers, new_containers)

    def recreate_container(self, container, **override_options):
        if container.is_running:
            container.stop(timeout=1)

        intermediate_container = Container.create(
            self.client,
            image='ubuntu',
            command='echo',
            volumes_from=container.id,
        )
        intermediate_container.start()
        intermediate_container.wait()
        container.remove()

        options = dict(override_options)
        options['volumes_from'] = intermediate_container.id
        new_container = self.create_container(**options)

        return (intermediate_container, new_container)

    def start_container(self, container=None, **override_options):
        if container is None:
            container = self.create_container(**override_options)

        options = self.options.copy()
        options.update(override_options)

        port_bindings = {}

        if options.get('ports', None) is not None:
            for port in options['ports']:
                port = str(port)
                if ':' in port:
                    external_port, internal_port = port.split(':', 1)
                    port_bindings[int(internal_port)] = int(external_port)
                else:
                    port_bindings[int(port)] = None

        volume_bindings = {}

        if options.get('volumes', None) is not None:
            for volume in options['volumes']:
                if ':' in volume:
                    external_dir, internal_dir = volume.split(':')
                    volume_bindings[os.path.abspath(external_dir)] = internal_dir

        container.start(
            links=self._get_links(),
            port_bindings=port_bindings,
            binds=volume_bindings,
        )
        return container

    def next_container_name(self, one_off=False):
        bits = [self.project, self.name]
        if one_off:
            bits.append('run')
        return '_'.join(bits + [str(self.next_container_number(one_off=one_off))])

    def next_container_number(self, one_off=False):
        numbers = [parse_name(c.name)[2] for c in self.containers(stopped=True, one_off=one_off)]

        if len(numbers) == 0:
            return 1
        else:
            return max(numbers) + 1

    def _get_links(self):
        links = {}
        for service in self.links:
            for container in service.containers():
                links[container.name] = container.name
        return links

    def _get_container_options(self, override_options, one_off=False):
        keys = ['image', 'command', 'hostname', 'user', 'detach', 'stdin_open', 'tty', 'mem_limit', 'ports', 'environment', 'dns', 'volumes', 'volumes_from']
        container_options = dict((k, self.options[k]) for k in keys if k in self.options)
        container_options.update(override_options)

        container_options['name'] = self.next_container_name(one_off)

        if 'ports' in container_options:
            ports = []
            for port in container_options['ports']:
                port = str(port)
                if ':' in port:
                    port = port.split(':')[-1]
                if '/' not in port:
                    port = "%s/tcp" % port
                ports.append(port)
            container_options['ports'] = ports

        if 'volumes' in container_options:
            container_options['volumes'] = dict((split_volume(v)[1], {}) for v in container_options['volumes'])

        if self.can_be_built():
            if len(self.client.images(name=self._build_tag_name())) == 0:
                self.build()
            container_options['image'] = self._build_tag_name()

        return container_options

    def build(self):
        log.info('Building %s...' % self.name)

        build_output = self.client.build(
            self.options['build'],
            tag=self._build_tag_name(),
            stream=True
        )

        image_id = None

        for line in build_output:
            if line:
                match = re.search(r'Successfully built ([0-9a-f]+)', line)
                if match:
                    image_id = match.group(1)
            sys.stdout.write(line)

        if image_id is None:
            raise BuildError()

        return image_id

    def can_be_built(self):
        return 'build' in self.options

    def _build_tag_name(self):
        """
        The tag to give to images built for this service.
        """
        return '%s_%s' % (self.project, self.name)

    def can_be_scaled(self):
        for port in self.options.get('ports', []):
            if ':' in str(port):
                return False
        return True

NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$')

def is_valid_name(name, one_off=False):
    match = NAME_RE.match(name)
    if match is None:
        return False
    if one_off:
        return match.group(3) == 'run_'
    else:
        return match.group(3) is None

def parse_name(name, one_off=False):
    match = NAME_RE.match(name)
    (project, service_name, _, suffix) = match.groups()
    return (project, service_name, int(suffix))

def get_container_name(container):
    if not container.get('Name') and not container.get('Names'):
        return None
    # inspect
    if 'Name' in container:
        return container['Name']
    # ps
    for name in container['Names']:
        if len(name.split('/')) == 2:
            return name[1:]

def split_volume(v):
    """
    If v is of the format EXTERNAL:INTERNAL, returns (EXTERNAL, INTERNAL).
    If v is of the format INTERNAL, returns (None, INTERNAL).
    """
    if ':' in v:
        return v.split(':', 1)
    else:
        return (None, v)
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from __future__ import absolute_import
from .packages.docker.errors import APIError
import logging
import re
import os
import sys
from .container import Container
from .progress_stream import stream_output, StreamOutputError

log = logging.getLogger(__name__)

DOCKER_CONFIG_KEYS = ['image', 'command', 'hostname', 'user', 'detach', 'stdin_open', 'tty', 'mem_limit', 'ports', 'environment', 'dns', 'volumes', 'entrypoint', 'privileged', 'volumes_from', 'net', 'working_dir']
DOCKER_CONFIG_HINTS = {
    'link'      : 'links',
    'port'      : 'ports',
    'privilege' : 'privileged',
    'priviliged': 'privileged',
    'privilige' : 'privileged',
    'volume'    : 'volumes',
    'workdir'   : 'working_dir',
}

VALID_NAME_CHARS = '[a-zA-Z0-9]'

class BuildError(Exception):
    def __init__(self, service, reason):
        self.service = service
        self.reason = reason

class CannotBeScaledError(Exception):
    pass

class ConfigError(ValueError):
    pass

class Service(object):
    def __init__(self, name, client=None, project='default', links=[], volumes_from=[], **options):
        if not re.match('^%s+$' % VALID_NAME_CHARS, name):
            raise ConfigError('Invalid service name "%s" - only %s are allowed' % (name, VALID_NAME_CHARS))
        if not re.match('^%s+$' % VALID_NAME_CHARS, project):
            raise ConfigError('Invalid project name "%s" - only %s are allowed' % (project, VALID_NAME_CHARS))
        if 'image' in options and 'build' in options:
            raise ConfigError('Service %s has both an image and build path specified. A service can either be built to image or use an existing image, not both.' % name)

        supported_options = DOCKER_CONFIG_KEYS + ['build', 'expose']

        for k in options:
            if k not in supported_options:
                msg = "Unsupported config option for %s service: '%s'" % (name, k)
                if k in DOCKER_CONFIG_HINTS:
                    msg += " (did you mean '%s'?)" % DOCKER_CONFIG_HINTS[k]
                raise ConfigError(msg)

        self.name = name
        self.client = client
        self.project = project
        self.links = links or []
        self.volumes_from = volumes_from or []
        self.options = options

    def containers(self, stopped=False, one_off=False):
        l = []
        for container in self.client.containers(all=stopped):
            name = get_container_name(container)
            if not name or not is_valid_name(name, one_off):
                continue
            project, name, number = parse_name(name)
            if project == self.project and name == self.name:
                l.append(Container.from_ps(self.client, container))
        return l

    def start(self, **options):
        for c in self.containers(stopped=True):
            self.start_container_if_stopped(c, **options)

    def stop(self, **options):
        for c in self.containers():
            log.info("Stopping %s..." % c.name)
            c.stop(**options)

    def kill(self, **options):
        for c in self.containers():
            log.info("Killing %s..." % c.name)
            c.kill(**options)

    def scale(self, desired_num):
        """
        Adjusts the number of containers to the specified number and ensures they are running.

        - creates containers until there are at least `desired_num`
        - stops containers until there are at most `desired_num` running
        - starts containers until there are at least `desired_num` running
        - removes all stopped containers
        """
        if not self.can_be_scaled():
            raise CannotBeScaledError()

        # Create enough containers
        containers = self.containers(stopped=True)
        while len(containers) < desired_num:
            containers.append(self.create_container())

        running_containers = []
        stopped_containers = []
        for c in containers:
            if c.is_running:
                running_containers.append(c)
            else:
                stopped_containers.append(c)
        running_containers.sort(key=lambda c: c.number)
        stopped_containers.sort(key=lambda c: c.number)

        # Stop containers
        while len(running_containers) > desired_num:
            c = running_containers.pop()
            log.info("Stopping %s..." % c.name)
            c.stop(timeout=1)
            stopped_containers.append(c)

        # Start containers
        while len(running_containers) < desired_num:
            c = stopped_containers.pop(0)
            log.info("Starting %s..." % c.name)
            self.start_container(c)
            running_containers.append(c)

        self.remove_stopped()

    def remove_stopped(self, **options):
        for c in self.containers(stopped=True):
            if not c.is_running:
                log.info("Removing %s..." % c.name)
                c.remove(**options)

    def create_container(self, one_off=False, **override_options):
        """
        Create a container for this service. If the image doesn't exist, attempt to pull
        it.
        """
        container_options = self._get_container_create_options(override_options, one_off=one_off)
        try:
            return Container.create(self.client, **container_options)
        except APIError as e:
            if e.response.status_code == 404 and e.explanation and 'No such image' in str(e.explanation):
                log.info('Pulling image %s...' % container_options['image'])
                output = self.client.pull(container_options['image'], stream=True)
                stream_output(output, sys.stdout)
                return Container.create(self.client, **container_options)
            raise

    def recreate_containers(self, **override_options):
        """
        If a container for this service doesn't exist, create and start one. If there are
        any, stop them, create+start new ones, and remove the old containers.
        """
        containers = self.containers(stopped=True)

        if len(containers) == 0:
            log.info("Creating %s..." % self.next_container_name())
            container = self.create_container(**override_options)
            self.start_container(container)
            return [(None, container)]
        else:
            tuples = []

            for c in containers:
                log.info("Recreating %s..." % c.name)
                tuples.append(self.recreate_container(c, **override_options))

            return tuples

    def recreate_container(self, container, **override_options):
        if container.is_running:
            container.stop(timeout=1)

        intermediate_container = Container.create(
            self.client,
            image=container.image,
            entrypoint=['echo'],
            command=[],
        )
        intermediate_container.start(volumes_from=container.id)
        intermediate_container.wait()
        container.remove()

        options = dict(override_options)
        new_container = self.create_container(**options)
        self.start_container(new_container, intermediate_container=intermediate_container)

        intermediate_container.remove()

        return (intermediate_container, new_container)

    def start_container_if_stopped(self, container, **options):
        if container.is_running:
            return container
        else:
            log.info("Starting %s..." % container.name)
            return self.start_container(container, **options)

    def start_container(self, container=None, intermediate_container=None,**override_options):
        if container is None:
            container = self.create_container(**override_options)

        options = self.options.copy()
        options.update(override_options)

        port_bindings = {}

        if options.get('ports', None) is not None:
            for port in options['ports']:
                internal_port, external_port = split_port(port)
                port_bindings[internal_port] = external_port

        volume_bindings = {}

        if options.get('volumes', None) is not None:
            for volume in options['volumes']:
                if ':' in volume:
                    external_dir, internal_dir = volume.split(':')
                    volume_bindings[os.path.abspath(external_dir)] = {
                        'bind': internal_dir,
                        'ro': False,
                    }

        privileged = options.get('privileged', False)
        net = options.get('net', 'bridge')

        container.start(
            links=self._get_links(link_to_self=override_options.get('one_off', False)),
            port_bindings=port_bindings,
            binds=volume_bindings,
            volumes_from=self._get_volumes_from(intermediate_container),
            privileged=privileged,
            network_mode=net,
        )
        return container

    def start_or_create_containers(self):
        containers = self.containers(stopped=True)

        if len(containers) == 0:
            log.info("Creating %s..." % self.next_container_name())
            new_container = self.create_container()
            return [self.start_container(new_container)]
        else:
            return [self.start_container_if_stopped(c) for c in containers]

    def get_linked_names(self):
        return [s.name for (s, _) in self.links]

    def next_container_name(self, one_off=False):
        bits = [self.project, self.name]
        if one_off:
            bits.append('run')
        return '_'.join(bits + [str(self.next_container_number(one_off=one_off))])

    def next_container_number(self, one_off=False):
        numbers = [parse_name(c.name)[2] for c in self.containers(stopped=True, one_off=one_off)]

        if len(numbers) == 0:
            return 1
        else:
            return max(numbers) + 1

    def _get_links(self, link_to_self):
        links = []
        for service, link_name in self.links:
            for container in service.containers():
                if link_name:
                    links.append((container.name, link_name))
                links.append((container.name, container.name))
                links.append((container.name, container.name_without_project))
        if link_to_self:
            for container in self.containers():
                links.append((container.name, container.name))
                links.append((container.name, container.name_without_project))
        return links

    def _get_volumes_from(self, intermediate_container=None):
        volumes_from = []
        for v in self.volumes_from:
            if isinstance(v, Service):
                for container in v.containers(stopped=True):
                    volumes_from.append(container.id)
            elif isinstance(v, Container):
                volumes_from.append(v.id)

        if intermediate_container:
            volumes_from.append(intermediate_container.id)

        return volumes_from

    def _get_container_create_options(self, override_options, one_off=False):
        container_options = dict((k, self.options[k]) for k in DOCKER_CONFIG_KEYS if k in self.options)
        container_options.update(override_options)

        container_options['name'] = self.next_container_name(one_off)

        if 'ports' in container_options or 'expose' in self.options:
            ports = []
            all_ports = container_options.get('ports', []) + self.options.get('expose', [])
            for port in all_ports:
                port = str(port)
                if ':' in port:
                    port = port.split(':')[-1]
                if '/' in port:
                    port = tuple(port.split('/'))
                ports.append(port)
            container_options['ports'] = ports

        if 'volumes' in container_options:
            container_options['volumes'] = dict((split_volume(v)[1], {}) for v in container_options['volumes'])

        if self.can_be_built():
            if len(self.client.images(name=self._build_tag_name())) == 0:
                self.build()
            container_options['image'] = self._build_tag_name()

        # Priviliged is only required for starting containers, not for creating them
        if 'privileged' in container_options:
            del container_options['privileged']

        # net is only required for starting containers, not for creating them
        if 'net' in container_options:
            del container_options['net']

        return container_options

    def build(self):
        log.info('Building %s...' % self.name)

        build_output = self.client.build(
            self.options['build'],
            tag=self._build_tag_name(),
            stream=True,
            rm=True
        )

        try:
            all_events = stream_output(build_output, sys.stdout)
        except StreamOutputError, e:
            raise BuildError(self, unicode(e))

        image_id = None

        for event in all_events:
            if 'stream' in event:
                match = re.search(r'Successfully built ([0-9a-f]+)', event.get('stream', ''))
                if match:
                    image_id = match.group(1)

        if image_id is None:
            raise BuildError(self)

        return image_id

    def can_be_built(self):
        return 'build' in self.options

    def _build_tag_name(self):
        """
        The tag to give to images built for this service.
        """
        return '%s_%s' % (self.project, self.name)

    def can_be_scaled(self):
        for port in self.options.get('ports', []):
            if ':' in str(port):
                return False
        return True

NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$')

def is_valid_name(name, one_off=False):
    match = NAME_RE.match(name)
    if match is None:
        return False
    if one_off:
        return match.group(3) == 'run_'
    else:
        return match.group(3) is None

def parse_name(name, one_off=False):
    match = NAME_RE.match(name)
    (project, service_name, _, suffix) = match.groups()
    return (project, service_name, int(suffix))

def get_container_name(container):
    if not container.get('Name') and not container.get('Names'):
        return None
    # inspect
    if 'Name' in container:
        return container['Name']
    # ps
    for name in container['Names']:
        if len(name.split('/')) == 2:
            return name[1:]

def split_volume(v):
    """
    If v is of the format EXTERNAL:INTERNAL, returns (EXTERNAL, INTERNAL).
    If v is of the format INTERNAL, returns (None, INTERNAL).
    """
    if ':' in v:
        return v.split(':', 1)
    else:
        return (None, v)

def split_port(port):
    port = str(port)
    external_ip = None
    if ':' in port:
        external_port, internal_port = port.rsplit(':', 1)
        if ':' in external_port:
            external_ip, external_port = external_port.split(':', 1)
    else:
        external_port, internal_port = (None, port)
    if external_ip:
        if external_port:
            external_port = (external_ip, external_port)
        else:
            external_port = (external_ip,)
    return internal_port, external_port

<commit_msg>Add domainname to allowed container config.<commit_after>from __future__ import unicode_literals
from __future__ import absolute_import
from .packages.docker.errors import APIError
import logging
import re
import os
import sys
from .container import Container
from .progress_stream import stream_output, StreamOutputError

log = logging.getLogger(__name__)

DOCKER_CONFIG_KEYS = ['image', 'command', 'hostname', 'domainname', 'user', 'detach', 'stdin_open', 'tty', 'mem_limit', 'ports', 'environment', 'dns', 'volumes', 'entrypoint', 'privileged', 'volumes_from', 'net', 'working_dir']
DOCKER_CONFIG_HINTS = {
    'link'      : 'links',
    'port'      : 'ports',
    'privilege' : 'privileged',
    'priviliged': 'privileged',
    'privilige' : 'privileged',
    'volume'    : 'volumes',
    'workdir'   : 'working_dir',
}

VALID_NAME_CHARS = '[a-zA-Z0-9]'

class BuildError(Exception):
    def __init__(self, service, reason):
        self.service = service
        self.reason = reason

class CannotBeScaledError(Exception):
    pass

class ConfigError(ValueError):
    pass

class Service(object):
    def __init__(self, name, client=None, project='default', links=[], volumes_from=[], **options):
        if not re.match('^%s+$' % VALID_NAME_CHARS, name):
            raise ConfigError('Invalid service name "%s" - only %s are allowed' % (name, VALID_NAME_CHARS))
        if not re.match('^%s+$' % VALID_NAME_CHARS, project):
            raise ConfigError('Invalid project name "%s" - only %s are allowed' % (project, VALID_NAME_CHARS))
        if 'image' in options and 'build' in options:
            raise ConfigError('Service %s has both an image and build path specified. A service can either be built to image or use an existing image, not both.' % name)

        supported_options = DOCKER_CONFIG_KEYS + ['build', 'expose']

        for k in options:
            if k not in supported_options:
                msg = "Unsupported config option for %s service: '%s'" % (name, k)
                if k in DOCKER_CONFIG_HINTS:
                    msg += " (did you mean '%s'?)" % DOCKER_CONFIG_HINTS[k]
                raise ConfigError(msg)

        self.name = name
        self.client = client
        self.project = project
        self.links = links or []
        self.volumes_from = volumes_from or []
        self.options = options

    def containers(self, stopped=False, one_off=False):
        l = []
        for container in self.client.containers(all=stopped):
            name = get_container_name(container)
            if not name or not is_valid_name(name, one_off):
                continue
            project, name, number = parse_name(name)
            if project == self.project and name == self.name:
                l.append(Container.from_ps(self.client, container))
        return l

    def start(self, **options):
        for c in self.containers(stopped=True):
            self.start_container_if_stopped(c, **options)

    def stop(self, **options):
        for c in self.containers():
            log.info("Stopping %s..." % c.name)
            c.stop(**options)

    def kill(self, **options):
        for c in self.containers():
            log.info("Killing %s..." % c.name)
            c.kill(**options)

    def scale(self, desired_num):
        """
        Adjusts the number of containers to the specified number and ensures they are running.

        - creates containers until there are at least `desired_num`
        - stops containers until there are at most `desired_num` running
        - starts containers until there are at least `desired_num` running
        - removes all stopped containers
        """
        if not self.can_be_scaled():
            raise CannotBeScaledError()

        # Create enough containers
        containers = self.containers(stopped=True)
        while len(containers) < desired_num:
            containers.append(self.create_container())

        running_containers = []
        stopped_containers = []
        for c in containers:
            if c.is_running:
                running_containers.append(c)
            else:
                stopped_containers.append(c)
        running_containers.sort(key=lambda c: c.number)
        stopped_containers.sort(key=lambda c: c.number)

        # Stop containers
        while len(running_containers) > desired_num:
            c = running_containers.pop()
            log.info("Stopping %s..." % c.name)
            c.stop(timeout=1)
            stopped_containers.append(c)

        # Start containers
        while len(running_containers) < desired_num:
            c = stopped_containers.pop(0)
            log.info("Starting %s..." % c.name)
            self.start_container(c)
            running_containers.append(c)

        self.remove_stopped()

    def remove_stopped(self, **options):
        for c in self.containers(stopped=True):
            if not c.is_running:
                log.info("Removing %s..." % c.name)
                c.remove(**options)

    def create_container(self, one_off=False, **override_options):
        """
        Create a container for this service. If the image doesn't exist, attempt to pull
        it.
        """
        container_options = self._get_container_create_options(override_options, one_off=one_off)
        try:
            return Container.create(self.client, **container_options)
        except APIError as e:
            if e.response.status_code == 404 and e.explanation and 'No such image' in str(e.explanation):
                log.info('Pulling image %s...' % container_options['image'])
                output = self.client.pull(container_options['image'], stream=True)
                stream_output(output, sys.stdout)
                return Container.create(self.client, **container_options)
            raise

    def recreate_containers(self, **override_options):
        """
        If a container for this service doesn't exist, create and start one. If there are
        any, stop them, create+start new ones, and remove the old containers.
        """
        containers = self.containers(stopped=True)

        if len(containers) == 0:
            log.info("Creating %s..." % self.next_container_name())
            container = self.create_container(**override_options)
            self.start_container(container)
            return [(None, container)]
        else:
            tuples = []

            for c in containers:
                log.info("Recreating %s..." % c.name)
                tuples.append(self.recreate_container(c, **override_options))

            return tuples

    def recreate_container(self, container, **override_options):
        if container.is_running:
            container.stop(timeout=1)

        intermediate_container = Container.create(
            self.client,
            image=container.image,
            entrypoint=['echo'],
            command=[],
        )
        intermediate_container.start(volumes_from=container.id)
        intermediate_container.wait()
        container.remove()

        options = dict(override_options)
        new_container = self.create_container(**options)
        self.start_container(new_container, intermediate_container=intermediate_container)

        intermediate_container.remove()

        return (intermediate_container, new_container)

    def start_container_if_stopped(self, container, **options):
        if container.is_running:
            return container
        else:
            log.info("Starting %s..." % container.name)
            return self.start_container(container, **options)

    def start_container(self, container=None, intermediate_container=None,**override_options):
        if container is None:
            container = self.create_container(**override_options)

        options = self.options.copy()
        options.update(override_options)

        port_bindings = {}

        if options.get('ports', None) is not None:
            for port in options['ports']:
                internal_port, external_port = split_port(port)
                port_bindings[internal_port] = external_port

        volume_bindings = {}

        if options.get('volumes', None) is not None:
            for volume in options['volumes']:
                if ':' in volume:
                    external_dir, internal_dir = volume.split(':')
                    volume_bindings[os.path.abspath(external_dir)] = {
                        'bind': internal_dir,
                        'ro': False,
                    }

        privileged = options.get('privileged', False)
        net = options.get('net', 'bridge')

        container.start(
            links=self._get_links(link_to_self=override_options.get('one_off', False)),
            port_bindings=port_bindings,
            binds=volume_bindings,
            volumes_from=self._get_volumes_from(intermediate_container),
            privileged=privileged,
            network_mode=net,
        )
        return container

    def start_or_create_containers(self):
        containers = self.containers(stopped=True)

        if len(containers) == 0:
            log.info("Creating %s..." % self.next_container_name())
            new_container = self.create_container()
            return [self.start_container(new_container)]
        else:
            return [self.start_container_if_stopped(c) for c in containers]

    def get_linked_names(self):
        return [s.name for (s, _) in self.links]

    def next_container_name(self, one_off=False):
        bits = [self.project, self.name]
        if one_off:
            bits.append('run')
        return '_'.join(bits + [str(self.next_container_number(one_off=one_off))])

    def next_container_number(self, one_off=False):
        numbers = [parse_name(c.name)[2] for c in self.containers(stopped=True, one_off=one_off)]

        if len(numbers) == 0:
            return 1
        else:
            return max(numbers) + 1

    def _get_links(self, link_to_self):
        links = []
        for service, link_name in self.links:
            for container in service.containers():
                if link_name:
                    links.append((container.name, link_name))
                links.append((container.name, container.name))
                links.append((container.name, container.name_without_project))
        if link_to_self:
            for container in self.containers():
                links.append((container.name, container.name))
                links.append((container.name, container.name_without_project))
        return links

    def _get_volumes_from(self, intermediate_container=None):
        volumes_from = []
        for v in self.volumes_from:
            if isinstance(v, Service):
                for container in v.containers(stopped=True):
                    volumes_from.append(container.id)
            elif isinstance(v, Container):
                volumes_from.append(v.id)

        if intermediate_container:
            volumes_from.append(intermediate_container.id)

        return volumes_from

    def _get_container_create_options(self, override_options, one_off=False):
        container_options = dict((k, self.options[k]) for k in DOCKER_CONFIG_KEYS if k in self.options)
        container_options.update(override_options)

        container_options['name'] = self.next_container_name(one_off)

        if 'ports' in container_options or 'expose' in self.options:
            ports = []
            all_ports = container_options.get('ports', []) + self.options.get('expose', [])
            for port in all_ports:
                port = str(port)
                if ':' in port:
                    port = port.split(':')[-1]
                if '/' in port:
                    port = tuple(port.split('/'))
                ports.append(port)
            container_options['ports'] = ports

        if 'volumes' in container_options:
            container_options['volumes'] = dict((split_volume(v)[1], {}) for v in container_options['volumes'])

        if self.can_be_built():
            if len(self.client.images(name=self._build_tag_name())) == 0:
                self.build()
            container_options['image'] = self._build_tag_name()

        # Priviliged is only required for starting containers, not for creating them
        if 'privileged' in container_options:
            del container_options['privileged']

        # net is only required for starting containers, not for creating them
        if 'net' in container_options:
            del container_options['net']

        return container_options

    def build(self):
        log.info('Building %s...' % self.name)

        build_output = self.client.build(
            self.options['build'],
            tag=self._build_tag_name(),
            stream=True,
            rm=True
        )

        try:
            all_events = stream_output(build_output, sys.stdout)
        except StreamOutputError, e:
            raise BuildError(self, unicode(e))

        image_id = None

        for event in all_events:
            if 'stream' in event:
                match = re.search(r'Successfully built ([0-9a-f]+)', event.get('stream', ''))
                if match:
                    image_id = match.group(1)

        if image_id is None:
            raise BuildError(self)

        return image_id

    def can_be_built(self):
        return 'build' in self.options

    def _build_tag_name(self):
        """
        The tag to give to images built for this service.
        """
        return '%s_%s' % (self.project, self.name)

    def can_be_scaled(self):
        for port in self.options.get('ports', []):
            if ':' in str(port):
                return False
        return True

NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$')

def is_valid_name(name, one_off=False):
    match = NAME_RE.match(name)
    if match is None:
        return False
    if one_off:
        return match.group(3) == 'run_'
    else:
        return match.group(3) is None

def parse_name(name, one_off=False):
    match = NAME_RE.match(name)
    (project, service_name, _, suffix) = match.groups()
    return (project, service_name, int(suffix))

def get_container_name(container):
    if not container.get('Name') and not container.get('Names'):
        return None
    # inspect
    if 'Name' in container:
        return container['Name']
    # ps
    for name in container['Names']:
        if len(name.split('/')) == 2:
            return name[1:]

def split_volume(v):
    """
    If v is of the format EXTERNAL:INTERNAL, returns (EXTERNAL, INTERNAL).
    If v is of the format INTERNAL, returns (None, INTERNAL).
    """
    if ':' in v:
        return v.split(':', 1)
    else:
        return (None, v)

def split_port(port):
    port = str(port)
    external_ip = None
    if ':' in port:
        external_port, internal_port = port.rsplit(':', 1)
        if ':' in external_port:
            external_ip, external_port = external_port.split(':', 1)
    else:
        external_port, internal_port = (None, port)
    if external_ip:
        if external_port:
            external_port = (external_ip, external_port)
        else:
            external_port = (external_ip,)
    return internal_port, external_port

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

class Fetcher(object):

    def __init__(self, queue):

        """
        Parameters
        ----------
        queue : Queue.Queue
        """

        self._queue = queue

    def pop(self, timeout):

        """
        Parameters
        ----------
        timeout : float

        Returns
        -------
        clare.application.messaging.client.records.Record
        """

        record = self._queue.get(timeout=timeout)
        return record

    def __repr__(self):
        repr_ = '{}(queue={})'
        return repr_.format(self.__class__.__name__, self._queue)
<commit_msg>ME-653: Changed Fetcher queue attribute name to "message_queue".<commit_after># -*- coding: utf-8 -*-

class Fetcher(object):

    def __init__(self, message_queue):

        """
        Parameters
        ----------
        message_queue : Queue.Queue
        """

        self._message_queue = message_queue

    def pop(self, timeout):

        """
        Parameters
        ----------
        timeout : float

        Returns
        -------
        clare.application.messaging.client.records.Record
        """

        record = self._message_queue.get(timeout=timeout)
        return record

    def __repr__(self):
        repr_ = '{}(message_queue={})'
        return repr_.format(self.__class__.__name__, self._message_queue)
<|endoftext|>


<commit_before># Copyright 2014, 2015 The ODL development group
#
# This file is part of ODL.
#
# ODL is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ODL is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with ODL.  If not, see <http://www.gnu.org/licenses/>.

"""Run all doctests."""

import nose
import sys

arg = sys.argv[:1]
arg.append('--verbosity=2')
arg.append('--with-doctest')
arg.append('--doctest-options=+NORMALIZE_WHITESPACE')
try:
    # pylint: disable=unused-import
    import odl.space.cuda
except ImportError:
    arg.append('--ignore-files=cuda.py')
out = nose.run(defaultTest='./odl/.', argv=arg)
<commit_msg>Added ellipsis support to docstring tests<commit_after># Copyright 2014, 2015 The ODL development group
#
# This file is part of ODL.
#
# ODL is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ODL is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with ODL.  If not, see <http://www.gnu.org/licenses/>.

"""Run all doctests."""

import nose
import sys

arg = sys.argv[:1]
arg.append('--verbosity=2')
arg.append('--with-doctest')
arg.append('--doctest-options=+NORMALIZE_WHITESPACE,+ELLIPSIS')
try:
    # pylint: disable=unused-import
    import odl.space.cuda
except ImportError:
    arg.append('--ignore-files=cuda.py')
out = nose.run(defaultTest='./odl/.', argv=arg)
<|endoftext|>


<commit_before>import random
from collections import namedtuple, defaultdict, deque

Point = namedtuple('Point', ['x', 'y'])
Size = namedtuple('Size', ['w', 'h'])

EMPTY = 0
BLACK = 1
WHITE = 2

BOARD_LETTERS = 'ABCDEFGHJKLMNOPQRST'

class Group:
    def __init__(self, color):
        self.color = color
        self.points = set()
        self.liberties = set()

    @property
    def is_alive(self):
        return len(self.liberties) > 0

    @property
    def is_dead(self):
        return len(self.liberties) == 0

    def __len__(self):
        return len(self.points)

    def __repr__(self):
        return '<group color={} {} points {} liberties>'.format(
            self.color, len(self.points), len(self.liberties))

class Board:
    def __init__(self, size):
        self.size = size
        self.stones = {}

    def random_fill(self, seed=None):
        rand = random.Random(seed)

        for point in self.iter_points():
            self.stones[point] = rand.choice([EMPTY, BLACK, WHITE])

    def is_inside(self, point):
        return 0 <= point.x < self.size.w and 0 <= point.y < self.size.h

    def get_color(self, point):
        return self.stones.get(point, 0)

    def get_neighbours(self, point):
        x, y = point

        points = [Point(x-1, y), Point(x+1, y), Point(x, y-1), Point(x, y+1)]
        points = [p for p in points if self.is_inside(p)]

        return points

    def iter_points(self):
        for x in range(self.size.w):
            for y in range(self.size.h):
                yield Point(x, y)

    def find_groups(self):
        groups = []
        grouped_points = set()

        for point, color in self.stones.items():
            # This should not happen but let's test just in case.
            if color == EMPTY:
                continue

            if point in grouped_points:
                continue

            group = Group(color)

            todo = [point]
            while todo:
                point = todo.pop()
                color = self.stones[point]

                if point in grouped_points:
                    continue
                elif color == EMPTY:
                    group.liberties.add(point)
                elif color == group.color:
                    group.points.add(point)
                    grouped_points.add(point)
                    todo.extend(self.get_neighbours(point))

            groups.append(group)

        return groups

    def print(self):
        color_chars = {
            # Characters that are easy to tell apart at a glance.
            EMPTY: '.',
            BLACK: '*',
            WHITE: '|',
        }

        print()
        print('    ', ' '.join(BOARD_LETTERS[:self.size.w]))
        print()

        for y in range(self.size.h):
            line = []
            for x in reversed(range(self.size.w)):
                line.append(color_chars[self.get_color(Point(x, y))])

            rownum = self.size.h - y

            print(' {:2} '.format(rownum), ' '.join(line))
        print()

def print_dead_groups(groups, board_size):
    board = Board(board_size)
    for group in groups:
        if group.is_dead:
            for point in group.points:
                board.stones[point] = group.color

    board.print()

board = Board(Size(19, 19))
board.random_fill(seed=None)

print('Board:')
board.print()

groups = board.find_groups()

print('Dead groups:')
print_dead_groups(groups, board.size)
<commit_msg>Changed color char for white in Python prototype.<commit_after>import random
from collections import namedtuple, defaultdict, deque

Point = namedtuple('Point', ['x', 'y'])
Size = namedtuple('Size', ['w', 'h'])

EMPTY = 0
BLACK = 1
WHITE = 2

BOARD_LETTERS = 'ABCDEFGHJKLMNOPQRST'

class Group:
    def __init__(self, color):
        self.color = color
        self.points = set()
        self.liberties = set()

    @property
    def is_alive(self):
        return len(self.liberties) > 0

    @property
    def is_dead(self):
        return len(self.liberties) == 0

    def __len__(self):
        return len(self.points)

    def __repr__(self):
        return '<group color={} {} points {} liberties>'.format(
            self.color, len(self.points), len(self.liberties))

class Board:
    def __init__(self, size):
        self.size = size
        self.stones = {}

    def random_fill(self, seed=None):
        rand = random.Random(seed)

        for point in self.iter_points():
            self.stones[point] = rand.choice([EMPTY, BLACK, WHITE])

    def is_inside(self, point):
        return 0 <= point.x < self.size.w and 0 <= point.y < self.size.h

    def get_color(self, point):
        return self.stones.get(point, 0)

    def get_neighbours(self, point):
        x, y = point

        points = [Point(x-1, y), Point(x+1, y), Point(x, y-1), Point(x, y+1)]
        points = [p for p in points if self.is_inside(p)]

        return points

    def iter_points(self):
        for x in range(self.size.w):
            for y in range(self.size.h):
                yield Point(x, y)

    def find_groups(self):
        groups = []
        grouped_points = set()

        for point, color in self.stones.items():
            # This should not happen but let's test just in case.
            if color == EMPTY:
                continue

            if point in grouped_points:
                continue

            group = Group(color)

            todo = [point]
            while todo:
                point = todo.pop()
                color = self.stones[point]

                if point in grouped_points:
                    continue
                elif color == EMPTY:
                    group.liberties.add(point)
                elif color == group.color:
                    group.points.add(point)
                    grouped_points.add(point)
                    todo.extend(self.get_neighbours(point))

            groups.append(group)

        return groups

    def print(self):
        color_chars = {
            # Characters that are easy to tell apart at a glance.
            EMPTY: '.',
            BLACK: '*',
            WHITE: 'o',
        }

        print()
        print('    ', ' '.join(BOARD_LETTERS[:self.size.w]))
        print()

        for y in range(self.size.h):
            line = []
            for x in reversed(range(self.size.w)):
                line.append(color_chars[self.get_color(Point(x, y))])

            rownum = self.size.h - y

            print(' {:2} '.format(rownum), ' '.join(line))
        print()

def print_dead_groups(groups, board_size):
    board = Board(board_size)
    for group in groups:
        if group.is_dead:
            for point in group.points:
                board.stones[point] = group.color

    board.print()

board = Board(Size(19, 19))
board.random_fill(seed=None)

print('Board:')
board.print()

groups = board.find_groups()

print('Dead groups:')
print_dead_groups(groups, board.size)
<|endoftext|>


<commit_before>from core import *
from command import *
<commit_msg>Force local import<commit_after>from .core import *
from .command import *
<|endoftext|>


<commit_before>import rospy
import yaml
import tf
import tf_conversions.posemath as pm

from predicator_msgs.srv import *
from predicator_msgs.msg import *

from librarian_msgs.srv import *
from librarian_msgs.msg import *

from costar_objrec_msgs.msg import *

from predicator_landmark import GetWaypointsService

'''
SmartWaypointManager
This class will create and load smart waypoints from the $LIBRARIAN_HOME/smart_waypoints directory
It also provides ways to use and access these methods
'''
class SmartWaypointManager:

    def __init__(self,world="world",ns="",endpoint="/endpoint"):
        self.get_waypoints_srv = GetWaypointsService(world=world,service=False)

        rospy.loginfo("[SmartMove] Waiting for LIBRARIAN to handle file I/O...")
        rospy.wait_for_service('/librarian/add_type',5)
        self.add_type_service = rospy.ServiceProxy('/librarian/add_type', librarian_msgs.srv.AddType)
        self.save_service = rospy.ServiceProxy('/librarian/save', librarian_msgs.srv.Save)
        self.load_service = rospy.ServiceProxy('/librarian/load', librarian_msgs.srv.Load)
        self.list_service = rospy.ServiceProxy('/librarian/list', librarian_msgs.srv.List)
        self.delete_service = rospy.ServiceProxy('/librarian/delete', librarian_msgs.srv.Delete)

        self.get_assignment_service = rospy.ServiceProxy('/predicator/get_assignment', predicator_msgs.srv.GetAssignment)

        #self.detected_objects = rospy.Subscriber(ns + '/detected_object_list', DetectedObjectList, self.detected_objects_cb)

        self.broadcaster = tf.TransformBroadcaster()
        self.listener = tf.TransformListener()
        self.world = world
        self.endpoint = endpoint

        self.folder = 'smartmove_waypoint'

        self.add_type_service(self.folder)

        self.waypoints = {}
        self.waypoint_names = {}

        self.all_moves = []

        self.objs = []
        self.obj_classes = []
        self.obj_class = {}

        self.add_type_service("smartmove_info")
        self.available_obj_classes = yaml.load(self.load_service(type="smartmove_info",id="obj_classes").text)
        print "Available classes = " + str(self.available_obj_classes)

    def detected_objects_cb(self,msg):
        self.objs = [obj.id for obj in msg.objects]
        self.obj_classes = [obj.object_class for obj in msg.objects]
        for (o,c) in zip(self.objs,self.obj_classes):
            self.obj_class[o] = c

        #print self.objs
        #print self.obj_classes

    '''
    get all waypoints from the disk
    '''
    def load_all(self):

        self.waypoints = {}
        self.all_moves = []

        waypoint_filenames = self.list_service(self.folder).entries

        for name in waypoint_filenames:
            data = yaml.load(self.load_service(id=name,type=self.folder).text)
            if not data[1] in self.waypoints.keys():
                self.waypoints[data[1]] = []
                self.waypoint_names[data[1]] = []

            self.waypoints[data[1]].append(data[0])
            self.waypoint_names[data[1]].append(name)
            self.all_moves.append(data[1] + "/" + name)

        print " === LOADING === "
        print self.waypoint_names
        print self.waypoints

    def get_all_moves(self):
        return self.all_moves

    def get_detected_objects(self):
        #print self.available_obj_classes
        self.objs = []
        for oc in self.available_obj_classes:
            resp = self.get_assignment_service(PredicateStatement(predicate=oc,params=["*","",""]))
            oc_objs = [p.params[0] for p in resp.values]
            for obj in oc_objs:
                self.obj_class[obj] = oc
            self.objs += oc_objs

        return self.objs

    def delete(self,move):
        self.delete_service(id=move.strip('/'),type=self.folder)

    def get_detected_object_classes(self):
        return self.obj_classes

    def save_new_waypoint(self,obj,name):
        pose = self.get_new_waypoint(obj)
        if not pose is None:
            self.save_service(id=name.strip('/'),type=self.folder,text=yaml.dump([pose,self.obj_class[obj]]))

    def get_new_waypoint(self,obj):
        try:
            (trans,rot) = self.listener.lookupTransform(obj,self.endpoint,rospy.Time(0))
            return pm.toMsg(pm.fromTf((trans,rot)))
        except (tf.LookupException, tf.ConnectivityException, tf.ExtrapolationException):
            rospy.logerr('Failed to lookup transform from %s to %s'%(obj,self.endpoint))
        return None

    def get_moves_for_object(self,obj):
        print "obj class = %s"%self.obj_class[obj]

        wpts = []
        try:
            wpts = self.waypoint_names[self.obj_class[obj]]
        except KeyError:
            wpts = []
        return wpts

    def update_tf(self):

        for key in self.waypoints.keys():
            (poses,names) = self.get_waypoints_srv.get_waypoints(key,[],self.waypoints[key],self.waypoint_names[key])
            for (pose,name) in zip(poses,names):
                (trans,rot) = pm.toTf(pm.fromMsg(pose))
                self.broadcaster.sendTransform(trans,rot,rospy.Time.now(),name,self.world)
                #print (trans, rot, name)

    def add_waypoint(self):
        pass

<commit_msg>load references and regions from disc for now<commit_after>import rospy
import yaml
import tf
import tf_conversions.posemath as pm

from predicator_msgs.srv import *
from predicator_msgs.msg import *

from librarian_msgs.srv import *
from librarian_msgs.msg import *

from costar_objrec_msgs.msg import *

from predicator_landmark import GetWaypointsService

'''
SmartWaypointManager
This class will create and load smart waypoints from the $LIBRARIAN_HOME/smart_waypoints directory
It also provides ways to use and access these methods
'''
class SmartWaypointManager:

    def __init__(self,world="world",ns="",endpoint="/endpoint"):
        self.get_waypoints_srv = GetWaypointsService(world=world,service=False)

        rospy.loginfo("[SmartMove] Waiting for LIBRARIAN to handle file I/O...")
        rospy.wait_for_service('/librarian/add_type',5)
        self.add_type_service = rospy.ServiceProxy('/librarian/add_type', librarian_msgs.srv.AddType)
        self.save_service = rospy.ServiceProxy('/librarian/save', librarian_msgs.srv.Save)
        self.load_service = rospy.ServiceProxy('/librarian/load', librarian_msgs.srv.Load)
        self.list_service = rospy.ServiceProxy('/librarian/list', librarian_msgs.srv.List)
        self.delete_service = rospy.ServiceProxy('/librarian/delete', librarian_msgs.srv.Delete)

        self.get_assignment_service = rospy.ServiceProxy('/predicator/get_assignment', predicator_msgs.srv.GetAssignment)

        #self.detected_objects = rospy.Subscriber(ns + '/detected_object_list', DetectedObjectList, self.detected_objects_cb)

        self.broadcaster = tf.TransformBroadcaster()
        self.listener = tf.TransformListener()
        self.world = world
        self.endpoint = endpoint

        self.folder = 'smartmove_waypoint'

        self.add_type_service(self.folder)

        self.waypoints = {}
        self.waypoint_names = {}

        self.all_moves = []

        self.objs = []
        self.obj_classes = []
        self.obj_class = {}

        self.add_type_service("smartmove_info")
        self.available_obj_classes = yaml.load(self.load_service(type="smartmove_info",id="obj_classes").text)
        self.available_regions = yaml.load(self.load_service(type="smartmove_info",id="regions").text)
        self.available_references = yaml.load(self.load_service(type="smartmove_info",id="references").text)
        print "Available classes = " + str(self.available_obj_classes)

    def detected_objects_cb(self,msg):
        self.objs = [obj.id for obj in msg.objects]
        self.obj_classes = [obj.object_class for obj in msg.objects]
        for (o,c) in zip(self.objs,self.obj_classes):
            self.obj_class[o] = c

        #print self.objs
        #print self.obj_classes

    '''
    get all waypoints from the disk
    '''
    def load_all(self):

        self.waypoints = {}
        self.all_moves = []

        waypoint_filenames = self.list_service(self.folder).entries

        for name in waypoint_filenames:
            data = yaml.load(self.load_service(id=name,type=self.folder).text)
            if not data[1] in self.waypoints.keys():
                self.waypoints[data[1]] = []
                self.waypoint_names[data[1]] = []

            self.waypoints[data[1]].append(data[0])
            self.waypoint_names[data[1]].append(name)
            self.all_moves.append(data[1] + "/" + name)

        print " === LOADING === "
        print self.waypoint_names
        print self.waypoints

    def get_all_moves(self):
        return self.all_moves

    def get_detected_objects(self):
        #print self.available_obj_classes
        self.objs = []
        for oc in self.available_obj_classes:
            resp = self.get_assignment_service(PredicateStatement(predicate=oc,params=["*","",""]))
            oc_objs = [p.params[0] for p in resp.values]
            for obj in oc_objs:
                self.obj_class[obj] = oc
            self.objs += oc_objs

        return self.objs

    def delete(self,move):
        self.delete_service(id=move.strip('/'),type=self.folder)

    def get_detected_object_classes(self):
        return self.obj_classes

    def save_new_waypoint(self,obj,name):
        pose = self.get_new_waypoint(obj)
        if not pose is None:
            self.save_service(id=name.strip('/'),type=self.folder,text=yaml.dump([pose,self.obj_class[obj]]))

    def get_new_waypoint(self,obj):
        try:
            (trans,rot) = self.listener.lookupTransform(obj,self.endpoint,rospy.Time(0))
            return pm.toMsg(pm.fromTf((trans,rot)))
        except (tf.LookupException, tf.ConnectivityException, tf.ExtrapolationException):
            rospy.logerr('Failed to lookup transform from %s to %s'%(obj,self.endpoint))
        return None

    def get_moves_for_object(self,obj):
        print "obj class = %s"%self.obj_class[obj]

        wpts = []
        try:
            wpts = self.waypoint_names[self.obj_class[obj]]
        except KeyError:
            wpts = []
        return wpts

    def update_tf(self):

        for key in self.waypoints.keys():
            (poses,names) = self.get_waypoints_srv.get_waypoints(key,[],self.waypoints[key],self.waypoint_names[key])
            for (pose,name) in zip(poses,names):
                (trans,rot) = pm.toTf(pm.fromMsg(pose))
                self.broadcaster.sendTransform(trans,rot,rospy.Time.now(),name,self.world)
                #print (trans, rot, name)

    def add_waypoint(self):
        pass

<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Remote application that interacts with gqrx using rigctl protocol.
Gqrx partially implements rigctl since version 2.3.

Please refer to:
http://gqrx.dk/
http://gqrx.dk/doc/remote-control
http://sourceforge.net/apps/mediawiki/hamlib/index.php?title=Documentation

Author: Rafael Marmelo <rafael@defying.me>
License: MIT License

Copyright (c) 2014 Rafael Marmelo
"""

import argparse
import logging
import os
import time
import tkinter as tk
from modules.ui import GqrxRemote
from modules.app_config import AppConfig

def input_arguments():
    """Argument parser.

    """

    parser = argparse.ArgumentParser(
        description="""Remote application
        that interacts with gqrx using
        rigctl protocol.
        Gqrx partially implements rigctl
        since version 2.3""",
        epilog="""Please refer to:
        http://gqrx.dk/,
        http://gqrx.dk/doc/remote-control,
        http://sourceforge.net/apps/mediawiki/hamlib/index.php?title=Documentation

        Author: Rafael Marmelo <rafael@defying.me>
        License: MIT License

        Copyright (c) 2014 Rafael Marmelo""")

    parser.add_argument("--file",
                        "-f",
                        type=str,
                        required=False,
                        dest="alternate_config_file",
                        help="Overrides the default config file.")

    parser.add_argument("--verbose",
                        "-v",
                        dest="verbose",
                        action="store_true",
                        help="Increase log verbosity.")

    return parser.parse_args()

def log_configuration(verbose):
    """Logger configuration: time/date formatting.

    """

    os.environ["TZ"] = "UTC"
    time.tzset()
    if verbose:
        logging.basicConfig(format="%(asctime)s %(message)s",
                            datefmt="%m/%d/%Y %I:%M:%S %p %Z",
                            level=logging.WARNING)
    else:
        logging.basicConfig(format="%(asctime)s %(message)s",
                            datefmt="%m/%d/%Y %I:%M:%S %p %Z",
                            level=logging.INFO)

# entry point
if __name__ == "__main__":

    args = input_arguments()
    log_configuration(args.verbose)
    logger = logging.getLogger(__name__)

    config_file = args.alternate_config_file
    root = tk.Tk()
    ac = AppConfig(config_file)
    app = GqrxRemote(root, ac)
    app.mainloop()
<commit_msg>updated logging configuration.<commit_after>#!/usr/bin/env python3

"""
Remote application that interacts with gqrx using rigctl protocol.
Gqrx partially implements rigctl since version 2.3.

Please refer to:
http://gqrx.dk/
http://gqrx.dk/doc/remote-control
http://sourceforge.net/apps/mediawiki/hamlib/index.php?title=Documentation

Author: Rafael Marmelo <rafael@defying.me>
License: MIT License

Copyright (c) 2014 Rafael Marmelo
"""

import argparse
import logging
import os
import time
import tkinter as tk
from modules.ui import GqrxRemote
from modules.app_config import AppConfig

def input_arguments():
    """Argument parser.

    """

    parser = argparse.ArgumentParser(
        description="""Remote application
        that interacts with gqrx using
        rigctl protocol.
        Gqrx partially implements rigctl
        since version 2.3""",
        epilog="""Please refer to:
        http://gqrx.dk/,
        http://gqrx.dk/doc/remote-control,
        http://sourceforge.net/apps/mediawiki/hamlib/index.php?title=Documentation

        Author: Rafael Marmelo <rafael@defying.me>
        License: MIT License

        Copyright (c) 2014 Rafael Marmelo""")

    parser.add_argument("--file",
                        "-f",
                        type=str,
                        required=False,
                        dest="alternate_config_file",
                        help="Overrides the default config file.")

    parser.add_argument("--verbose",
                        "-v",
                        dest="verbose",
                        action="store_true",
                        help="Increase log verbosity.")

    return parser.parse_args()

def log_configuration(verbose):
    """Logger configuration: time/date formatting.

    """

    os.environ["TZ"] = "UTC"
    time.tzset()
    if verbose:
        logging.basicConfig(level=logging.INFO)
    else:
        logging.basicConfig(level=logging.WARNING)

    logging.basicConfig(format="%(asctime)s %(message)s",
                        datefmt="%m/%d/%Y %I:%M:%S %p %Z")

    return logging.getLogger(__name__)

# entry point
if __name__ == "__main__":

    args = input_arguments()
    logger = log_configuration(args.verbose)

    config_file = args.alternate_config_file
    root = tk.Tk()
    ac = AppConfig(config_file)
    app = GqrxRemote(root, ac)
    app.mainloop()
<|endoftext|>


<commit_before>from datapackage_pipelines_knesset.common.processors.base_processor import BaseProcessor
from knesset_data.protocols.committee import CommitteeMeetingProtocol
from knesset_data.protocols.exceptions import AntiwordException
import os, csv, json, subprocess, logging, shutil

class ParseCommitteeMeetingProtocolsProcessor(BaseProcessor):

    def __init__(self, *args, **kwargs):
        super(ParseCommitteeMeetingProtocolsProcessor, self).__init__(*args, **kwargs)
        self._schema["fields"] = [{"name": "committee_id", "type": "integer"},
                                  {"name": "meeting_id", "type": "integer"},
                                  {"name": "protocol_file", "type": "string",
                                   "description": "relative path to the protocol file"},
                                  {"name": "text_file", "type": "string"},
                                  {"name": "parts_file", "type": "string"},]
        self._all_filenames = []

    def _process(self, datapackage, resources):
        return self._process_filter(datapackage, resources)

    def _get_filename(self, relpath):
        return os.path.join(self._parameters["out-path"], relpath)

    def _filter_row(self, meeting_protocol, **kwargs):
        committee_id = meeting_protocol["committee_id"]
        meeting_id = meeting_protocol["meeting_id"]
        parts_relpath = os.path.join(str(committee_id), "{}.csv".format(meeting_id))
        text_relpath = os.path.join(str(committee_id), "{}.txt".format(meeting_id))
        parts_filename = self._get_filename(parts_relpath)
        text_filename = self._get_filename(text_relpath)
        protocol_filename = meeting_protocol["protocol_file"]
        protocol_ext = protocol_filename.strip()[-4:]
        if not os.path.exists(parts_filename) or os.path.getsize(parts_filename) < 5:
            self._ensure_parts_path_exists(parts_filename, parts_relpath)
            if protocol_ext == ".doc":
                parse_res = self._parse_doc_protocol(committee_id, meeting_id, protocol_filename, parts_filename,
                                                     text_filename)
            elif protocol_ext == ".rtf":
                parse_res = self._parse_rtf_protocol(committee_id, meeting_id, protocol_filename, parts_filename,
                                                     text_filename)
            else:
                raise Exception("unknown extension: {}".format(protocol_ext))
            if not parse_res:
                if os.path.exists(text_filename):
                    os.unlink(text_filename)
                if os.path.exists(parts_filename):
                    os.unlink(parts_filename)
                text_filename = None
                parts_filename = None
        if parts_filename:
            self._all_filenames += [parts_relpath]
        if text_filename:
            self._all_filenames += [text_relpath]
        yield {"committee_id": committee_id,
               "meeting_id": meeting_id,
               "protocol_file": protocol_filename,
               "text_file": text_filename,
               "parts_file": parts_filename}

    def _ensure_parts_path_exists(self, parts_filename, parts_relpath):
        if parts_relpath not in self._all_filenames:
            os.makedirs(os.path.dirname(parts_filename), exist_ok=True)

    def _parse_rtf_protocol(self, committee_id, meeting_id, protocol_filename, parts_filename, text_filename):
        rtf_extractor = os.environ.get("RTF_EXTRACTOR_BIN")
        if rtf_extractor:
            cmd = rtf_extractor + ' ' + protocol_filename + ' ' + text_filename
            subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
            with open(text_filename) as f:
                protocol_text = f.read()
            with CommitteeMeetingProtocol.get_from_text(protocol_text) as protocol:
                self._parse_protocol_parts(parts_filename, protocol)
            return True
        else:
            logging.warning("missing RTF_EXTRACTOR_BIN environment variable, skipping rtf parsing")
            return False

    def _parse_doc_protocol(self, committee_id, meeting_id, protocol_filename, parts_filename, text_filename):
        try:
            with CommitteeMeetingProtocol.get_from_filename(protocol_filename) as protocol:
                with open(text_filename, "w") as f:
                    f.write(protocol.text)
                    logging.info("parsed doc to text -> {}".format(text_filename))
                self._parse_protocol_parts(parts_filename, protocol)
        except (AntiwordException, subprocess.SubprocessError):
            logging.exception("committee {} meeting {}: failed to parse doc file, skipping".format(committee_id,
                                                                                meeting_id))
            return False
        return True

    def _parse_protocol_parts(self, parts_filename, protocol):
        with open(parts_filename, "w") as f:
            csv_writer = csv.writer(f)
            csv_writer.writerow(["header", "body"])
            for part in protocol.parts:
                csv_writer.writerow([part.header, part.body])
            logging.info("parsed parts file -> {}".format(parts_filename))

    def _process_cleanup(self):
        filename = self._get_filename("datapackage.json")
        os.makedirs(os.path.dirname(filename), exist_ok=True)
        with open(filename, "w") as f:
            descriptor = {"name": "_", "path": self._all_filenames}
            descriptor.update(**self._parameters.get("data-resource-descriptor", {}))
            json.dump({"name": "_", "resources": [descriptor]}, f)

if __name__ == '__main__':
    ParseCommitteeMeetingProtocolsProcessor.main()
<commit_msg>skip rtf protocols which fail to parse<commit_after>from datapackage_pipelines_knesset.common.processors.base_processor import BaseProcessor
from knesset_data.protocols.committee import CommitteeMeetingProtocol
from knesset_data.protocols.exceptions import AntiwordException
import os, csv, json, subprocess, logging, shutil

class ParseCommitteeMeetingProtocolsProcessor(BaseProcessor):

    def __init__(self, *args, **kwargs):
        super(ParseCommitteeMeetingProtocolsProcessor, self).__init__(*args, **kwargs)
        self._schema["fields"] = [{"name": "committee_id", "type": "integer"},
                                  {"name": "meeting_id", "type": "integer"},
                                  {"name": "protocol_file", "type": "string",
                                   "description": "relative path to the protocol file"},
                                  {"name": "text_file", "type": "string"},
                                  {"name": "parts_file", "type": "string"},]
        self._all_filenames = []

    def _process(self, datapackage, resources):
        return self._process_filter(datapackage, resources)

    def _get_filename(self, relpath):
        return os.path.join(self._parameters["out-path"], relpath)

    def _filter_row(self, meeting_protocol, **kwargs):
        committee_id = meeting_protocol["committee_id"]
        meeting_id = meeting_protocol["meeting_id"]
        parts_relpath = os.path.join(str(committee_id), "{}.csv".format(meeting_id))
        text_relpath = os.path.join(str(committee_id), "{}.txt".format(meeting_id))
        parts_filename = self._get_filename(parts_relpath)
        text_filename = self._get_filename(text_relpath)
        protocol_filename = meeting_protocol["protocol_file"]
        protocol_ext = protocol_filename.strip()[-4:]
        if not os.path.exists(parts_filename) or os.path.getsize(parts_filename) < 5:
            self._ensure_parts_path_exists(parts_filename, parts_relpath)
            if protocol_ext == ".doc":
                parse_res = self._parse_doc_protocol(committee_id, meeting_id, protocol_filename, parts_filename,
                                                     text_filename)
            elif protocol_ext == ".rtf":
                parse_res = self._parse_rtf_protocol(committee_id, meeting_id, protocol_filename, parts_filename,
                                                     text_filename)
            else:
                raise Exception("unknown extension: {}".format(protocol_ext))
            if not parse_res:
                if os.path.exists(text_filename):
                    os.unlink(text_filename)
                if os.path.exists(parts_filename):
                    os.unlink(parts_filename)
                text_filename = None
                parts_filename = None
        if parts_filename:
            self._all_filenames += [parts_relpath]
        if text_filename:
            self._all_filenames += [text_relpath]
        yield {"committee_id": committee_id,
               "meeting_id": meeting_id,
               "protocol_file": protocol_filename,
               "text_file": text_filename,
               "parts_file": parts_filename}

    def _ensure_parts_path_exists(self, parts_filename, parts_relpath):
        if parts_relpath not in self._all_filenames:
            os.makedirs(os.path.dirname(parts_filename), exist_ok=True)

    def _parse_rtf_protocol(self, committee_id, meeting_id, protocol_filename, parts_filename, text_filename):
        rtf_extractor = os.environ.get("RTF_EXTRACTOR_BIN")
        if rtf_extractor:
            cmd = rtf_extractor + ' ' + protocol_filename + ' ' + text_filename
            try:
                subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
                with open(text_filename) as f:
                    protocol_text = f.read()
                with CommitteeMeetingProtocol.get_from_text(protocol_text) as protocol:
                    self._parse_protocol_parts(parts_filename, protocol)
            except subprocess.SubprocessError:
                logging.exception("committee {} meeting {}: failed to parse rtf file, skipping".format(committee_id,
                                                                                meeting_id))
                return False
            return True
        else:
            logging.warning("missing RTF_EXTRACTOR_BIN environment variable, skipping rtf parsing")
            return False

    def _parse_doc_protocol(self, committee_id, meeting_id, protocol_filename, parts_filename, text_filename):
        try:
            with CommitteeMeetingProtocol.get_from_filename(protocol_filename) as protocol:
                with open(text_filename, "w") as f:
                    f.write(protocol.text)
                    logging.info("parsed doc to text -> {}".format(text_filename))
                self._parse_protocol_parts(parts_filename, protocol)
        except (AntiwordException, subprocess.SubprocessError):
            logging.exception("committee {} meeting {}: failed to parse doc file, skipping".format(committee_id,
                                                                                meeting_id))
            return False
        return True

    def _parse_protocol_parts(self, parts_filename, protocol):
        with open(parts_filename, "w") as f:
            csv_writer = csv.writer(f)
            csv_writer.writerow(["header", "body"])
            for part in protocol.parts:
                csv_writer.writerow([part.header, part.body])
            logging.info("parsed parts file -> {}".format(parts_filename))

    def _process_cleanup(self):
        filename = self._get_filename("datapackage.json")
        os.makedirs(os.path.dirname(filename), exist_ok=True)
        with open(filename, "w") as f:
            descriptor = {"name": "_", "path": self._all_filenames}
            descriptor.update(**self._parameters.get("data-resource-descriptor", {}))
            json.dump({"name": "_", "resources": [descriptor]}, f)

if __name__ == '__main__':
    ParseCommitteeMeetingProtocolsProcessor.main()
<|endoftext|>