<commit_before># -*- coding: utf-8 -*-

# Copyright 2016 Juca Crispim <juca@poraodojuca.net>

# This file is part of toxicbuild.

# toxicbuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# toxicbuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with toxicbuild. If not, see <http://www.gnu.org/licenses/>.

import asyncio
import traceback
from mongomotor import Document
from mongomotor.fields import (StringField, IntField, BooleanField)
from toxicbuild.core.exceptions import ToxicClientException, BadJsonData
from toxicbuild.core.utils import string2datetime, LoggerMixin, now
from toxicbuild.master.build import BuildStep, Builder
from toxicbuild.master.client import get_build_client
from toxicbuild.master.signals import (build_started, build_finished,
                                       step_started, step_finished,
                                       step_output_arrived)

class Slave(Document, LoggerMixin):

    """ Slaves are the entities that actualy do the work
    of execute steps. The comunication to slaves is through
    the network (using :class:`toxicbuild.master.client.BuildClient`).
    The steps are actually decided by the slave.
    """
    name = StringField(required=True, unique=True)
    host = StringField(required=True)
    port = IntField(required=True)
    token = StringField(required=True)
    is_alive = BooleanField(default=False)

    meta = {
        'ordering': ['name']
    }

    @classmethod
    @asyncio.coroutine
    def create(cls, **kwargs):
        slave = cls(**kwargs)
        yield from slave.save()
        return slave

    def to_dict(self, id_as_str=False):
        my_dict = {'name': self.name, 'host': self.host,
                   'port': self.port, 'token': self.token,
                   'is_alive': self.is_alive, 'id': self.id}
        if id_as_str:
            my_dict['id'] = str(self.id)
        return my_dict

    @classmethod
    @asyncio.coroutine
    def get(cls, **kwargs):
        slave = yield from cls.objects.get(**kwargs)
        return slave

    @asyncio.coroutine
    def get_client(self):
        """ Returns a :class:`toxicbuild.master.client.BuildClient` instance
        already connected to the server.
        """
        connected_client = yield from get_build_client(self, self.host,
                                                       self.port)
        return connected_client

    @asyncio.coroutine
    def healthcheck(self):
        """ Check if the build server is up and running
        """
        with (yield from self.get_client()) as client:
            alive = yield from client.healthcheck()

        self.is_alive = alive
        # using yield instead of yield from because mongomotor's
        # save returns a tornado Future, not a asyncio Future
        yield from self.save()
        return self.is_alive

    @asyncio.coroutine
    def list_builders(self, revision):
        """ List builder available in for a given revision

        :param revision: An instance of
          :class:`toxicbuild.master.repository.RepositoryRevision`
        """
        repository = yield from revision.repository
        repo_url = repository.url
        vcs_type = repository.vcs_type
        branch = revision.branch
        named_tree = revision.commit

        with (yield from self.get_client()) as client:
            builders = yield from client.list_builders(repo_url, vcs_type,
                                                       branch, named_tree)

        builders = [(yield from Builder.get_or_create(repository=repository,
                                                      name=bname))
                    for bname in builders]

        builders = yield from builders
        return list(builders)

    @asyncio.coroutine
    def build(self, build):
        """ Connects to a build server and requests a build on that server

        :param build: An instance of :class:`toxicbuild.master.build.Build`
        """

        with (yield from self.get_client()) as client:

            try:
                build_info = yield from client.build(
                    build, process_coro=self._process_info)
            except (ToxicClientException, BadJsonData):
                output = traceback.format_exc()
                build.status = 'exception'
                build.started = build.started or now()
                build.finished = build.finished or now()
                exception_step = BuildStep(output=output, started=now(),
                                           finished=now(), status='exception',
                                           command='', name='exception')
                build.steps.append(exception_step)

                yield from build.update()
                build_info = build.to_dict()

        return build_info

    @asyncio.coroutine
    def _process_info(self, build, info):
        """ Method used to process information sent by
        the build server about an in progress build.

        :param build: The build that is being executed
        :param info: A dictionary. The information sent by the
          slave that is executing the build.
        """

        # if we need one more conditional here is better to use
        # a map...
        if info['info_type'] == 'build_info':
            yield from self._process_build_info(build, info)

        elif info['info_type'] == 'step_info':
            yield from self._process_step_info(build, info)

        else:
            yield from self._process_step_output_info(build, info)

    @asyncio.coroutine
    def _process_build_info(self, build, build_info):
        repo = yield from build.repository
        build.status = build_info['status']
        build.started = string2datetime(build_info['started'])
        finished = build_info['finished']
        if finished:
            build.finished = string2datetime(finished)

        yield from build.update()

        if not build.finished:
            msg = 'build started at {}'.format(build_info['started'])
            self.log(msg)
            build_started.send(repo, build=build)
        else:
            msg = 'build finished at {} with status {}'.format(
                build_info['finished'], build.status)
            self.log(msg)
            build_finished.send(repo, build=build)

    @asyncio.coroutine
    def _process_step_info(self, build, step_info):

        cmd = step_info['cmd']
        name = step_info['name']
        status = step_info['status']
        output = step_info['output']
        started = step_info['started']
        finished = step_info['finished']
        index = step_info['index']
        uuid = step_info['uuid']

        repo = yield from build.repository
        requested_step = self._get_step(build, uuid)

        if requested_step:
            requested_step.status = status
            requested_step.output = output
            requested_step.finished = string2datetime(finished)
            msg = 'step {} finished at {} with status {}'.format(
                requested_step.command, finished, requested_step.status)
            self.log(msg, level='debug')
            step_finished.send(repo, build=build, step=requested_step)

        else:
            requested_step = BuildStep(name=name, command=cmd,
                                       status=status, output=output,
                                       started=string2datetime(started),
                                       index=index, uuid=uuid)
            msg = 'step {} started at {}'.format(requested_step.command,
                                                 started)
            self.log(msg, level='debug')
            step_started.send(repo, build=build, step=requested_step)
            build.steps.append(requested_step)

        yield from build.update()

    @asyncio.coroutine
    def _process_step_output_info(self, build, info):
        uuid = info['uuid']
        output = info['output']
        repo = yield from build.repository
        step = self._get_step(build, uuid)
        step.output = ''.join([step.output or '', output])
        yield from build.update()
        step_output_arrived.send(repo, info=info)

    def _get_step(self, build, step_uuid):
        """Returns a step from ``build``. Returns None if the requested
        step is not present in the build.

        :param build: A :class:`toxicbuild.master.build.Build` instance.
        :param step_uuid: The uuid of the requested step.
        """

        for step in build.steps:
            if step.uuid == step_uuid:
                return step
<commit_msg>adding log<commit_after># -*- coding: utf-8 -*-

# Copyright 2016 Juca Crispim <juca@poraodojuca.net>

# This file is part of toxicbuild.

# toxicbuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# toxicbuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with toxicbuild. If not, see <http://www.gnu.org/licenses/>.

import asyncio
import traceback
from mongomotor import Document
from mongomotor.fields import (StringField, IntField, BooleanField)
from toxicbuild.core.exceptions import ToxicClientException, BadJsonData
from toxicbuild.core.utils import string2datetime, LoggerMixin, now
from toxicbuild.master.build import BuildStep, Builder
from toxicbuild.master.client import get_build_client
from toxicbuild.master.signals import (build_started, build_finished,
                                       step_started, step_finished,
                                       step_output_arrived)

class Slave(Document, LoggerMixin):

    """ Slaves are the entities that actualy do the work
    of execute steps. The comunication to slaves is through
    the network (using :class:`toxicbuild.master.client.BuildClient`).
    The steps are actually decided by the slave.
    """
    name = StringField(required=True, unique=True)
    host = StringField(required=True)
    port = IntField(required=True)
    token = StringField(required=True)
    is_alive = BooleanField(default=False)

    meta = {
        'ordering': ['name']
    }

    @classmethod
    @asyncio.coroutine
    def create(cls, **kwargs):
        slave = cls(**kwargs)
        yield from slave.save()
        return slave

    def to_dict(self, id_as_str=False):
        my_dict = {'name': self.name, 'host': self.host,
                   'port': self.port, 'token': self.token,
                   'is_alive': self.is_alive, 'id': self.id}
        if id_as_str:
            my_dict['id'] = str(self.id)
        return my_dict

    @classmethod
    @asyncio.coroutine
    def get(cls, **kwargs):
        slave = yield from cls.objects.get(**kwargs)
        return slave

    @asyncio.coroutine
    def get_client(self):
        """ Returns a :class:`toxicbuild.master.client.BuildClient` instance
        already connected to the server.
        """
        connected_client = yield from get_build_client(self, self.host,
                                                       self.port)
        return connected_client

    @asyncio.coroutine
    def healthcheck(self):
        """ Check if the build server is up and running
        """
        with (yield from self.get_client()) as client:
            alive = yield from client.healthcheck()

        self.is_alive = alive
        # using yield instead of yield from because mongomotor's
        # save returns a tornado Future, not a asyncio Future
        yield from self.save()
        return self.is_alive

    @asyncio.coroutine
    def list_builders(self, revision):
        """ List builder available in for a given revision

        :param revision: An instance of
          :class:`toxicbuild.master.repository.RepositoryRevision`
        """
        repository = yield from revision.repository
        repo_url = repository.url
        vcs_type = repository.vcs_type
        branch = revision.branch
        named_tree = revision.commit

        with (yield from self.get_client()) as client:
            builders = yield from client.list_builders(repo_url, vcs_type,
                                                       branch, named_tree)

        builders = [(yield from Builder.get_or_create(repository=repository,
                                                      name=bname))
                    for bname in builders]

        builders = yield from builders
        return list(builders)

    @asyncio.coroutine
    def build(self, build):
        """ Connects to a build server and requests a build on that server

        :param build: An instance of :class:`toxicbuild.master.build.Build`
        """

        with (yield from self.get_client()) as client:

            try:
                build_info = yield from client.build(
                    build, process_coro=self._process_info)
            except (ToxicClientException, BadJsonData):
                output = traceback.format_exc()
                build.status = 'exception'
                build.started = build.started or now()
                build.finished = build.finished or now()
                exception_step = BuildStep(output=output, started=now(),
                                           finished=now(), status='exception',
                                           command='', name='exception')
                build.steps.append(exception_step)

                yield from build.update()
                build_info = build.to_dict()

        return build_info

    @asyncio.coroutine
    def _process_info(self, build, info):
        """ Method used to process information sent by
        the build server about an in progress build.

        :param build: The build that is being executed
        :param info: A dictionary. The information sent by the
          slave that is executing the build.
        """

        # if we need one more conditional here is better to use
        # a map...
        if info['info_type'] == 'build_info':
            yield from self._process_build_info(build, info)

        elif info['info_type'] == 'step_info':
            yield from self._process_step_info(build, info)

        else:
            yield from self._process_step_output_info(build, info)

    @asyncio.coroutine
    def _process_build_info(self, build, build_info):
        repo = yield from build.repository
        build.status = build_info['status']
        build.started = string2datetime(build_info['started'])
        finished = build_info['finished']
        if finished:
            build.finished = string2datetime(finished)

        yield from build.update()

        if not build.finished:
            msg = 'build started at {}'.format(build_info['started'])
            self.log(msg)
            build_started.send(repo, build=build)
        else:
            msg = 'build finished at {} with status {}'.format(
                build_info['finished'], build.status)
            self.log(msg)
            build_finished.send(repo, build=build)

    @asyncio.coroutine
    def _process_step_info(self, build, step_info):

        cmd = step_info['cmd']
        name = step_info['name']
        status = step_info['status']
        output = step_info['output']
        started = step_info['started']
        finished = step_info['finished']
        index = step_info['index']
        uuid = step_info['uuid']

        repo = yield from build.repository
        requested_step = self._get_step(build, uuid)

        if requested_step:
            requested_step.status = status
            requested_step.output = output
            requested_step.finished = string2datetime(finished)
            msg = 'step {} finished at {} with status {}'.format(
                requested_step.command, finished, requested_step.status)
            self.log(msg, level='debug')
            step_finished.send(repo, build=build, step=requested_step)

        else:
            requested_step = BuildStep(name=name, command=cmd,
                                       status=status, output=output,
                                       started=string2datetime(started),
                                       index=index, uuid=uuid)
            msg = 'step {} started at {}'.format(requested_step.command,
                                                 started)
            self.log(msg, level='debug')
            step_started.send(repo, build=build, step=requested_step)
            build.steps.append(requested_step)

        yield from build.update()

    @asyncio.coroutine
    def _process_step_output_info(self, build, info):
        uuid = info['uuid']
        output = info['output']
        repo = yield from build.repository
        step = self._get_step(build, uuid)
        step.output = ''.join([step.output or '', output])
        yield from build.update()
        msg = 'step_output_arrived for {}'.format(uuid)
        self.log(msg, level='debug')
        step_output_arrived.send(repo, step_info=info)

    def _get_step(self, build, step_uuid):
        """Returns a step from ``build``. Returns None if the requested
        step is not present in the build.

        :param build: A :class:`toxicbuild.master.build.Build` instance.
        :param step_uuid: The uuid of the requested step.
        """

        for step in build.steps:
            if step.uuid == step_uuid:
                return step
<|endoftext|>


<commit_before># -*- coding: iso-8859-1 -*-
#
# Copyright (C) 2005 Edgewall Software
# Copyright (C) 2005 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.com/license.html.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://projects.edgewall.com/trac/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

from heapq import heappop, heappush

from trac.core import *
from trac.perm import PermissionError

class IRepositoryConnector(Interface):
    """Extension point interface for components that provide support for a
    specific version control system."""

    def get_supported_types():
        """Return the types of version control systems that are supported by
        this connector, and their relative priorities.

        Highest number is highest priority.
        """

    def get_repository(repos_type, repos_dir, authname):
        """Return the Repository object for the given repository type and
        directory.
        """

class RepositoryManager(Component):
    """Component that keeps track of the supported version control systems, and
    provides easy access to the configured implementation."""

    connectors = ExtensionPoint(IRepositoryConnector)

    def __init__(self):
        self._connector = None

    def get_repository(self, repos_type, repos_dir, authname):
        if not self._connector:
            candidates = []
            for connector in self.connectors:
                for repos_type_, prio in connector.get_supported_types():
                    if repos_type_ != repos_type:
                        continue
                    heappush(candidates, (-prio, connector))
            if not candidates:
                raise TracError, 'Unsupported version control system "%s"' \
                                 % repos_type
            self._connector = heappop(candidates)[1]
        return self._connector.get_repository(repos_type, repos_dir, authname)

class Repository(object):
    """
    Base class for a repository provided by a version control system.
    """

    def __init__(self, name, authz, log):
        self.name = name
        self.authz = authz or Authorizer()
        self.log = log

    def close(self):
        """
        Close the connection to the repository.
        """
        raise NotImplementedError

    def get_changeset(self, rev):
        """
        Retrieve a Changeset object that describes the changes made in
        revision 'rev'.
        """
        raise NotImplementedError

    def get_changesets(self, start, stop):
        """
        Generate Changeset belonging to the given time period (start, stop).
        """
        rev = self.youngest_rev
        while rev:
            if self.authz.has_permission_for_changeset(rev):
                chgset = self.get_changeset(rev)
                if chgset.date < start:
                    return
                if chgset.date < stop:
                    yield chgset
            rev = self.previous_rev(rev)

    def has_node(self, path, rev):
        """
        Tell if there's a node at the specified (path,rev) combination.
        """
        try:
            self.get_node()
            return True
        except TracError:
            return False

    def get_node(self, path, rev=None):
        """
        Retrieve a Node (directory or file) from the repository at the
        given path. If the rev parameter is specified, the version of the
        node at that revision is returned, otherwise the latest version
        of the node is returned.
        """
        raise NotImplementedError

    def get_oldest_rev(self):
        """
        Return the oldest revision stored in the repository.
        """
        raise NotImplementedError
    oldest_rev = property(lambda x: x.get_oldest_rev())

    def get_youngest_rev(self):
        """
        Return the youngest revision in the repository.
        """
        raise NotImplementedError
    youngest_rev = property(lambda x: x.get_youngest_rev())

    def previous_rev(self, rev):
        """
        Return the revision immediately preceding the specified revision.
        """
        raise NotImplementedError

    def next_rev(self, rev, path=''):
        """
        Return the revision immediately following the specified revision.
        """
        raise NotImplementedError

    def rev_older_than(self, rev1, rev2):
        """
        Return True if rev1 is older than rev2, i.e. if rev1 comes before rev2
        in the revision sequence.
        """
        raise NotImplementedError

    def get_youngest_rev_in_cache(self, db):
        """
        Return the youngest revision currently cached.
        The way revisions are sequenced is version control specific.
        By default, one assumes that the revisions are sequenced in time.
        """
        cursor = db.cursor()
        cursor.execute("SELECT rev FROM revision ORDER BY time DESC LIMIT 1")
        row = cursor.fetchone()
        return row and row[0] or None

    def get_path_history(self, path, rev=None, limit=None):
        """
        Retrieve all the revisions containing this path (no newer than 'rev').
        The result format should be the same as the one of Node.get_history()
        """
        raise NotImplementedError

    def normalize_path(self, path):
        """
        Return a canonical representation of path in the repos.
        """
        return NotImplementedError

    def normalize_rev(self, rev):
        """
        Return a canonical representation of a revision in the repos.
        'None' is a valid revision value and represents the youngest revision.
        """
        return NotImplementedError

    def short_rev(self, rev):
        """
        Return a compact representation of a revision in the repos.
        """
        return self.normalize_rev(rev)

    def get_changes(self, old_path, old_rev, new_path, new_rev,
                    ignore_ancestry=1):
        """
        Generator that yields change tuples (old_node, new_node, kind, change)
        for each node change between the two arbitrary (path,rev) pairs.

        The old_node is assumed to be None when the change is an ADD,
        the new_node is assumed to be None when the change is a DELETE.
        """
        raise NotImplementedError

class Node(object):
    """
    Represents a directory or file in the repository.
    """

    DIRECTORY = "dir"
    FILE = "file"

    def __init__(self, path, rev, kind):
        assert kind in (Node.DIRECTORY, Node.FILE), "Unknown node kind %s" % kind
        self.path = str(path)
        self.rev = rev
        self.kind = kind

    def get_content(self):
        """
        Return a stream for reading the content of the node. This method
        will return None for directories. The returned object should provide
        a read([len]) function.
        """
        raise NotImplementedError

    def get_entries(self):
        """
        Generator that yields the immediate child entries of a directory, in no
        particular order. If the node is a file, this method returns None.
        """
        raise NotImplementedError

    def get_history(self, limit=None):
        """
        Generator that yields (path, rev, chg) tuples, one for each revision in which
        the node was changed. This generator will follow copies and moves of a
        node (if the underlying version control system supports that), which
        will be indicated by the first element of the tuple (i.e. the path)
        changing.
        Starts with an entry for the current revision.
        """
        raise NotImplementedError

    def get_previous(self):
        """
        Return the (path, rev, chg) tuple corresponding to the previous
        revision for that node.
        """
        skip = True
        for p in self.get_history(2):
            if skip:
                skip = False
            else:
                return p

    def get_properties(self):
        """
        Returns a dictionary containing the properties (meta-data) of the node.
        The set of properties depends on the version control system.
        """
        raise NotImplementedError

    def get_content_length(self):
        raise NotImplementedError
    content_length = property(lambda x: x.get_content_length())

    def get_content_type(self):
        raise NotImplementedError
    content_type = property(lambda x: x.get_content_type())

    def get_name(self):
        return self.path.split('/')[-1]
    name = property(lambda x: x.get_name())

    def get_last_modified(self):
        raise NotImplementedError
    last_modified = property(lambda x: x.get_last_modified())

    isdir = property(lambda x: x.kind == Node.DIRECTORY)
    isfile = property(lambda x: x.kind == Node.FILE)

class Changeset(object):
    """
    Represents a set of changes of a repository.
    """

    ADD = 'add'
    COPY = 'copy'
    DELETE = 'delete'
    EDIT = 'edit'
    MOVE = 'move'

    def __init__(self, rev, message, author, date):
        self.rev = rev
        self.message = message
        self.author = author
        self.date = date

    def get_changes(self):
        """
        Generator that produces a (path, kind, change, base_rev, base_path)
        tuple for every change in the changeset, where change can be one of
        Changeset.ADD, Changeset.COPY, Changeset.DELETE, Changeset.EDIT or
        Changeset.MOVE, and kind is one of Node.FILE or Node.DIRECTORY.
        """
        raise NotImplementedError

class PermissionDenied(PermissionError):
    """
    Exception raised by an authorizer if the user has insufficient permissions
    to view a specific part of the repository.
    """
    def __str__(self):
        return self.action

class Authorizer(object):
    """
    Base class for authorizers that are responsible to granting or denying
    access to view certain parts of a repository.
    """

    def assert_permission(self, path):
        if not self.has_permission(path):
            raise PermissionDenied, \
                  'Insufficient permissions to access %s' % path

    def assert_permission_for_changeset(self, rev):
        if not self.has_permission_for_changeset(rev):
            raise PermissionDenied, \
                  'Insufficient permissions to access changeset %s' % rev

    def has_permission(self, path):
        return True

    def has_permission_for_changeset(self, rev):
        return True
<commit_msg>The default implementation for `Repository.has_node` should propagate its arguments. Fixes #2731.<commit_after># -*- coding: iso-8859-1 -*-
#
# Copyright (C) 2005 Edgewall Software
# Copyright (C) 2005 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.com/license.html.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://projects.edgewall.com/trac/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

from heapq import heappop, heappush

from trac.core import *
from trac.perm import PermissionError

class IRepositoryConnector(Interface):
    """Extension point interface for components that provide support for a
    specific version control system."""

    def get_supported_types():
        """Return the types of version control systems that are supported by
        this connector, and their relative priorities.

        Highest number is highest priority.
        """

    def get_repository(repos_type, repos_dir, authname):
        """Return the Repository object for the given repository type and
        directory.
        """

class RepositoryManager(Component):
    """Component that keeps track of the supported version control systems, and
    provides easy access to the configured implementation."""

    connectors = ExtensionPoint(IRepositoryConnector)

    def __init__(self):
        self._connector = None

    def get_repository(self, repos_type, repos_dir, authname):
        if not self._connector:
            candidates = []
            for connector in self.connectors:
                for repos_type_, prio in connector.get_supported_types():
                    if repos_type_ != repos_type:
                        continue
                    heappush(candidates, (-prio, connector))
            if not candidates:
                raise TracError, 'Unsupported version control system "%s"' \
                                 % repos_type
            self._connector = heappop(candidates)[1]
        return self._connector.get_repository(repos_type, repos_dir, authname)

class Repository(object):
    """
    Base class for a repository provided by a version control system.
    """

    def __init__(self, name, authz, log):
        self.name = name
        self.authz = authz or Authorizer()
        self.log = log

    def close(self):
        """
        Close the connection to the repository.
        """
        raise NotImplementedError

    def get_changeset(self, rev):
        """
        Retrieve a Changeset object that describes the changes made in
        revision 'rev'.
        """
        raise NotImplementedError

    def get_changesets(self, start, stop):
        """
        Generate Changeset belonging to the given time period (start, stop).
        """
        rev = self.youngest_rev
        while rev:
            if self.authz.has_permission_for_changeset(rev):
                chgset = self.get_changeset(rev)
                if chgset.date < start:
                    return
                if chgset.date < stop:
                    yield chgset
            rev = self.previous_rev(rev)

    def has_node(self, path, rev=None):
        """
        Tell if there's a node at the specified (path,rev) combination.

        When `rev` is `None`, the latest revision is implied.
        """
        try:
            self.get_node(path, rev)
            return True
        except TracError:
            return False

    def get_node(self, path, rev=None):
        """
        Retrieve a Node (directory or file) from the repository at the
        given path. If the rev parameter is specified, the version of the
        node at that revision is returned, otherwise the latest version
        of the node is returned.
        """
        raise NotImplementedError

    def get_oldest_rev(self):
        """
        Return the oldest revision stored in the repository.
        """
        raise NotImplementedError
    oldest_rev = property(lambda x: x.get_oldest_rev())

    def get_youngest_rev(self):
        """
        Return the youngest revision in the repository.
        """
        raise NotImplementedError
    youngest_rev = property(lambda x: x.get_youngest_rev())

    def previous_rev(self, rev):
        """
        Return the revision immediately preceding the specified revision.
        """
        raise NotImplementedError

    def next_rev(self, rev, path=''):
        """
        Return the revision immediately following the specified revision.
        """
        raise NotImplementedError

    def rev_older_than(self, rev1, rev2):
        """
        Return True if rev1 is older than rev2, i.e. if rev1 comes before rev2
        in the revision sequence.
        """
        raise NotImplementedError

    def get_youngest_rev_in_cache(self, db):
        """
        Return the youngest revision currently cached.
        The way revisions are sequenced is version control specific.
        By default, one assumes that the revisions are sequenced in time.
        """
        cursor = db.cursor()
        cursor.execute("SELECT rev FROM revision ORDER BY time DESC LIMIT 1")
        row = cursor.fetchone()
        return row and row[0] or None

    def get_path_history(self, path, rev=None, limit=None):
        """
        Retrieve all the revisions containing this path (no newer than 'rev').
        The result format should be the same as the one of Node.get_history()
        """
        raise NotImplementedError

    def normalize_path(self, path):
        """
        Return a canonical representation of path in the repos.
        """
        return NotImplementedError

    def normalize_rev(self, rev):
        """
        Return a canonical representation of a revision in the repos.
        'None' is a valid revision value and represents the youngest revision.
        """
        return NotImplementedError

    def short_rev(self, rev):
        """
        Return a compact representation of a revision in the repos.
        """
        return self.normalize_rev(rev)

    def get_changes(self, old_path, old_rev, new_path, new_rev,
                    ignore_ancestry=1):
        """
        Generator that yields change tuples (old_node, new_node, kind, change)
        for each node change between the two arbitrary (path,rev) pairs.

        The old_node is assumed to be None when the change is an ADD,
        the new_node is assumed to be None when the change is a DELETE.
        """
        raise NotImplementedError

class Node(object):
    """
    Represents a directory or file in the repository.
    """

    DIRECTORY = "dir"
    FILE = "file"

    def __init__(self, path, rev, kind):
        assert kind in (Node.DIRECTORY, Node.FILE), "Unknown node kind %s" % kind
        self.path = str(path)
        self.rev = rev
        self.kind = kind

    def get_content(self):
        """
        Return a stream for reading the content of the node. This method
        will return None for directories. The returned object should provide
        a read([len]) function.
        """
        raise NotImplementedError

    def get_entries(self):
        """
        Generator that yields the immediate child entries of a directory, in no
        particular order. If the node is a file, this method returns None.
        """
        raise NotImplementedError

    def get_history(self, limit=None):
        """
        Generator that yields (path, rev, chg) tuples, one for each revision in which
        the node was changed. This generator will follow copies and moves of a
        node (if the underlying version control system supports that), which
        will be indicated by the first element of the tuple (i.e. the path)
        changing.
        Starts with an entry for the current revision.
        """
        raise NotImplementedError

    def get_previous(self):
        """
        Return the (path, rev, chg) tuple corresponding to the previous
        revision for that node.
        """
        skip = True
        for p in self.get_history(2):
            if skip:
                skip = False
            else:
                return p

    def get_properties(self):
        """
        Returns a dictionary containing the properties (meta-data) of the node.
        The set of properties depends on the version control system.
        """
        raise NotImplementedError

    def get_content_length(self):
        raise NotImplementedError
    content_length = property(lambda x: x.get_content_length())

    def get_content_type(self):
        raise NotImplementedError
    content_type = property(lambda x: x.get_content_type())

    def get_name(self):
        return self.path.split('/')[-1]
    name = property(lambda x: x.get_name())

    def get_last_modified(self):
        raise NotImplementedError
    last_modified = property(lambda x: x.get_last_modified())

    isdir = property(lambda x: x.kind == Node.DIRECTORY)
    isfile = property(lambda x: x.kind == Node.FILE)

class Changeset(object):
    """
    Represents a set of changes of a repository.
    """

    ADD = 'add'
    COPY = 'copy'
    DELETE = 'delete'
    EDIT = 'edit'
    MOVE = 'move'

    def __init__(self, rev, message, author, date):
        self.rev = rev
        self.message = message
        self.author = author
        self.date = date

    def get_changes(self):
        """
        Generator that produces a (path, kind, change, base_rev, base_path)
        tuple for every change in the changeset, where change can be one of
        Changeset.ADD, Changeset.COPY, Changeset.DELETE, Changeset.EDIT or
        Changeset.MOVE, and kind is one of Node.FILE or Node.DIRECTORY.
        """
        raise NotImplementedError

class PermissionDenied(PermissionError):
    """
    Exception raised by an authorizer if the user has insufficient permissions
    to view a specific part of the repository.
    """
    def __str__(self):
        return self.action

class Authorizer(object):
    """
    Base class for authorizers that are responsible to granting or denying
    access to view certain parts of a repository.
    """

    def assert_permission(self, path):
        if not self.has_permission(path):
            raise PermissionDenied, \
                  'Insufficient permissions to access %s' % path

    def assert_permission_for_changeset(self, rev):
        if not self.has_permission_for_changeset(rev):
            raise PermissionDenied, \
                  'Insufficient permissions to access changeset %s' % rev

    def has_permission(self, path):
        return True

    def has_permission_for_changeset(self, rev):
        return True
<|endoftext|>


<commit_before>import re

from fabric.api import env, hide

from ...fabutils import needs_packages, run, task
from ...utils import flo, query_input, query_yes_no

@task
@needs_packages('nginx', 'postgresql', 'python-pip', 'git')
def trac():
    '''Set up a trac project.

    This trac installation uses python2, git, sqlite (trac-default), gunicorn,
    and nginx.

    More infos:
     * https://trac.edgewall.org/wiki/TracInstall
     * https://trac.edgewall.org/wiki/TracFastCgi#NginxConfiguration
     * https://trac.edgewall.org/wiki/TracNginxRecipe
     * https://trac.edgewall.org/wiki/Gunicorn
     * http://www.obeythetestinggoat.com/book/chapter_08.html#_getting_to_a_production_ready_deployment
    '''
    run('sudo pip install --upgrade virtualenv')

    hostname = re.sub(r'^[^@]+@', '', env.host)  # without username if any
    sitename = query_input(
                   question='\nEnter site-name of Your trac web service',
                   default=flo('trac.{hostname}'))
    username = env.user
    site_dir = flo('/home/{username}/sites/{sitename}')
    run('mkdir -p {site_dir}')
    python_version = 'python2'  # FIXME take latest python via pyenv
    run(flo('virtualenv --python={python_version}  {site_dir}/virtualenv'))

    bin_dir = flo('{site_dir}/virtualenv/bin')
    run(flo('{bin_dir}/pip install --upgrade  genshi trac'))

    if query_yes_no('Restore trac environment from backup tarball?', default=None):
        # FIXME stop trac is running already
        filename = query_input('tarball path?')
        run(flo('mkdir -p {site_dir}/tmp'))
        run(flo('tar xf {filename}  --directory={site_dir}/tmp'))
        run(flo('[ -d {site_dir}/tracenv ] && mv {site_dir}/tracenv  {site_dir}/tracenv.bak'))
        run(flo('mv {site_dir}/tmp/tracenv_hotcopy  {site_dir}/tracenv'))
        run(flo('rmdir {site_dir}/tmp'))
        run(flo('{bin_dir}/trac-admin {site_dir}/tracenv  upgrade'))
        run(flo('{bin_dir}/trac-admin {site_dir}/tracenv  wiki upgrade'))
    elif query_yes_no('Create a new trac environment?', default=None):
        run(flo('{bin_dir}/trac-admin  {site_dir}/tracenv  initenv'))

    # test-run:
    run(flo('{bin_dir}/tracd --port 8000 {site_dir}/tracenv'))

<commit_msg>task setup.service.trac: Refactored provisioning steps into several functions<commit_after>import re

from fabric.api import env, hide

from ...fabutils import needs_packages, run, task
from ...utils import flo, query_input, query_yes_no

@task
@needs_packages('nginx', 'postgresql', 'python-pip', 'git')
def trac():
    '''Set up a trac project.

    This trac installation uses python2, git, sqlite (trac-default), gunicorn,
    and nginx.

    More infos:
     * https://trac.edgewall.org/wiki/TracInstall
     * https://trac.edgewall.org/wiki/TracFastCgi#NginxConfiguration
     * https://trac.edgewall.org/wiki/TracNginxRecipe
     * https://trac.edgewall.org/wiki/Gunicorn
     * http://www.obeythetestinggoat.com/book/chapter_08.html#_getting_to_a_production_ready_deployment
    '''
    hostname = re.sub(r'^[^@]+@', '', env.host)  # without username if any
    sitename = query_input(
                   question='\nEnter site-name of Your trac web service',
                   default=flo('trac.{hostname}'))
    username = env.user

    site_dir = flo('/home/{username}/sites/{sitename}')
    bin_dir = flo('{site_dir}/virtualenv/bin')

    install_upgrade_virtualenv()
    create_directory_structure(site_dir)
    create_virtualenv(site_dir)

    if query_yes_no('Restore trac environment from backup tarball?',
                    default=None):
        restore_tracenv_from_backup_tarball(site_dir, bin_dir)
    elif query_yes_no('Create a new trac environment?', default=None):
        init_tracenv(site_dir, bin_dir)

    # FIXME test-run:
    run(flo('{bin_dir}/tracd --port 8000 {site_dir}/tracenv'))

def install_upgrade_virtualenv():
    run('sudo pip install --upgrade virtualenv')

def create_directory_structure(site_dir):
    run('mkdir -p {site_dir}')

def create_virtualenv(site_dir):
    python_version = 'python2'  # FIXME take latest python via pyenv
    run(flo('virtualenv --python={python_version}  {site_dir}/virtualenv'))
    run(flo('{site_dir}/virtualenv/bin/'
            'pip install --upgrade  genshi trac gunicorn'))

def restore_tracenv_from_backup_tarball(site_dir, bin_dir):
    # FIXME stop trac is running already
    filename = query_input('tarball path?')
    run(flo('mkdir -p {site_dir}/tmp'))
    run(flo('tar xf {filename}  --directory={site_dir}/tmp'))
    run(flo('[ -d {site_dir}/tracenv ] && '
            'mv {site_dir}/tracenv  {site_dir}/tracenv.bak'))
    run(flo('mv {site_dir}/tmp/tracenv_hotcopy  {site_dir}/tracenv'))
    run(flo('rmdir {site_dir}/tmp'))
    run(flo('{bin_dir}/trac-admin {site_dir}/tracenv  upgrade'))
    run(flo('{bin_dir}/trac-admin {site_dir}/tracenv  wiki upgrade'))

def init_tracenv(site_dir, bin_dir):
    run(flo('{bin_dir}/trac-admin  {site_dir}/tracenv  initenv'))
<|endoftext|>


<commit_before># Copyright (c) 2017 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.
from hamcrest import *
import unittest

from ...test.hamcrest import evaluates_to
from .matchers import *
from ..mutable_tree import MutableTree

class TreeHelpers:

    def new_tree (self, value = None):
        return MutableTree(value=value)

    def set_value (self, value):
        self.tree.value = value

    def assert_tree (self, matcher):
        assert_that(self.tree, matcher)

class GivenEmptyTree (TreeHelpers):

    def setUp (self):
        self.tree = self.new_tree()

class GivenTreeWithOneEmptyChild (GivenEmptyTree):

    def setUp (self):
        super().setUp()

        self.first_child = self.new_tree(1)
        self.tree.insert(0, self.first_child)

    def test_first_child_has_value_of_1 (self):
        assert_that(self.first_child, has_node_value(1))

class GivenTreeWithTwoEmptyChildren (GivenTreeWithOneEmptyChild):

    def setUp (self):
        super().setUp()

        self.second_child = self.new_tree(2)
        self.tree.insert(1, self.second_child)

    def test_second_child_has_value_of_2 (self):
        assert_that(self.second_child, has_node_value(2))

class TestGivenNothing (unittest.TestCase):

    def test_can_init_tree_with_value (self):
        tree = MutableTree(value="hi")
        assert_that(tree, has_node_value("hi"))

class TestEmptyTree (GivenEmptyTree, unittest.TestCase):

    def test_evaluates_to_false (self):
        self.assert_tree(evaluates_to(False))

    def test_has_length_of_0 (self):
        self.assert_tree(has_length(0))

    def test_has_full_length_of_0 (self):
        self.assert_tree(has_full_length(0))

    def test_iterates_into_empty_list (self):
        self.assert_tree(iterates_into_list([]))

    def test_walk_iterates_into_empty_list (self):
        self.assert_tree(walks_into_list([]))

    def test_has_no_first_item (self):
        assert_that(calling(lambda x: x[0]).with_args(self.tree),
                    raises(IndexError))

    def test_value_is_none (self):
        self.assert_tree(has_node_value(None))

    def test_can_modify_value (self):
        self.set_value("hello")
        self.assert_tree(has_node_value("hello"))

        self.set_value(235813)
        self.assert_tree(has_node_value(235813))

    def test_cannot_delete_value (self):
        assert_that(calling(delattr).with_args(self.tree, "value"),
                    raises(AttributeError))

    def test_when_value_changes_it_still_cannot_be_deleted (self):
        self.set_value("hello")
        assert_that(calling(delattr).with_args(self.tree, "value"),
                    raises(AttributeError))

class TestTreeWithOneEmptyChild (GivenTreeWithOneEmptyChild,
                                 unittest.TestCase):

    def test_evaluates_to_true (self):
        self.assert_tree(evaluates_to(True))

    def test_value_is_none (self):
        self.assert_tree(has_node_value(None))

    def test_has_length_of_1 (self):
        self.assert_tree(has_length(1))

    def test_has_full_length_of_1 (self):
        self.assert_tree(has_full_length(1))

    def test_first_item_is_child (self):
        assert_that(self.tree[0], is_(same_instance(self.first_child)))

    def test_there_is_no_second_item (self):
        assert_that(calling(lambda x: x[1]).with_args(self.tree),
                    raises(IndexError))

    def test_iterates_into_list_with_child (self):
        self.assert_tree(iterates_into_list([self.first_child]))

    def test_walks_into_list_with_child (self):
        self.assert_tree(walks_into_list([self.first_child]))

    def test_child_has_length_of_0 (self):
        assert_that(self.first_child, has_length(0))

    def test_child_has_full_length_of_0 (self):
        assert_that(self.first_child, has_full_length(0))

class TestTreeWithTwoEmptyChildren (GivenTreeWithTwoEmptyChildren,
                                    unittest.TestCase):

    def test_evaluates_to_true (self):
        self.assert_tree(evaluates_to(True))

    @unittest.skip
    def test_has_length_of_2 (self):
        self.assert_tree(has_length(2))
<commit_msg>Test passes<commit_after># Copyright (c) 2017 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.
from hamcrest import *
import unittest

from ...test.hamcrest import evaluates_to
from .matchers import *
from ..mutable_tree import MutableTree

class TreeHelpers:

    def new_tree (self, value = None):
        return MutableTree(value=value)

    def set_value (self, value):
        self.tree.value = value

    def assert_tree (self, matcher):
        assert_that(self.tree, matcher)

class GivenEmptyTree (TreeHelpers):

    def setUp (self):
        self.tree = self.new_tree()

class GivenTreeWithOneEmptyChild (GivenEmptyTree):

    def setUp (self):
        super().setUp()

        self.first_child = self.new_tree(1)
        self.tree.insert(0, self.first_child)

    def test_first_child_has_value_of_1 (self):
        assert_that(self.first_child, has_node_value(1))

class GivenTreeWithTwoEmptyChildren (GivenTreeWithOneEmptyChild):

    def setUp (self):
        super().setUp()

        self.second_child = self.new_tree(2)
        self.tree.insert(1, self.second_child)

    def test_second_child_has_value_of_2 (self):
        assert_that(self.second_child, has_node_value(2))

class TestGivenNothing (unittest.TestCase):

    def test_can_init_tree_with_value (self):
        tree = MutableTree(value="hi")
        assert_that(tree, has_node_value("hi"))

class TestEmptyTree (GivenEmptyTree, unittest.TestCase):

    def test_evaluates_to_false (self):
        self.assert_tree(evaluates_to(False))

    def test_has_length_of_0 (self):
        self.assert_tree(has_length(0))

    def test_has_full_length_of_0 (self):
        self.assert_tree(has_full_length(0))

    def test_iterates_into_empty_list (self):
        self.assert_tree(iterates_into_list([]))

    def test_walk_iterates_into_empty_list (self):
        self.assert_tree(walks_into_list([]))

    def test_has_no_first_item (self):
        assert_that(calling(lambda x: x[0]).with_args(self.tree),
                    raises(IndexError))

    def test_value_is_none (self):
        self.assert_tree(has_node_value(None))

    def test_can_modify_value (self):
        self.set_value("hello")
        self.assert_tree(has_node_value("hello"))

        self.set_value(235813)
        self.assert_tree(has_node_value(235813))

    def test_cannot_delete_value (self):
        assert_that(calling(delattr).with_args(self.tree, "value"),
                    raises(AttributeError))

    def test_when_value_changes_it_still_cannot_be_deleted (self):
        self.set_value("hello")
        assert_that(calling(delattr).with_args(self.tree, "value"),
                    raises(AttributeError))

class TestTreeWithOneEmptyChild (GivenTreeWithOneEmptyChild,
                                 unittest.TestCase):

    def test_evaluates_to_true (self):
        self.assert_tree(evaluates_to(True))

    def test_value_is_none (self):
        self.assert_tree(has_node_value(None))

    def test_has_length_of_1 (self):
        self.assert_tree(has_length(1))

    def test_has_full_length_of_1 (self):
        self.assert_tree(has_full_length(1))

    def test_first_item_is_child (self):
        assert_that(self.tree[0], is_(same_instance(self.first_child)))

    def test_there_is_no_second_item (self):
        assert_that(calling(lambda x: x[1]).with_args(self.tree),
                    raises(IndexError))

    def test_iterates_into_list_with_child (self):
        self.assert_tree(iterates_into_list([self.first_child]))

    def test_walks_into_list_with_child (self):
        self.assert_tree(walks_into_list([self.first_child]))

    def test_child_has_length_of_0 (self):
        assert_that(self.first_child, has_length(0))

    def test_child_has_full_length_of_0 (self):
        assert_that(self.first_child, has_full_length(0))

class TestTreeWithTwoEmptyChildren (GivenTreeWithTwoEmptyChildren,
                                    unittest.TestCase):

    def test_evaluates_to_true (self):
        self.assert_tree(evaluates_to(True))

    def test_has_length_of_2 (self):
        self.assert_tree(has_length(2))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Usage:
    create room <room_type> <room_name> ...
    add person <first_name> <last_name> <FELLOW>|<STAFF> [<wants_accommodation>]
    print_room <room_name>
    print_allocations <filename>
    print_unallocated <filename>
    reallocate_person <person_identifier> <new_room_name>
    load_people <filename>

Arguments:
    FELLOW|STAFF           Person type to create
    wants_accommodation    Specify if fellow wants a living space
Options:
    -h, --help           : To show the command's help messsage
'''

import os
from docopt import docopt, DocoptExit
import cmd
from models import model
from pyfiglet import figlet_format
from termcolor import cprint, colored
import csv
from views import ui
from core import logic

class App(cmd.Cmd):
    # my shell promt format
    os.system('clear')
    cprint(figlet_format("Office LivingSpace Allocation System " ), 'white', attrs=['bold'])
    prompt = "INPUT $ > "
    dojo = model.Dojo("Andela-Kenya")
    ui.print_welcome()

    def do_top(self, args):
        '''
        Usage:
        top
        '''
        os.system('clear')

    def do_quit(self, args):
        '''
        Usage:
        quit
        '''
        cprint(figlet_format("GoodBye!"), 'white', attrs=['bold'])
        exit(0)

    def do_create_room(self, args):
        """
        Usage:
           create_room <room_type> <room_name>...
        """
        # alist of all status message form create room
        status_messages = []

        try:
            room_information = docopt(self.do_create_room.__doc__, args)
        except DocoptExit as e:
            ui.print_message(e)
        except KeyboardInterrupt:
            pass
        else:
            for name in room_information['<room_name>']:
                try:
                    # call helper to create this
                    status_messages.append(logic.helper_create_and_addroom(App.dojo,room_information['<room_type>'], name))
                except TypeError:
                    # get view for this
                    status_messages.append({'status' : 'fail', 'message' : "Room type: [{}] can not be  created!".format(room_information['<room_type>'])})

        # call ui from views to display our status messages
        for msg in status_messages:
            ui.print_message(msg['message'])

    def do_add_person(self, args):
        """
        Usage:
            add_person <firstname> <secondname> <person_type> [<choice>]
        """
        status_messages = []
        try:
            person_information = docopt(self.do_add_person.__doc__, args)

        except DocoptExit as e:
            ui.print_message(e)
        except KeyboardInterrupt:
            pass
        else:
            firstname = person_information['<firstname>']
            secondname = person_information['<secondname>']
            wants_room = person_information['<choice>']
            person_type = person_information['<person_type>']

            status_messages.append(logic.helper_addsperson_chooseroom(App.dojo,firstname, secondname, person_type, wants_room))

        for messages in status_messages:
            for message in messages['message']:
                ui.print_message(message)

    def do_print_room(self, args):
        """
        Usage:
            print_room <room_name>
        """
        try:
            room_name = docopt(self.do_print_room.__doc__, args)

        except DocoptExit as e:
            ui.print_message(e)
        except KeyboardInterrupt:
            pass
        else:
            ui.print_message("Room :" + room_name['<room_name>'])
            ui.print_message("_" * len("Room :" + room_name['<room_name>']))
            try:
                occupants = logic.people_inroom(App.dojo, room_name['<room_name>'])
                if len(occupants) > 0:
                    ui.print_room_members(occupants)
                else:
                    ui.print_message("Empty room :-( ")
            except logic.NotFoundException:
                ui.print_message("Room Not Found :-( ")
            ui.print_message(" ")

    def do_print_allocations(self, args):
        """
        Usage:
            print_allocations [<-o=FILE>]
        """
        try:
            file_name = docopt(self.do_print_allocations.__doc__, args)

        except DocoptExit as e:
            ui.print_message(e)
        except KeyboardInterrupt:
            pass
        else:
            allocations = logic.dict_allocations(App.dojo)
            if file_name['<-o=FILE>']:
                mode = 'wt'
                for raw_data in allocations.values():
                    logic.save_data_txt(file_name['<-o=FILE>'], raw_data, mode)
                    mode = 'at'
                ui.print_message("Data saved succefully to file: " + file_name['<-o=FILE>'])
            else:
                empty = True
                for room_name in allocations:
                    empty = False
                    ui.print_message("Room :" + room_name)
                    ui.print_message("_" * len("Room :" + room_name))

                    if len(allocations[room_name]) > 0:
                        ui.print_room_members(allocations[room_name])
                    else:
                        ui.print_message("Empty room :-( ")
                if empty:
                    ui.print_message("No Allocations are available")

    def do_print_unallocated(self, args):
        """
        Usage:
            print_unallocations [<-o=FILE>]
        """
        try:
            file_name = docopt(self.do_print_unallocated.__doc__, args)

        except DocoptExit as e:
            ui.print_message(e)
            # call view to display Error message
        except KeyboardInterrupt:
            pass
        else:
            unallocated_person = logic.list_unallocated(App.dojo)
            if file_name['<-o=FILE>']:
                logic.save_data_txt(file_name['<-o=FILE>'], unallocated_person)
                ui.print_message("Data saved succefully to file: " + file_name['<-o=FILE>'])
            else:
                ui.print_message("*" * 40)
                ui.print_message('      LIST OF UNALLOCATED PEOPLE')
                ui.print_message("*" * 40)

                # build display message
                if len(unallocated_person) > 0:
                    user_info = ''
                    for person in unallocated_person:
                        user_info = person.name.upper()
                        if isinstance(person, model.Fellow):
                            wants_living = 'N'
                            user_info += "  FELLOW "
                            if person.wants_living:
                                wants_living = 'Y'
                            user_info += wants_living
                        else:
                            user_info += "  STAFFF   "
                        ui.print_not_allocated(user_info)
                else:
                    ui.print_message("Every one is allocated ;-)")

    def do_reallocate_person(self, args):
        """
        Usage:
            reallocate_person <person_id> <new_room_name>
        """
        try:
            reallocate_information = docopt(self.do_reallocate_person.__doc__, args)
        except DocoptExit as e:
            ui.print_message(e)
        except KeyboardInterrupt:
            pass
        else:
            room_name = reallocate_information['<new_room_name>']
            person_id = reallocate_information['<person_id>']
            status = logic.reallocate_person(room_name, person_id, App.dojo)
            ui.print_message(status)

    def do_load_people(self, args):
        """
        Usage:
            load_people <file_name>
        """
        try:
            file_name = docopt(self.do_load_people.__doc__, args)

        except DocoptExit as e:
            ui.print_message(e)
        except KeyboardInterrupt:
            pass
        else:
            file_name = file_name['<file_name>']
            status_messages = logic.load_data_txt(file_name, App.dojo)

            for status in status_messages:
                for message in status['message']:
                    ui.print_message(message)
                print()

if __name__ == '__main__':
    print(__doc__)
    App().cmdloop()
<commit_msg>Chore: Refactored code create room improve readability<commit_after># -*- coding: utf-8 -*-
import os
from docopt import docopt, DocoptExit
import cmd
from models import model
from pyfiglet import figlet_format
from views import ui, template
from core import logic

class App(cmd.Cmd):
    # my shell promt format
    prompt = "INPUT $ > "
    dojo = model.Dojo("Andela-Kenya")
    ui.print_welcome()
    ui.print_usage()

    def do_top(self, args):
        '''
        Usage:
        top
        '''
        ui.clear_console()

    def do_quit(self, args):
        '''
        Usage:
        quit
        '''
        ui.print_exit()
        exit(0)

    def do_create_room(self, args):
        """
        Usage:
           create_room <room_type> <room_name>...
        """
        try:
            room_information = docopt(self.do_create_room.__doc__, args)
        except DocoptExit as e:
            ui.print_message(e)
        else:
            room_type = room_information['<room_type>']
            for name in room_information['<room_name>']:
                status_message = logic.helper_create_and_addroom(App.dojo, room_type, name)

                # call ui from views to display our status messages
                room_type = status_message['room_type']
                room_name = status_message['room_name']
                status = status_message['status']
                messsage = None
                color = None
                fail = True
                if status_message['status'] == 'ok':
                    message = template.room_created_message
                    fail = False
                elif status_message['status'] == 'failed':
                    message = template.room_typeerror_message
                elif status_message['status'] == 'Duplicate name':
                    message = template.room_dup_name_message
                elif status_message['status'] == 'Invalid name':
                    message = template.room_not_created_message
                ui.print_template_room(message, room_type, room_name, fail=fail, status=status)

    def do_add_person(self, args):
        """
        Usage:
            add_person <firstname> <secondname> <person_type> [<choice>]
        """
        status_messages = []
        try:
            person_information = docopt(self.do_add_person.__doc__, args)

        except DocoptExit as e:
            ui.print_message(e)
        else:
            firstname = person_information['<firstname>']
            secondname = person_information['<secondname>']
            wants_room = person_information['<choice>']
            person_type = person_information['<person_type>']

            status_messages.append(logic.helper_addsperson_chooseroom(App.dojo,firstname, secondname, person_type, wants_room))

        for messages in status_messages:
            for message in messages['message']:
                ui.print_message(message)

    def do_print_room(self, args):
        """
        Usage:
            print_room <room_name>
        """
        try:
            room_name = docopt(self.do_print_room.__doc__, args)
        except DocoptExit as e:
            ui.print_message(e)
        else:
            ui.print_message("Room :" + room_name['<room_name>'])
            ui.print_message("_" * len("Room :" + room_name['<room_name>']))
            try:
                occupants = logic.people_inroom(App.dojo, room_name['<room_name>'])
                if len(occupants) > 0:
                    ui.print_room_members(occupants)
                else:
                    ui.print_message("Empty room :-( ")
            except logic.NotFoundException:
                ui.print_message("Room Not Found :-( ")
            ui.print_message(" ")

    def do_print_allocations(self, args):
        """
        Usage:
            print_allocations [<-o=FILE>]
        """
        try:
            file_name = docopt(self.do_print_allocations.__doc__, args)

        except DocoptExit as e:
            ui.print_message(e)
        else:
            allocations = logic.dict_allocations(App.dojo)
            if file_name['<-o=FILE>']:
                mode = 'wt'
                for raw_data in allocations.values():
                    logic.save_data_txt(file_name['<-o=FILE>'], raw_data, mode)
                    mode = 'at'
                ui.print_message("Data saved succefully to file: " + file_name['<-o=FILE>'])
            else:
                empty = True
                for room_name in allocations:
                    empty = False
                    ui.print_message("Room :" + room_name)
                    ui.print_message("_" * len("Room :" + room_name))

                    if len(allocations[room_name]) > 0:
                        ui.print_room_members(allocations[room_name])
                    else:
                        ui.print_message("Empty room :-( ")
                if empty:
                    ui.print_message("No Allocations are available")

    def do_print_unallocated(self, args):
        """
        Usage:
            print_unallocations [<-o=FILE>]
        """
        try:
            file_name = docopt(self.do_print_unallocated.__doc__, args)

        except DocoptExit as e:
            ui.print_message(e)
            # call view to display Error message
        except KeyboardInterrupt:
            pass
        else:
            unallocated_person = logic.list_unallocated(App.dojo)
            if file_name['<-o=FILE>']:
                logic.save_data_txt(file_name['<-o=FILE>'], unallocated_person)
                ui.print_message("Data saved succefully to file: " + file_name['<-o=FILE>'])
            else:
                ui.print_message("*" * 40)
                ui.print_message('      LIST OF UNALLOCATED PEOPLE')
                ui.print_message("*" * 40)

                # build display message
                if len(unallocated_person) > 0:
                    user_info = ''
                    for person in unallocated_person:
                        user_info = person.name.upper()
                        if isinstance(person, model.Fellow):
                            wants_living = 'N'
                            user_info += "  FELLOW "
                            if person.wants_living:
                                wants_living = 'Y'
                            user_info += wants_living
                        else:
                            user_info += "  STAFFF   "
                        ui.print_not_allocated(user_info)
                else:
                    ui.print_message("Every one is allocated ;-)")

    def do_reallocate_person(self, args):
        """
        Usage:
            reallocate_person <person_id> <new_room_name>
        """
        try:
            reallocate_information = docopt(self.do_reallocate_person.__doc__, args)
        except DocoptExit as e:
            ui.print_message(e)
        except KeyboardInterrupt:
            pass
        else:
            room_name = reallocate_information['<new_room_name>']
            person_id = reallocate_information['<person_id>']
            status = logic.reallocate_person(room_name, person_id, App.dojo)
            ui.print_message(status)

    def do_load_people(self, args):
        """
        Usage:
            load_people <file_name>
        """
        try:
            file_name = docopt(self.do_load_people.__doc__, args)

        except DocoptExit as e:
            ui.print_message(e)
        except KeyboardInterrupt:
            pass
        else:
            file_name = file_name['<file_name>']
            status_messages = logic.load_data_txt(file_name, App.dojo)

            for status in status_messages:
                for message in status['message']:
                    ui.print_message(message)
                print()

if __name__ == '__main__':
    App().cmdloop()
<|endoftext|>


<commit_before><commit_msg>first attempt at using redis for freqdist storage<commit_after><|endoftext|>


<commit_before># No shebang line, this module is meant to be imported
#
# This file is part of PyFarm.
# Copyright (C) 2008-2012 Oliver Palmer
#
# PyFarm is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyFarm is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with PyFarm.  If not, see <http://www.gnu.org/licenses/>.

'''Provides functions and classes for transaction management'''

import time

import sqlalchemy
import sqlalchemy.orm
import sqlalchemy.schema
from twisted.python import log

from pyfarm import logger, prefs
from pyfarm.db import session

class Transaction(object):
    '''
    context manager for transactions

    :param sqlalchemy.Table table:
        the table to perform the query on

    :param object base:
        optional base to map query onto
    '''
    def __init__(self, table, base=None, system=None):
        class BaseObject(object):
            pass
        # end BaseOjbect
        self.__system = system

        if not isinstance(table, sqlalchemy.schema.Table):
            raise TypeError("unexpected type %s for table" % type(table))

        self.base = base or BaseObject
        self.table = table
        self.tablename = self.table.fullname

        # single engine per run (this MAY be needed later)
        self.engine = session.ENGINE
        self.Session = sqlalchemy.orm.scoped_session(session.Session)
#        sqlalchemy.orm.sessionmaker()

    # end __init__

    def log(self, msg, level='SQL'):
        log.msg(msg, level=level, system=self.__system or 'Transaction')
    # end

    def __enter__(self):
        self.start = time.time()
        self.log("opening database transaction on %s" % self.tablename)

        # map the object and prepare the session
        sqlalchemy.orm.mapper(self.base, self.table)
        self.session = self.Session()
        self.query = self.session.query(self.base)

        return self
    # end __enter__

    def __exit__(self, exc_type, exc_val, exc_tb):
        # roll back the transaction in the event of an error
        if exc_type is not None:
            self.log("rolling back database transaction: %s" % exc_val)
            self.session.rollback()

        # commit changes if there are any pending
        else:
            if self.session.dirty or self.session.new:
                self.session.commit()
                log.msg("committing database entry to %s" % self.tablename)

        # close the session
        self.query.session.close()

        # close the database connection
        if prefs.get('database.setup.close-connections'):
            self.log("closing connections")
            self.query.session.bind.dispose()

        self.end = time.time()
        args = (self.tablename, self.end-self.start)
        self.log("closed database transaction on %s (%ss)" % args)
    # end __exit__
# end Transaction

class Connection(object):
    '''manages a single connection to the database'''
    def __init__(self, connection=None):
        self.connection = connection
    # end __init__

    def __enter__(self):
        if self.connection is None:
            self.connection = session.ENGINE.connect()

        return self.connection
    # end __enter__

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.connection.close()
        session.ENGINE.dispose()
    # end __exit__
# end Connection
<commit_msg>comment fix<commit_after># No shebang line, this module is meant to be imported
#
# This file is part of PyFarm.
# Copyright (C) 2008-2012 Oliver Palmer
#
# PyFarm is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyFarm is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with PyFarm.  If not, see <http://www.gnu.org/licenses/>.

'''Provides functions and classes for transaction management'''

import time

import sqlalchemy
import sqlalchemy.orm
import sqlalchemy.schema
from twisted.python import log

from pyfarm import logger, prefs
from pyfarm.db import session

class Transaction(object):
    '''
    context manager for transactions

    :param sqlalchemy.Table table:
        the table to perform the query on

    :param object base:
        optional base to map query onto
    '''
    def __init__(self, table, base=None, system=None):
        class BaseObject(object):
            pass
        # end BaseOjbect
        self.__system = system

        if not isinstance(table, sqlalchemy.schema.Table):
            raise TypeError("unexpected type %s for table" % type(table))

        self.base = base or BaseObject
        self.table = table
        self.tablename = self.table.fullname

        # single engine per run (this MAY be needed later)
        self.engine = session.ENGINE
        self.Session = sqlalchemy.orm.scoped_session(session.Session)
    # end __init__

    def log(self, msg, level='SQL'):
        log.msg(msg, level=level, system=self.__system or 'Transaction')
    # end log

    def __enter__(self):
        self.start = time.time()
        self.log("opening database transaction on %s" % self.tablename)

        # map the object and prepare the session
        sqlalchemy.orm.mapper(self.base, self.table)
        self.session = self.Session()
        self.query = self.session.query(self.base)

        return self
    # end __enter__

    def __exit__(self, exc_type, exc_val, exc_tb):
        # roll back the transaction in the event of an error
        if exc_type is not None:
            self.log("rolling back database transaction: %s" % exc_val)
            self.session.rollback()

        # commit changes if there are any pending
        else:
            if self.session.dirty or self.session.new:
                self.session.commit()
                log.msg("committing database entry to %s" % self.tablename)

        # close the session
        self.query.session.close()

        # close the database connection
        if prefs.get('database.setup.close-connections'):
            self.log("closing connections")
            self.query.session.bind.dispose()

        self.end = time.time()
        args = (self.tablename, self.end-self.start)
        self.log("closed database transaction on %s (%ss)" % args)
    # end __exit__
# end Transaction

class Connection(object):
    '''manages a single connection to the database'''
    def __init__(self, connection=None):
        self.connection = connection
    # end __init__

    def __enter__(self):
        if self.connection is None:
            self.connection = session.ENGINE.connect()

        return self.connection
    # end __enter__

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.connection.close()
        session.ENGINE.dispose()
    # end __exit__
# end Connection
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Created on 2015-03-04

@author: joschi
"""
from __future__ import print_function
import time as time_lib
from datetime import datetime, timedelta
import shelve
import sys
import os.path
import csv
#import simplejson as json
import json
import BitVector

sys.path.append('..')

import build_dictionary
import opd_get_patient

path_correction = '../'
flush_threshold = 650

from_time = -float('Inf')
to_time = float('Inf')

def toTime(s):
    return int(time_lib.mktime(datetime.strptime(s, "%Y%m%d").timetuple()))

def handleRow(row, id, eventCache):
    obj = {
        "info": [],
        "events": []
    }
    opd_get_patient.handleRow(row, obj)
    eventCache.extend(filter(lambda e: e['time'] >= from_time and e['time'] <= to_time, obj["events"]))

def processFile(inputFile, id_column, cb):
    print("processing file: {0}".format(inputFile), file=sys.stderr)
    id_event_cache = {}

    def handleRows(csvDictReader):
        for row in csvDictReader:
            id = row[id_column]
            if id in id_event_cache:
                eventCache = id_event_cache[id]
            else:
                eventCache = []
            handleRow(row, id, eventCache)
            if len(eventCache) > flush_threshold:
                processDict(eventCache, id)
                eventCache = []
            id_event_cache[id] = eventCache

    def processDict(events, id):
        if len(events) == 0:
            return
        print("processing {1} with {0} events".format(len(events), id), file=sys.stderr)
        obj = {
            "events": events
        }
        dict = {}
        build_dictionary.extractEntries(dict, obj)
        for group in dict.keys():
            cb(id, group, dict[group].keys())

    if inputFile == '-':
        handleRows(csv.DictReader(sys.stdin))
    else:
        with open(inputFile) as csvFile:
            handleRows(csv.DictReader(csvFile))

    num_total = len(id_event_cache.keys())
    num = 0
    last_print = 0
    for id in id_event_cache.keys():
        eventCache = id_event_cache[id]
        processDict(eventCache, id)
        del eventCache[:]
        num += 1
        if num / num_total > last_print + 0.01 or num == num_total:
            last_print = num / num_total
            print("processing file: {0} {1:.2%} complete".format(inputFile, last_print), file=sys.stderr)

def processDirectory(dir, id_column, cb):
    for (_, _, files) in os.walk(dir):
        for file in files:
            if file.endswith(".csv"):
                processFile(dir + '/' + file, id_column, cb)

def getBitVector(vectors, header_list, id):
    if id in vectors:
        bitvec = vectors[id]
    else:
        bitvec = BitVector.BitVector(size=0)
    if len(bitvec) < len(header_list):
        diff = len(header_list) - len(bitvec)
        bitvec = bitvec + BitVector.BitVector(size=diff)
    vectors[id] = bitvec
    return bitvec

def getHead(group, type):
    return group + "__" + type

def processAll(vectors, header_list, path_tuples):
    header = {}

    def handle(id, group, types):
        if "__" in group:
            print("group name is using __: {0}".format(group), file=sys.stderr)
        for type in types:
            head = getHead(group, type)
            if head not in header:
                header[head] = len(header_list)
                header_list.append(head)
        bitvec = getBitVector(vectors, header_list, id)
        for type in types:
            head = getHead(group, type)
            bitvec[header[head]] = True

    id_column = opd_get_patient.input_format["patient_id"]
    for (path, isfile) in path_tuples:
        if isfile:
            processFile(path, id_column, handle)
        else:
            processDirectory(path, id_column, handle)

def printResult(vectors, header_list, delim, quote, out):

    def doQuote(cell):
        cell = str(cell)
        if cell.find(delim) < 0 and cell.find(quote) < 0:
            return cell
        return  quote + cell.replace(quote, quote + quote) + quote

    str = doQuote("id") + delim + delim.join(map(doQuote, header_list))
    print(str, file=out)

    empty = BitVector.BitVector(size=0)
    for id in vectors.keys():
        bitvec = getBitVector(vectors, header_list, id)
        str = doQuote(id) + delim + delim.join(map(doQuote, map(lambda v: 1 if v else 0, bitvec)))
        vectors[id] = empty
        print(str, file=out)

def usage():
    print('usage: {0} [-h] [--from <date>] [--to <date>] [-o <output>] -f <format> -c <config> -- <file or path>...'.format(sys.argv[0]), file=sys.stderr)
    print('-h: print help', file=sys.stderr)
    print('--from <date>: specifies the start date as "YYYYMMDD". can be omitted', file=sys.stderr)
    print('--to <date>: specifies the end date as "YYYYMMDD". can be omitted', file=sys.stderr)
    print('-o <output>: specifies output file. stdout if omitted or "-"', file=sys.stderr)
    print('-f <format>: specifies table format file', file=sys.stderr)
    print('-c <config>: specify config file. "-" uses default settings', file=sys.stderr)
    print('<file or path>: a list of input files or paths containing them. "-" represents stdin', file=sys.stderr)
    exit(1)

if __name__ == '__main__':
    output = '-'
    settings = {
        'delim': ',',
        'quote': '"',
        'filename': build_dictionary.globalSymbolsFile,
        'ndc_prod': build_dictionary.productFile,
        'ndc_package': build_dictionary.packageFile,
        'icd9': build_dictionary.icd9File,
        'ccs_diag': build_dictionary.ccs_diag_file,
        'ccs_proc': build_dictionary.ccs_proc_file
    }
    args = sys.argv[:]
    args.pop(0)
    while args:
        arg = args.pop(0)
        if arg == '--':
            break
        if arg == '-h':
            usage()
        if arg == '--from':
            if not args or args[0] == '--':
                print('--from requires a date', file=sys.stderr)
                usage()
            from_time = toTime(args.pop(0))
        elif arg == '--to':
            if not args or args[0] == '--':
                print('--to requires a date', file=sys.stderr)
                usage()
            to_time = toTime(args.pop(0))
        elif arg == '-f':
            if not args or args[0] == '--':
                print('-f requires format file', file=sys.stderr)
                usage()
            opd_get_patient.read_format(args.pop(0))
        elif arg == '-o':
            if not args or args[0] == '--':
                print('-o requires output file', file=sys.stderr)
                usage()
            output = args.pop(0)
        elif arg == '-c':
            if not args or args[0] == '--':
                print('-c requires argument', file=sys.stderr)
                usage()
            build_dictionary.readConfig(settings, args.pop(0))
        else:
            print('unrecognized argument: ' + arg, file=sys.stderr)
            usage()

    build_dictionary.globalSymbolsFile = path_correction + settings['filename']
    build_dictionary.icd9File = path_correction + settings['icd9']
    build_dictionary.ccs_diag_file = path_correction + settings['ccs_diag']
    build_dictionary.ccs_proc_file = path_correction + settings['ccs_proc']
    build_dictionary.productFile = path_correction + settings['ndc_prod']
    build_dictionary.packageFile = path_correction + settings['ndc_package']
    build_dictionary.reportMissingEntries = False
    build_dictionary.init()

    allPaths = []
    while args:
        path = args.pop(0)
        if os.path.isfile(path) or path == '-':
            allPaths.append((path, True))
        elif os.path.isdir(path):
            allPaths.append((path, False))
        else:
            print('illegal argument: '+path+' is neither file nor directory', file=sys.stderr)

    vectors = {}
    header_list = []
    processAll(vectors, header_list, allPaths)
    if output == '-':
        printResult(vectors, header_list, settings['delim'], settings['quote'], sys.stdout)
    else:
        with open(output, 'w') as file:
            printResult(vectors, header_list, settings['delim'], settings['quote'], file)
<commit_msg>add age etc<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Created on 2015-03-04

@author: joschi
"""
from __future__ import print_function
import time as time_lib
from datetime import datetime, timedelta
import shelve
import sys
import os.path
import csv
#import simplejson as json
import json
import BitVector

sys.path.append('..')

import build_dictionary
import opd_get_patient

path_correction = '../'
flush_threshold = 650

from_time = -float('Inf')
to_time = float('Inf')
age_time = None
age_bin = 7
ignore = {
    "prescribed": True
}

def toTime(s):
    return int(time_lib.mktime(datetime.strptime(s, "%Y%m%d").timetuple()))

def toAge(s):
    # TODO there could be a more precise way
    return datetime.fromtimestamp(toTime(s + "0101")).year - datetime.fromtimestamp(age_time).year

def handleRow(row, id, eventCache, infoCache):
    obj = {
        "info": [],
        "events": []
    }
    opd_get_patient.handleRow(row, obj)
    eventCache.extend(filter(lambda e: e['time'] >= from_time and e['time'] <= to_time and e['group'] not in ignore, obj["events"]))
    for info in obj["info"]:
        if info["id"] == "age":
            try:
                bin = (int(info["value"]) // age_bin) * age_bin
                infoCache.append("age_" + str(bin) + "_" + str(bin + age_bin))
            except ValueError:
                pass
        elif info["id"] == "born":
            try:
                if info["value"] != "N/A" and age_time is not None:
                    bin = (toAge(info["value"]) // age_bin) * age_bin
                    infoCache.append("age_" + str(bin) + "_" + str(bin + age_bin))
            except ValueError:
                pass
        elif info["id"] == "death" and info["value"] != "N/A":
            infoCache.append("dead")
        elif info["id"] == "gender":
            if info["value"] == "M":
                infoCache.append("sex_m")
            elif info["value"] == "F":
                infoCache.append("sex_f")

def processFile(inputFile, id_column, cb):
    print("processing file: {0}".format(inputFile), file=sys.stderr)
    id_event_cache = {}
    id_info_cache = {}

    def handleRows(csvDictReader):
        for row in csvDictReader:
            id = row[id_column]
            if id in id_event_cache:
                eventCache = id_event_cache[id]
            else:
                eventCache = []
            if id in id_info_cache:
                infoCache = id_info_cache[id]
            else:
                infoCache = []
            handleRow(row, id, eventCache, infoCache)
            if len(eventCache) > flush_threshold:
                processDict(eventCache, id)
                eventCache = []
            id_event_cache[id] = eventCache
            if len(infoCache) > 0:
                id_info_cache[id] = infoCache

    def processDict(events, id):
        if len(events) == 0:
            return
        print("processing {1} with {0} events".format(len(events), id), file=sys.stderr)
        obj = {
            "events": events
        }
        dict = {}
        build_dictionary.extractEntries(dict, obj)
        for group in dict.keys():
            cb(id, group, dict[group].keys())

    if inputFile == '-':
        handleRows(csv.DictReader(sys.stdin))
    else:
        with open(inputFile) as csvFile:
            handleRows(csv.DictReader(csvFile))

    num_total = len(id_event_cache.keys())
    num = 0
    last_print = 0
    for id in id_event_cache.keys():
        eventCache = id_event_cache[id]
        processDict(eventCache, id)
        del eventCache[:]
        num += 1
        if num / num_total > last_print + 0.01 or num == num_total:
            last_print = num / num_total
            print("processing file: {0} {1:.2%} complete".format(inputFile, last_print), file=sys.stderr)
    for id in id_info_cache.keys():
        infoCache = id_info_cache[id]
        cb(id, "info", infoCache)

def processDirectory(dir, id_column, cb):
    for (_, _, files) in os.walk(dir):
        for file in files:
            if file.endswith(".csv"):
                processFile(dir + '/' + file, id_column, cb)

def getBitVector(vectors, header_list, id):
    if id in vectors:
        bitvec = vectors[id]
    else:
        bitvec = BitVector.BitVector(size=0)
    if len(bitvec) < len(header_list):
        diff = len(header_list) - len(bitvec)
        bitvec = bitvec + BitVector.BitVector(size=diff)
    vectors[id] = bitvec
    return bitvec

def getHead(group, type):
    return group + "__" + type

def processAll(vectors, header_list, path_tuples):
    header = {}

    def handle(id, group, types):
        if "__" in group:
            print("group name is using __: {0}".format(group), file=sys.stderr)
        for type in types:
            head = getHead(group, type)
            if head not in header:
                header[head] = len(header_list)
                header_list.append(head)
        bitvec = getBitVector(vectors, header_list, id)
        for type in types:
            head = getHead(group, type)
            bitvec[header[head]] = True

    id_column = opd_get_patient.input_format["patient_id"]
    for (path, isfile) in path_tuples:
        if isfile:
            processFile(path, id_column, handle)
        else:
            processDirectory(path, id_column, handle)

def printResult(vectors, header_list, delim, quote, out):

    def doQuote(cell):
        cell = str(cell)
        if cell.find(delim) < 0 and cell.find(quote) < 0:
            return cell
        return  quote + cell.replace(quote, quote + quote) + quote

    str = doQuote("id") + delim + delim.join(map(doQuote, header_list))
    print(str, file=out)

    empty = BitVector.BitVector(size=0)
    for id in vectors.keys():
        bitvec = getBitVector(vectors, header_list, id)
        str = doQuote(id) + delim + delim.join(map(doQuote, map(lambda v: 1 if v else 0, bitvec)))
        vectors[id] = empty
        print(str, file=out)

def usage():
    print('usage: {0} [-h] [--from <date>] [--to <date>] [-o <output>] -f <format> -c <config> -- <file or path>...'.format(sys.argv[0]), file=sys.stderr)
    print('-h: print help', file=sys.stderr)
    print('--age-time <date>: specifies the date to compute the age as "YYYYMMDD". can be omitted', file=sys.stderr)
    print('--from <date>: specifies the start date as "YYYYMMDD". can be omitted', file=sys.stderr)
    print('--to <date>: specifies the end date as "YYYYMMDD". can be omitted', file=sys.stderr)
    print('-o <output>: specifies output file. stdout if omitted or "-"', file=sys.stderr)
    print('-f <format>: specifies table format file', file=sys.stderr)
    print('-c <config>: specify config file. "-" uses default settings', file=sys.stderr)
    print('<file or path>: a list of input files or paths containing them. "-" represents stdin', file=sys.stderr)
    exit(1)

if __name__ == '__main__':
    output = '-'
    settings = {
        'delim': ',',
        'quote': '"',
        'filename': build_dictionary.globalSymbolsFile,
        'ndc_prod': build_dictionary.productFile,
        'ndc_package': build_dictionary.packageFile,
        'icd9': build_dictionary.icd9File,
        'ccs_diag': build_dictionary.ccs_diag_file,
        'ccs_proc': build_dictionary.ccs_proc_file
    }
    args = sys.argv[:]
    args.pop(0)
    while args:
        arg = args.pop(0)
        if arg == '--':
            break
        if arg == '-h':
            usage()
        if arg == '--age-time':
            if not args or args[0] == '--':
                print('--age-time requires a date', file=sys.stderr)
                usage()
            age_time = toTime(args.pop(0))
        elif arg == '--from':
            if not args or args[0] == '--':
                print('--from requires a date', file=sys.stderr)
                usage()
            from_time = toTime(args.pop(0))
        elif arg == '--to':
            if not args or args[0] == '--':
                print('--to requires a date', file=sys.stderr)
                usage()
            to_time = toTime(args.pop(0))
        elif arg == '-f':
            if not args or args[0] == '--':
                print('-f requires format file', file=sys.stderr)
                usage()
            opd_get_patient.read_format(args.pop(0))
        elif arg == '-o':
            if not args or args[0] == '--':
                print('-o requires output file', file=sys.stderr)
                usage()
            output = args.pop(0)
        elif arg == '-c':
            if not args or args[0] == '--':
                print('-c requires argument', file=sys.stderr)
                usage()
            build_dictionary.readConfig(settings, args.pop(0))
        else:
            print('unrecognized argument: ' + arg, file=sys.stderr)
            usage()

    build_dictionary.globalSymbolsFile = path_correction + settings['filename']
    build_dictionary.icd9File = path_correction + settings['icd9']
    build_dictionary.ccs_diag_file = path_correction + settings['ccs_diag']
    build_dictionary.ccs_proc_file = path_correction + settings['ccs_proc']
    build_dictionary.productFile = path_correction + settings['ndc_prod']
    build_dictionary.packageFile = path_correction + settings['ndc_package']
    build_dictionary.reportMissingEntries = False
    build_dictionary.init()

    allPaths = []
    while args:
        path = args.pop(0)
        if os.path.isfile(path) or path == '-':
            allPaths.append((path, True))
        elif os.path.isdir(path):
            allPaths.append((path, False))
        else:
            print('illegal argument: '+path+' is neither file nor directory', file=sys.stderr)

    vectors = {}
    header_list = []
    processAll(vectors, header_list, allPaths)
    if output == '-':
        printResult(vectors, header_list, settings['delim'], settings['quote'], sys.stdout)
    else:
        with open(output, 'w') as file:
            printResult(vectors, header_list, settings['delim'], settings['quote'], file)
<|endoftext|>