<commit_before># Simplified from peyotl.phylesystem.phylesystem_umbrella
#
# A collection id should be a unique "path" string composed
#     of '{ownerid}/{slugified-collection-name}'
#     EXAMPLES: 'jimallman/trees-about-bees'
#               'jimallman/other-interesting-stuff'
#               'kcranston/trees-about-bees'
#               'jimallman/trees-about-bees-2'
#
from peyotl.utility import get_logger
from peyotl.utility.str_util import slugify, \
                                    increment_slug
try:
    from cStringIO import StringIO
except ImportError:
    from io import StringIO
import json
try:
    import anyjson
except:
    class Wrapper(object):
        pass
    anyjson = Wrapper()
    anyjson.loads = json.loads
try:
    from dogpile.cache.api import NO_VALUE
except:
    pass #caching is optional
from peyotl.git_storage import ShardedDocStore, \
                               TypeAwareDocStore
from peyotl.collections.collections_shard import TreeCollectionsShardProxy, \
                                                 TreeCollectionsShard
from peyotl.collections import get_empty_collection
from peyotl.collections.validation import validate_collection
from peyotl.collections.git_actions import TreeCollectionsGitAction
#from peyotl.phylesystem.git_workflows import commit_and_try_merge2master, \
#                                             delete_study, \
#                                             validate_and_convert_nexson
#from peyotl.nexson_validation import ot_validate
from threading import Lock
import os
import re

OWNER_ID_PATTERN =      re.compile(r'^[a-zA-Z0-9-]+$')
COLLECTION_ID_PATTERN = re.compile(r'^[a-zA-Z0-9-]+/[a-z0-9-]+$')
# Allow simple slug-ified strings and slash separator (no whitespace!)

_LOG = get_logger(__name__)

def prefix_from_collection_path(collection_id):
    # The collection id is a sort of "path", e.g. '{ownerid}/{collection-name-as-slug}'
    #   EXAMPLES: 'jimallman/trees-about-bees', 'kcranston/interesting-trees-2'
    # Assume that the ownerid will work as a prefix, esp. by assigning all of a
    # user's collections to a single shard.for grouping in shards
    _LOG.debug('> prefix_from_collection_path(), testing this id: {i}'.format(i=collection_id))
    path_parts = collection_id.split('/')
    _LOG.debug('> prefix_from_collection_path(), found {} path parts'.format(len(path_parts)))
    if len(path_parts) > 1:
        owner_id = path_parts[0]
    elif path_parts[0] == '':
        owner_id = 'anonymous'
    else:
        owner_id = 'anonymous'   # or perhaps None?
    return owner_id

class TreeCollectionStoreProxy(ShardedDocStore):
    '''Proxy for interacting with external resources if given the configuration of a remote TreeCollectionStore
    '''
    def __init__(self, config):
        ShardedDocStore.__init__(self,
                                 prefix_from_doc_id=prefix_from_collection_path)
        for s in config.get('shards', []):
            self._shards.append(TreeCollectionsShardProxy(s))
        d = {}
        for s in self._shards:
            for k in s.doc_index.keys():
                if k in d:
                    raise KeyError('Collection "{i}" found in multiple repos'.format(i=k))
                d[k] = s
        self._doc2shard_map = d

class _TreeCollectionStore(TypeAwareDocStore):
    '''Wrapper around a set of sharded git repos.
    '''
    def __init__(self,
                 repos_dict=None,
                 repos_par=None,
                 with_caching=True,
                 assumed_doc_version=None,
                 git_ssh=None,
                 pkey=None,
                 git_action_class=TreeCollectionsGitAction,
                 mirror_info=None,
                 infrastructure_commit_author='OpenTree API <api@opentreeoflife.org>',
                 **kwargs):
        '''
        Repos can be found by passing in a `repos_par` (a directory that is the parent of the repos)
            or by trusting the `repos_dict` mapping of name to repo filepath.
        `with_caching` should be True for non-debugging uses.
        `assumed_doc_version` is optional. If specified all TreeCollectionStoreShard repos are assumed to store
            files of this version of nexson syntax.
        `git_ssh` is the path of an executable for git-ssh operations.
        `pkey` is the PKEY that has to be in the env for remote, authenticated operations to work
        `git_action_class` is a subclass of GitActionBase to use. the __init__ syntax must be compatible
            with PhylesystemGitAction
        If you want to use a mirrors of the repo for pushes or pulls, send in a `mirror_info` dict:
            mirror_info['push'] and mirror_info['pull'] should be dicts with the following keys:
            'parent_dir' - the parent directory of the mirrored repos
            'remote_map' - a dictionary of remote name to prefix (the repo name + '.git' will be
                appended to create the URL for pushing).
        '''
        TypeAwareDocStore.__init__(self,
                                   prefix_from_doc_id=prefix_from_collection_path,
                                   repos_dict=repos_dict,
                                   repos_par=repos_par,
                                   with_caching=with_caching,
                                   assumed_doc_version=assumed_doc_version,
                                   git_ssh=git_ssh,
                                   pkey=pkey,
                                   git_action_class=TreeCollectionsGitAction,
                                   git_shard_class=TreeCollectionsShard,
                                   mirror_info=mirror_info,
                                   new_doc_prefix=None,
                                   infrastructure_commit_author='OpenTree API <api@opentreeoflife.org>',
                                   **kwargs)

    # rename some generic members in the base class, for clarity and backward compatibility
    @property
    def get_collection_ids(self):
        return self.get_doc_ids
    @property
    def delete_collection(self):
        return self.delete_doc

    def create_git_action_for_new_collection(self, new_collection_id=None):
        '''Checks out master branch of the shard as a side effect'''
        return self._growing_shard.create_git_action_for_new_collection(new_collection_id=new_collection_id)

    def add_new_collection(self,
                           ownerid,
                           json,
                           auth_info,
                           collection_id=None):
        """Validate and save this JSON. Ensure (and return) a unique collection id"""
        collection = self._coerce_json_to_collection(json)
        if collection is None:
            msg = "File failed to parse as JSON:\n{j}".format(j=json)
            raise ValueError(msg)
        if not self._is_valid_collection_json(collection):
            msg = "JSON is not a valid collection:\n{j}".format(j=json)
            raise ValueError(msg)
        if collection_id:
            # try to use this id
            found_ownerid, slug = collection_id.split('/')
            assert found_ownerid == ownerid
        else:
            # extract a working title and "slugify" it
            slug = self._slugify_internal_collection_name(json)
            collection_id = '{i}/{s}'.format(i=ownerid, s=slug)
        # Check the proposed id for uniqueness in any case. Increment until
        # we have a new id, then "reserve" it using a placeholder value.
        with self._index_lock:
            while collection_id in self._doc2shard_map:
                collection_id = increment_slug(collection_id)
            self._doc2shard_map[collection_id] = None
        # pass the id and collection JSON to a proper git action
        new_collection_id = None
        r = None
        try:
            # assign the new id to a shard (important prep for commit_and_try_merge2master)
            gd, new_collection_id = self.create_git_action_for_new_collection(new_collection_id=collection_id)
            try:
                # let's remove the 'url' field; it will be restored when the doc is fetched (via API)
                del collection['url']
                commit_msg = ''  # usually this is set downstream
                # keep it simple (collection is already validated! no annotations needed!)
                r = self.commit_and_try_merge2master(file_content=collection,
                                                     doc_id=new_collection_id,
                                                     auth_info=auth_info,
                                                     parent_sha=None,
                                                     commit_msg=commit_msg,
                                                     merged_sha=None)
            except:
                self._growing_shard.delete_doc_from_index(new_collection_id)
                raise
        except:
            with self._index_lock:
                if new_collection_id in self._doc2shard_map:
                    del self._doc2shard_map[new_collection_id]
            raise
        with self._index_lock:
            self._doc2shard_map[new_collection_id] = self._growing_shard
        return new_collection_id, r

    def update_existing_collection(self,
                                   ownerid,
                                   collection_id=None,
                                   json=None,
                                   auth_info=None,
                                   parent_sha=None,
                                   merged_sha=None):
        """Validate and save this JSON. Ensure (and return) a unique collection id"""
        collection = self._coerce_json_to_collection(json)
        if collection is None:
            msg = "File failed to parse as JSON:\n{j}".format(j=json)
            raise ValueError(msg)
        if not self._is_valid_collection_json(collection):
            msg = "JSON is not a valid collection:\n{j}".format(j=json)
            raise ValueError(msg)
        if not collection_id:
            raise ValueError("Collection id not provided (or invalid)")
        if not self.has_doc(collection_id):
            msg = "Unexpected collection id '{}' (expected an existing id!)".format(collection_id)
            raise ValueError(msg)
        # pass the id and collection JSON to a proper git action
        r = None
        try:
            # remove any 'url' field before saving; it will be restored when the doc is fetched (via API)
            del collection['url']
            commit_msg = ''  # usually this is set downstream
            # keep it simple (collection is already validated! no annotations needed!)
            r = self.commit_and_try_merge2master(file_content=collection,
                                                 doc_id=collection_id,
                                                 auth_info=auth_info,
                                                 parent_sha=parent_sha,
                                                 commit_msg=commit_msg,
                                                 merged_sha=merged_sha)
            # identify shard for this id!?
        except:
            raise
        return r

    def copy_existing_collection(self, ownerid, old_collection_id):
        """Ensure a unique id, whether from the same user or a different one"""
        raise NotImplementedError('TODO')

    def rename_existing_collection(self, ownerid, old_collection_id, new_slug=None):
        """Use slug provided, or use internal name to generate a new id"""
        raise NotImplementedError('TODO')

    def _slugify_internal_collection_name(self, json):
        """Parse the JSON, find its name, return a slug of its name"""
        collection = self._coerce_json_to_collection(json)
        if collection is None:
            return None
        internal_name = collection['name']
        return slugify(internal_name)

    def _is_valid_collection_id(self, test_id):
        """Test for the expected format '{ownerid}/{slug}', return T/F
        N.B. This does not test for a working GitHub username!"""
        return bool(COLLECTION_ID_PATTERN.match(test_id))

    def _is_existing_id(self, test_id):
        """Test to see if this id is non-unique (already exists in a shard)"""
        return test_id in self.get_collection_ids()

    def _is_valid_collection_json(self, json):
        """Call the primary validator for a quick test"""
        collection = self._coerce_json_to_collection(json)
        if collection is None:
            # invalid JSON, definitely broken
            return False
        aa = validate_collection(collection)
        errors = aa[0]
        for e in errors:
            _LOG.debug('> invalid JSON: {m}'.format(m=e.encode('utf-8')))
        if len(errors) > 0:
            return False
        return True

    def _coerce_json_to_collection(self, json):
        """Use to ensure that a JSON string (if found) is parsed to the equivalent dict in python.
        If the incoming value is already parsed, do nothing. If a string fails to parse, return None."""
        if type(json) is dict:
            collection = json
        else:
            try:
                collection = anyjson.loads(json)
            except:
                _LOG.warn('> invalid JSON (failed anyjson parsing)')
                return None
        return collection

_THE_TREE_COLLECTION_STORE = None
def TreeCollectionStore(repos_dict=None,
                        repos_par=None,
                        with_caching=True,
                        assumed_doc_version=None,
                        git_ssh=None,
                        pkey=None,
                        git_action_class=TreeCollectionsGitAction,
                        mirror_info=None,
                        infrastructure_commit_author='OpenTree API <api@opentreeoflife.org>'):
    '''Factory function for a _TreeCollectionStore object.

    A wrapper around the _TreeCollectionStore class instantiation for
    the most common use case: a singleton _TreeCollectionStore.
    If you need distinct _TreeCollectionStore objects, you'll need to
    call that class directly.
    '''
    global _THE_TREE_COLLECTION_STORE
    if _THE_TREE_COLLECTION_STORE is None:
        _THE_TREE_COLLECTION_STORE = _TreeCollectionStore(repos_dict=repos_dict,
                                                          repos_par=repos_par,
                                                          with_caching=with_caching,
                                                          assumed_doc_version=assumed_doc_version,
                                                          git_ssh=git_ssh,
                                                          pkey=pkey,
                                                          git_action_class=git_action_class,
                                                          mirror_info=mirror_info,
                                                          infrastructure_commit_author=infrastructure_commit_author)
    return _THE_TREE_COLLECTION_STORE

<commit_msg>Accept user-supplied commit messages.<commit_after># Simplified from peyotl.phylesystem.phylesystem_umbrella
#
# A collection id should be a unique "path" string composed
#     of '{ownerid}/{slugified-collection-name}'
#     EXAMPLES: 'jimallman/trees-about-bees'
#               'jimallman/other-interesting-stuff'
#               'kcranston/trees-about-bees'
#               'jimallman/trees-about-bees-2'
#
from peyotl.utility import get_logger
from peyotl.utility.str_util import slugify, \
                                    increment_slug
try:
    from cStringIO import StringIO
except ImportError:
    from io import StringIO
import json
try:
    import anyjson
except:
    class Wrapper(object):
        pass
    anyjson = Wrapper()
    anyjson.loads = json.loads
try:
    from dogpile.cache.api import NO_VALUE
except:
    pass #caching is optional
from peyotl.git_storage import ShardedDocStore, \
                               TypeAwareDocStore
from peyotl.collections.collections_shard import TreeCollectionsShardProxy, \
                                                 TreeCollectionsShard
from peyotl.collections import get_empty_collection
from peyotl.collections.validation import validate_collection
from peyotl.collections.git_actions import TreeCollectionsGitAction
#from peyotl.phylesystem.git_workflows import commit_and_try_merge2master, \
#                                             delete_study, \
#                                             validate_and_convert_nexson
#from peyotl.nexson_validation import ot_validate
from threading import Lock
import os
import re

OWNER_ID_PATTERN =      re.compile(r'^[a-zA-Z0-9-]+$')
COLLECTION_ID_PATTERN = re.compile(r'^[a-zA-Z0-9-]+/[a-z0-9-]+$')
# Allow simple slug-ified strings and slash separator (no whitespace!)

_LOG = get_logger(__name__)

def prefix_from_collection_path(collection_id):
    # The collection id is a sort of "path", e.g. '{ownerid}/{collection-name-as-slug}'
    #   EXAMPLES: 'jimallman/trees-about-bees', 'kcranston/interesting-trees-2'
    # Assume that the ownerid will work as a prefix, esp. by assigning all of a
    # user's collections to a single shard.for grouping in shards
    _LOG.debug('> prefix_from_collection_path(), testing this id: {i}'.format(i=collection_id))
    path_parts = collection_id.split('/')
    _LOG.debug('> prefix_from_collection_path(), found {} path parts'.format(len(path_parts)))
    if len(path_parts) > 1:
        owner_id = path_parts[0]
    elif path_parts[0] == '':
        owner_id = 'anonymous'
    else:
        owner_id = 'anonymous'   # or perhaps None?
    return owner_id

class TreeCollectionStoreProxy(ShardedDocStore):
    '''Proxy for interacting with external resources if given the configuration of a remote TreeCollectionStore
    '''
    def __init__(self, config):
        ShardedDocStore.__init__(self,
                                 prefix_from_doc_id=prefix_from_collection_path)
        for s in config.get('shards', []):
            self._shards.append(TreeCollectionsShardProxy(s))
        d = {}
        for s in self._shards:
            for k in s.doc_index.keys():
                if k in d:
                    raise KeyError('Collection "{i}" found in multiple repos'.format(i=k))
                d[k] = s
        self._doc2shard_map = d

class _TreeCollectionStore(TypeAwareDocStore):
    '''Wrapper around a set of sharded git repos.
    '''
    def __init__(self,
                 repos_dict=None,
                 repos_par=None,
                 with_caching=True,
                 assumed_doc_version=None,
                 git_ssh=None,
                 pkey=None,
                 git_action_class=TreeCollectionsGitAction,
                 mirror_info=None,
                 infrastructure_commit_author='OpenTree API <api@opentreeoflife.org>',
                 **kwargs):
        '''
        Repos can be found by passing in a `repos_par` (a directory that is the parent of the repos)
            or by trusting the `repos_dict` mapping of name to repo filepath.
        `with_caching` should be True for non-debugging uses.
        `assumed_doc_version` is optional. If specified all TreeCollectionStoreShard repos are assumed to store
            files of this version of nexson syntax.
        `git_ssh` is the path of an executable for git-ssh operations.
        `pkey` is the PKEY that has to be in the env for remote, authenticated operations to work
        `git_action_class` is a subclass of GitActionBase to use. the __init__ syntax must be compatible
            with PhylesystemGitAction
        If you want to use a mirrors of the repo for pushes or pulls, send in a `mirror_info` dict:
            mirror_info['push'] and mirror_info['pull'] should be dicts with the following keys:
            'parent_dir' - the parent directory of the mirrored repos
            'remote_map' - a dictionary of remote name to prefix (the repo name + '.git' will be
                appended to create the URL for pushing).
        '''
        TypeAwareDocStore.__init__(self,
                                   prefix_from_doc_id=prefix_from_collection_path,
                                   repos_dict=repos_dict,
                                   repos_par=repos_par,
                                   with_caching=with_caching,
                                   assumed_doc_version=assumed_doc_version,
                                   git_ssh=git_ssh,
                                   pkey=pkey,
                                   git_action_class=TreeCollectionsGitAction,
                                   git_shard_class=TreeCollectionsShard,
                                   mirror_info=mirror_info,
                                   new_doc_prefix=None,
                                   infrastructure_commit_author='OpenTree API <api@opentreeoflife.org>',
                                   **kwargs)

    # rename some generic members in the base class, for clarity and backward compatibility
    @property
    def get_collection_ids(self):
        return self.get_doc_ids
    @property
    def delete_collection(self):
        return self.delete_doc

    def create_git_action_for_new_collection(self, new_collection_id=None):
        '''Checks out master branch of the shard as a side effect'''
        return self._growing_shard.create_git_action_for_new_collection(new_collection_id=new_collection_id)

    def add_new_collection(self,
                           ownerid,
                           json,
                           auth_info,
                           collection_id=None,
                           commit_msg=''):
        """Validate and save this JSON. Ensure (and return) a unique collection id"""
        collection = self._coerce_json_to_collection(json)
        if collection is None:
            msg = "File failed to parse as JSON:\n{j}".format(j=json)
            raise ValueError(msg)
        if not self._is_valid_collection_json(collection):
            msg = "JSON is not a valid collection:\n{j}".format(j=json)
            raise ValueError(msg)
        if collection_id:
            # try to use this id
            found_ownerid, slug = collection_id.split('/')
            assert found_ownerid == ownerid
        else:
            # extract a working title and "slugify" it
            slug = self._slugify_internal_collection_name(json)
            collection_id = '{i}/{s}'.format(i=ownerid, s=slug)
        # Check the proposed id for uniqueness in any case. Increment until
        # we have a new id, then "reserve" it using a placeholder value.
        with self._index_lock:
            while collection_id in self._doc2shard_map:
                collection_id = increment_slug(collection_id)
            self._doc2shard_map[collection_id] = None
        # pass the id and collection JSON to a proper git action
        new_collection_id = None
        r = None
        try:
            # assign the new id to a shard (important prep for commit_and_try_merge2master)
            gd, new_collection_id = self.create_git_action_for_new_collection(new_collection_id=collection_id)
            try:
                # let's remove the 'url' field; it will be restored when the doc is fetched (via API)
                del collection['url']
                # keep it simple (collection is already validated! no annotations needed!)
                r = self.commit_and_try_merge2master(file_content=collection,
                                                     doc_id=new_collection_id,
                                                     auth_info=auth_info,
                                                     parent_sha=None,
                                                     commit_msg=commit_msg,
                                                     merged_sha=None)
            except:
                self._growing_shard.delete_doc_from_index(new_collection_id)
                raise
        except:
            with self._index_lock:
                if new_collection_id in self._doc2shard_map:
                    del self._doc2shard_map[new_collection_id]
            raise
        with self._index_lock:
            self._doc2shard_map[new_collection_id] = self._growing_shard
        return new_collection_id, r

    def update_existing_collection(self,
                                   ownerid,
                                   collection_id=None,
                                   json=None,
                                   auth_info=None,
                                   parent_sha=None,
                                   merged_sha=None,
                                   commit_msg=''):
        """Validate and save this JSON. Ensure (and return) a unique collection id"""
        collection = self._coerce_json_to_collection(json)
        if collection is None:
            msg = "File failed to parse as JSON:\n{j}".format(j=json)
            raise ValueError(msg)
        if not self._is_valid_collection_json(collection):
            msg = "JSON is not a valid collection:\n{j}".format(j=json)
            raise ValueError(msg)
        if not collection_id:
            raise ValueError("Collection id not provided (or invalid)")
        if not self.has_doc(collection_id):
            msg = "Unexpected collection id '{}' (expected an existing id!)".format(collection_id)
            raise ValueError(msg)
        # pass the id and collection JSON to a proper git action
        r = None
        try:
            # remove any 'url' field before saving; it will be restored when the doc is fetched (via API)
            del collection['url']
            # keep it simple (collection is already validated! no annotations needed!)
            r = self.commit_and_try_merge2master(file_content=collection,
                                                 doc_id=collection_id,
                                                 auth_info=auth_info,
                                                 parent_sha=parent_sha,
                                                 commit_msg=commit_msg,
                                                 merged_sha=merged_sha)
            # identify shard for this id!?
        except:
            raise
        return r

    def copy_existing_collection(self, ownerid, old_collection_id):
        """Ensure a unique id, whether from the same user or a different one"""
        raise NotImplementedError('TODO')

    def rename_existing_collection(self, ownerid, old_collection_id, new_slug=None):
        """Use slug provided, or use internal name to generate a new id"""
        raise NotImplementedError('TODO')

    def _slugify_internal_collection_name(self, json):
        """Parse the JSON, find its name, return a slug of its name"""
        collection = self._coerce_json_to_collection(json)
        if collection is None:
            return None
        internal_name = collection['name']
        return slugify(internal_name)

    def _is_valid_collection_id(self, test_id):
        """Test for the expected format '{ownerid}/{slug}', return T/F
        N.B. This does not test for a working GitHub username!"""
        return bool(COLLECTION_ID_PATTERN.match(test_id))

    def _is_existing_id(self, test_id):
        """Test to see if this id is non-unique (already exists in a shard)"""
        return test_id in self.get_collection_ids()

    def _is_valid_collection_json(self, json):
        """Call the primary validator for a quick test"""
        collection = self._coerce_json_to_collection(json)
        if collection is None:
            # invalid JSON, definitely broken
            return False
        aa = validate_collection(collection)
        errors = aa[0]
        for e in errors:
            _LOG.debug('> invalid JSON: {m}'.format(m=e.encode('utf-8')))
        if len(errors) > 0:
            return False
        return True

    def _coerce_json_to_collection(self, json):
        """Use to ensure that a JSON string (if found) is parsed to the equivalent dict in python.
        If the incoming value is already parsed, do nothing. If a string fails to parse, return None."""
        if type(json) is dict:
            collection = json
        else:
            try:
                collection = anyjson.loads(json)
            except:
                _LOG.warn('> invalid JSON (failed anyjson parsing)')
                return None
        return collection

_THE_TREE_COLLECTION_STORE = None
def TreeCollectionStore(repos_dict=None,
                        repos_par=None,
                        with_caching=True,
                        assumed_doc_version=None,
                        git_ssh=None,
                        pkey=None,
                        git_action_class=TreeCollectionsGitAction,
                        mirror_info=None,
                        infrastructure_commit_author='OpenTree API <api@opentreeoflife.org>'):
    '''Factory function for a _TreeCollectionStore object.

    A wrapper around the _TreeCollectionStore class instantiation for
    the most common use case: a singleton _TreeCollectionStore.
    If you need distinct _TreeCollectionStore objects, you'll need to
    call that class directly.
    '''
    global _THE_TREE_COLLECTION_STORE
    if _THE_TREE_COLLECTION_STORE is None:
        _THE_TREE_COLLECTION_STORE = _TreeCollectionStore(repos_dict=repos_dict,
                                                          repos_par=repos_par,
                                                          with_caching=with_caching,
                                                          assumed_doc_version=assumed_doc_version,
                                                          git_ssh=git_ssh,
                                                          pkey=pkey,
                                                          git_action_class=git_action_class,
                                                          mirror_info=mirror_info,
                                                          infrastructure_commit_author=infrastructure_commit_author)
    return _THE_TREE_COLLECTION_STORE

<|endoftext|>


<commit_before>from pkg_resources import resource_filename
from pylons import config

import controllers.pokedex
from spline.lib.plugin import PluginBase

def add_routes_hook(map, *args, **kwargs):
    """Hook to inject some of our behavior into the routes configuration."""
    map.connect('/dex/media/*path', controller='dex', action='media')
    map.connect('/dex/pokemon/{name}', controller='dex', action='pokemon')
    map.connect('/dex/pokemon/{name}/flavor', controller='dex', action='pokemon_flavor')

class PokedexPlugin(PluginBase):
    def controllers(self):
        return {
            'dex': controllers.pokedex.PokedexController,
        }

    def template_dirs(self):
        return [ resource_filename(__name__, 'templates') ]

    def static_dirs(self):
        return [ resource_filename(__name__, 'public') ]

    def hooks(self):
        return [
            ('routes_mapping', 3, add_routes_hook),
        ]
<commit_msg>Updated plugin config to give template dirs priorities.  #19<commit_after>from pkg_resources import resource_filename
from pylons import config

import controllers.pokedex
from spline.lib.plugin import PluginBase

def add_routes_hook(map, *args, **kwargs):
    """Hook to inject some of our behavior into the routes configuration."""
    map.connect('/dex/media/*path', controller='dex', action='media')
    map.connect('/dex/pokemon/{name}', controller='dex', action='pokemon')
    map.connect('/dex/pokemon/{name}/flavor', controller='dex', action='pokemon_flavor')

class PokedexPlugin(PluginBase):
    def controllers(self):
        return {
            'dex': controllers.pokedex.PokedexController,
        }

    def template_dirs(self):
        return [
            (resource_filename(__name__, 'templates'), 3)
        ]

    def static_dirs(self):
        return [ resource_filename(__name__, 'public') ]

    def hooks(self):
        return [
            ('routes_mapping', 3, add_routes_hook),
        ]
<|endoftext|>


<commit_before>import unittest
try:
    import spotpy
except ImportError:
    import sys
    sys.path.append(".")
    import spotpy
from spotpy import parameter

#https://docs.python.org/3/library/unittest.html

class TestListParameterDistribution(unittest.TestCase):

    def setUp(self):
        self.values = [1, 2, 3, 4, 5]
        self.list_param = parameter.List('test', self.values)
        self.list_param_repeat = parameter.List('test2', self.values, repeat=True)

    def test_list_is_callable(self):
        self.assertTrue(callable(self.list_param), "List instance should be callable")

    def test_list_gives_throwaway_value_on_first_call(self):
        v = self.list_param()
        self.assertNotEqual(self.values[0], v)

    def test_list_gives_1_value_when_size_is_not_specified(self):
        _ = self.list_param()
        v = self.list_param()
        self.assertEqual(self.values[0], v)

    def test_list_gives_n_values_when_size_is_n(self):
        _ = self.list_param()
        v = self.list_param(len(self.values))
        self.assertEqual(self.values, list(v))

    def test_list_gives_cycled_values_with_repeat(self):
        _ = self.list_param_repeat()
        v1 = self.list_param_repeat()
        for k in range(len(self.values) - 1):
            self.list_param_repeat()

        v2 = self.list_param_repeat()

        self.assertEqual(v1, v2)

    def test_list_gives_cycled_values_with_repeat_and_size(self):
        _ = self.list_param_repeat()
        v1 = self.list_param_repeat(len(self.values))
        v2 = self.list_param_repeat(len(self.values))

        self.assertEqual(list(v1), list(v2))

    def test_list_without_repeat_raises_index_error_on_invalid_extra_call(self):
        _ = self.list_param()
        for _ in range(len(self.values)):
            self.list_param()

        self.assertRaises(IndexError, self.list_param)

    def test_astuple(self):
        _ = self.list_param()
        v, name, step, optguess, minbound, maxbound = self.list_param.astuple()
        self.assertEqual(self.values[0], v)
        self.assertEqual("test", name)

        # the values of step, optguess, minbound and maxbound don't matter

class TestParameterArguments(unittest.TestCase):

    def setUp(self):
        """
        Setup for 2 simple parameter cases
        :return:
        """
        self.classes = [parameter.Uniform, parameter.Triangular]
        self.rndargs = [(10, 20), (10, 15, 20)]

    def test_correct_with_default_step(self):
        for cl, args in zip(self.classes, self.rndargs):
            p_no_name = cl(*args)
            p_with_name = cl(cl.__name__, *args)
            self.assertTrue(10 <= p_no_name.optguess < 20, 'Optguess out of boundaries')
            self.assertTrue(10 <= p_with_name.optguess < 20, 'Optguess out of boundaries')
            self.assertTrue(p_no_name.step < 10, 'Step to large')
            self.assertTrue(p_with_name.step < 10 , 'Step to large')

    def test_correct_with_extra_args(self):
        for cl, args in zip(self.classes, self.rndargs):
            p_no_name = cl(*args, step=1, default=12)
            p_with_name = cl(cl.__name__, *args, step=1, default=12)
            self.assertTrue(p_no_name.optguess == 12, 'Optguess not found from default (name={})'.format(repr(p_no_name.name)))
            self.assertTrue(p_with_name.optguess == 12, 'Optguess not found from default (name={})'.format(repr(p_with_name.name)))
            self.assertTrue(p_no_name.step == 1, 'Step overridden by class (name={})'.format(repr(p_no_name.name)))
            self.assertTrue(p_with_name.step == 1, 'Step overridden by class (name={})'.format(repr(p_with_name.name)))

    def test_too_many_args(self):
        for cl, args in zip(self.classes, self.rndargs):
            # Double definition of step
            with self.assertRaises(TypeError):
                p_no_name = cl(*args, 1, step=1)
            with self.assertRaises(TypeError):
                p_with_name = cl(cl.__name__, *args, 1, step=1)

    def test_too_few_args(self):
        for cl, args in zip(self.classes, self.rndargs):
            # Double definition of step
            with self.assertRaises(TypeError):
                p_no_name = cl(*args[:-1], step=1)

            with self.assertRaises(TypeError):
                p_with_name = cl(cl.__name__, *args[:-1], step=1)

class TestParameterClasses(unittest.TestCase):

    def setUp(self):
        """
        Get all classes from spotpy.parameter module, except special cases
        """
        self.classes = []
        for cname, cls in vars(parameter).items():
            if (cls is type
                    and cls not in [parameter.Base, parameter.List]
                    and issubclass(cls, parameter.Base)):
                self.classes.append(cls)

    def test_create_posargs(self):
        """
        Checks if the right number of arguments is present
        :return:
        """
        for cname, cls in self.classes:
            args = tuple(range(1, len(cls.__rndargs__) + 1)) + (0.01,)
            p = cls(cname, *args)
            self.assertFalse(p.name)
            self.assertTrue(callable(p))
            a = p()
            self.assertTrue(p.step == 0.01,
                            '{} did not receive step as the correct value, check number of arguments'
                            .format(cls.__name__))

    def test_create_kwargs(self):
        """
        Check keyword arguments for distribution function
        :return:
        """
        for cname, cls in self.classes:
            kwargs = dict((kw, i+1) for i, kw in enumerate(cls.__rndargs__))
            p = cls(cname, **kwargs)
            self.assertFalse(p.name)
            self.assertTrue(callable(p))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fixed Py2 syntax error in test, when using positional argument after *args<commit_after>import unittest
try:
    import spotpy
except ImportError:
    import sys
    sys.path.append(".")
    import spotpy
from spotpy import parameter

#https://docs.python.org/3/library/unittest.html

class TestListParameterDistribution(unittest.TestCase):

    def setUp(self):
        self.values = [1, 2, 3, 4, 5]
        self.list_param = parameter.List('test', self.values)
        self.list_param_repeat = parameter.List('test2', self.values, repeat=True)

    def test_list_is_callable(self):
        self.assertTrue(callable(self.list_param), "List instance should be callable")

    def test_list_gives_throwaway_value_on_first_call(self):
        v = self.list_param()
        self.assertNotEqual(self.values[0], v)

    def test_list_gives_1_value_when_size_is_not_specified(self):
        _ = self.list_param()
        v = self.list_param()
        self.assertEqual(self.values[0], v)

    def test_list_gives_n_values_when_size_is_n(self):
        _ = self.list_param()
        v = self.list_param(len(self.values))
        self.assertEqual(self.values, list(v))

    def test_list_gives_cycled_values_with_repeat(self):
        _ = self.list_param_repeat()
        v1 = self.list_param_repeat()
        for k in range(len(self.values) - 1):
            self.list_param_repeat()

        v2 = self.list_param_repeat()

        self.assertEqual(v1, v2)

    def test_list_gives_cycled_values_with_repeat_and_size(self):
        _ = self.list_param_repeat()
        v1 = self.list_param_repeat(len(self.values))
        v2 = self.list_param_repeat(len(self.values))

        self.assertEqual(list(v1), list(v2))

    def test_list_without_repeat_raises_index_error_on_invalid_extra_call(self):
        _ = self.list_param()
        for _ in range(len(self.values)):
            self.list_param()

        self.assertRaises(IndexError, self.list_param)

    def test_astuple(self):
        _ = self.list_param()
        v, name, step, optguess, minbound, maxbound = self.list_param.astuple()
        self.assertEqual(self.values[0], v)
        self.assertEqual("test", name)

        # the values of step, optguess, minbound and maxbound don't matter

class TestParameterArguments(unittest.TestCase):

    def setUp(self):
        """
        Setup for 2 simple parameter cases
        :return:
        """
        self.classes = [parameter.Uniform, parameter.Triangular]
        self.rndargs = [(10, 20), (10, 15, 20)]

    def test_correct_with_default_step(self):
        for cl, args in zip(self.classes, self.rndargs):
            p_no_name = cl(*args)
            p_with_name = cl(cl.__name__, *args)
            self.assertTrue(10 <= p_no_name.optguess < 20, 'Optguess out of boundaries')
            self.assertTrue(10 <= p_with_name.optguess < 20, 'Optguess out of boundaries')
            self.assertTrue(p_no_name.step < 10, 'Step to large')
            self.assertTrue(p_with_name.step < 10 , 'Step to large')

    def test_correct_with_extra_args(self):
        for cl, args in zip(self.classes, self.rndargs):
            p_no_name = cl(*args, step=1, default=12)
            p_with_name = cl(cl.__name__, *args, step=1, default=12)
            self.assertTrue(p_no_name.optguess == 12, 'Optguess not found from default (name={})'.format(repr(p_no_name.name)))
            self.assertTrue(p_with_name.optguess == 12, 'Optguess not found from default (name={})'.format(repr(p_with_name.name)))
            self.assertTrue(p_no_name.step == 1, 'Step overridden by class (name={})'.format(repr(p_no_name.name)))
            self.assertTrue(p_with_name.step == 1, 'Step overridden by class (name={})'.format(repr(p_with_name.name)))

    def test_too_many_args(self):
        for cl, args in zip(self.classes, self.rndargs):
            # Double definition of step
            step_args = args + (1, )
            with self.assertRaises(TypeError):
                p_no_name = cl(*step_args, step=1)
            with self.assertRaises(TypeError):
                p_with_name = cl(cl.__name__, *step_args, step=1)

    def test_too_few_args(self):
        for cl, args in zip(self.classes, self.rndargs):
            # Double definition of step
            with self.assertRaises(TypeError):
                p_no_name = cl(*args[:-1], step=1)

            with self.assertRaises(TypeError):
                p_with_name = cl(cl.__name__, *args[:-1], step=1)

class TestParameterClasses(unittest.TestCase):

    def setUp(self):
        """
        Get all classes from spotpy.parameter module, except special cases
        """
        self.classes = []
        for cname, cls in vars(parameter).items():
            if (cls is type
                    and cls not in [parameter.Base, parameter.List]
                    and issubclass(cls, parameter.Base)):
                self.classes.append(cls)

    def test_create_posargs(self):
        """
        Checks if the right number of arguments is present
        :return:
        """
        for cname, cls in self.classes:
            args = tuple(range(1, len(cls.__rndargs__) + 1)) + (0.01,)
            p = cls(cname, *args)
            self.assertFalse(p.name)
            self.assertTrue(callable(p))
            a = p()
            self.assertTrue(p.step == 0.01,
                            '{} did not receive step as the correct value, check number of arguments'
                            .format(cls.__name__))

    def test_create_kwargs(self):
        """
        Check keyword arguments for distribution function
        :return:
        """
        for cname, cls in self.classes:
            kwargs = dict((kw, i+1) for i, kw in enumerate(cls.__rndargs__))
            p = cls(cname, **kwargs)
            self.assertFalse(p.name)
            self.assertTrue(callable(p))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2008 Tiny SPRL (<http://tiny.be>). All Rights Reserved
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import os, sys, imp
import itertools
from sets import Set

import osv
import tools
import pooler

import netsvc
from osv import fields

import zipfile

logger = netsvc.Logger()

opj = os.path.join

_ad = os.path.abspath(opj(tools.config['root_path'], 'addons'))     # default addons path (base)
ad = os.path.abspath(tools.config['addons_path'])           # alternate addons path

sys.path.insert(1, _ad)
if ad != _ad:
    sys.path.insert(1, ad)

class Graph(dict):

    def addNode(self, name, deps):
        max_depth, father = 0, None
        for n in [Node(x, self) for x in deps]:
            if n.depth >= max_depth:
                father = n
                max_depth = n.depth
        if father:
            father.addChild(name)
        else:
            Node(name, self)

    def __iter__(self):
        level = 0
        done = Set(self.keys())
        while done:
            level_modules = [(name, module) for name, module in self.items() if module.depth==level]
            for name, module in level_modules:
                done.remove(name)
                yield module
            level += 1

class Singleton(object):

    def __new__(cls, name, graph):
        if name in graph:
            inst = graph[name]
        else:
            inst = object.__new__(cls)
            inst.name = name
            graph[name] = inst
        return inst

class Node(Singleton):

    def __init__(self, name, graph):
        self.graph = graph
        if not hasattr(self, 'childs'):
            self.childs = []
        if not hasattr(self, 'depth'):
            self.depth = 0

    def addChild(self, name):
        node = Node(name, self.graph)
        node.depth = self.depth + 1
        if node not in self.childs:
            self.childs.append(node)
        for attr in ('init', 'update', 'demo'):
            if hasattr(self, attr):
                setattr(node, attr, True)
        self.childs.sort(lambda x,y: cmp(x.name, y.name))

    def hasChild(self, name):
        return Node(name, self.graph) in self.childs or \
                bool([c for c in self.childs if c.hasChild(name)])

    def __setattr__(self, name, value):
        super(Singleton, self).__setattr__(name, value)
        if name in ('init', 'update', 'demo'):
            tools.config[name][self.name] = 1
            for child in self.childs:
                setattr(child, name, value)
        if name == 'depth':
            for child in self.childs:
                setattr(child, name, value + 1)

    def __iter__(self):
        return itertools.chain(iter(self.childs), *map(iter, self.childs))

    def __str__(self):
        return self._pprint()

    def _pprint(self, depth=0):
        s = '%s\n' % self.name
        for c in self.childs:
            s += '%s`-> %s' % ('   ' * depth, c._pprint(depth+1))
        return s

def get_module_path(module):
    """Return the path of the given module.
    """

    if os.path.exists(opj(ad, module)) or os.path.exists(opj(ad, '%s.zip' % module)):
        return opj(ad, module)

    if os.path.exists(opj(_ad, module)) or os.path.exists(opj(_ad, '%s.zip' % module)):
        return opj(_ad, module)

    logger.notifyChannel('init', netsvc.LOG_WARNING, 'addon:%s:module not found' % (module,))
    return False
    raise IOError, 'Module not found : %s' % module

def get_module_resource(module, *args):
    """Return the full path of a resource of the given module.

    @param module: the module
    @param args: the resource path components

    @return: absolute path to the resource
    """
    a = get_module_path(module)
    return a and opj(a, *args) or False

def get_modules():
    """Returns the list of module names
    """

    module_list = os.listdir(ad)
    module_names = [os.path.basename(m) for m in module_list]
    module_list += [m for m in os.listdir(_ad) if m not in module_names]

    return module_list

def create_graph(module_list, force=None):
    if not force:
        force=[]
    graph = Graph()
    packages = []

    for module in module_list:
        if module[-4:]=='.zip':
            module = module[:-4]
        try:
            mod_path = get_module_path(module)
            if not mod_path:
                continue
        except IOError:
            continue
        terp_file = get_module_resource(module, '__terp__.py')
        if not terp_file: continue
        if os.path.isfile(terp_file) or zipfile.is_zipfile(mod_path+'.zip'):
            try:
                info = eval(tools.file_open(terp_file).read())
            except:
                logger.notifyChannel('init', netsvc.LOG_ERROR, 'addon:%s:eval file %s' % (module, terp_file))
                raise
            if info.get('installable', True):
                packages.append((module, info.get('depends', []), info))

    current,later = Set([p for p, dep, data in packages]), Set()
    while packages and current > later:
        package, deps, datas = packages[0]

        # if all dependencies of 'package' are already in the graph, add 'package' in the graph
        if reduce(lambda x,y: x and y in graph, deps, True):
            if not package in current:
                packages.pop(0)
                continue
            later.clear()
            current.remove(package)
            graph.addNode(package, deps)
            node = Node(package, graph)
            node.datas = datas
            for kind in ('init', 'demo', 'update'):
                if package in tools.config[kind] or 'all' in tools.config[kind] or kind in force:
                    setattr(node, kind, True)
        else:
            later.add(package)
            packages.append((package, deps, datas))
        packages.pop(0)

    for package in later:
        logger.notifyChannel('init', netsvc.LOG_ERROR, 'addon:%s:Unmet dependency' % package)

    return graph

def init_module_objects(cr, module_name, obj_list):
    pool = pooler.get_pool(cr.dbname)
    logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:creating or updating database tables' % module_name)
    for obj in obj_list:
        if hasattr(obj, 'init'):
            obj.init(cr)
        obj._auto_init(cr, {'module': module_name})
        cr.commit()

def load_module_graph(cr, graph, status=None, **kwargs):
    # **kwargs is passed directly to convert_xml_import
    if not status:
        status={}

    status = status.copy()
    package_todo = []
    statusi = 0
    for package in graph:
        status['progress'] = (float(statusi)+0.1)/len(graph)
        m = package.name
        logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s' % m)
        sys.stdout.flush()
        pool = pooler.get_pool(cr.dbname)
        modules = pool.instanciate(m, cr)

        cr.execute('select id from ir_module_module where name=%s', (m,))
        mid = int(cr.rowcount and cr.fetchone()[0] or 0)

        cr.execute('select state, demo from ir_module_module where id=%d', (mid,))
        (package_state, package_demo) = (cr.rowcount and cr.fetchone()) or ('uninstalled', False)
        idref = {}
        status['progress'] = (float(statusi)+0.4)/len(graph)
        if hasattr(package, 'init') or hasattr(package, 'update') or package_state in ('to install', 'to upgrade'):
            init_module_objects(cr, m, modules)
            for kind in ('init', 'update'):
                for filename in package.datas.get('%s_xml' % kind, []):
                    mode = 'update'
                    if hasattr(package, 'init') or package_state=='to install':
                        mode = 'init'
                    logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:loading %s' % (m, filename))
                    name, ext = os.path.splitext(filename)
                    if ext == '.csv':
                        tools.convert_csv_import(cr, m, os.path.basename(filename), tools.file_open(opj(m, filename)).read(), idref, mode=mode)
                    elif ext == '.sql':
                        queries = tools.file_open(opj(m, filename)).read().split(';')
                        for query in queries:
                            new_query = ' '.join(query.split())
                            if new_query:
                                cr.execute(new_query)
                    else:
                        tools.convert_xml_import(cr, m, tools.file_open(opj(m, filename)), idref, mode=mode, **kwargs)
            if hasattr(package, 'demo') or (package_demo and package_state != 'installed'):
                status['progress'] = (float(statusi)+0.75)/len(graph)
                for xml in package.datas.get('demo_xml', []):
                    name, ext = os.path.splitext(xml)
                    logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:loading %s' % (m, xml))
                    if ext == '.csv':
                        tools.convert_csv_import(cr, m, os.path.basename(xml), tools.file_open(opj(m, xml)).read(), idref, noupdate=True)
                    else:
                        tools.convert_xml_import(cr, m, tools.file_open(opj(m, xml)), idref, noupdate=True, **kwargs)
                cr.execute('update ir_module_module set demo=%s where id=%d', (True, mid))
            package_todo.append(package.name)
            cr.execute("update ir_module_module set state='installed' where state in ('to upgrade', 'to install') and id=%d", (mid,))

            cr.commit()

            # Update translations for all installed languages
            modobj = pool.get('ir.module.module')
            if modobj:
                modobj.update_translations(cr, 1, [mid], None)
                cr.commit()

        statusi+=1

    cr.execute("""select model,name from ir_model where id not in (select model_id from ir_model_access)""")
    for (model,name) in cr.fetchall():
        logger.notifyChannel('init', netsvc.LOG_WARNING, 'addon:object %s (%s) has no access rules!' % (model,name))

    pool = pooler.get_pool(cr.dbname)
    cr.execute('select * from ir_model where state=%s', ('manual',))
    for model in cr.dictfetchall():
        pool.get('ir.model').instanciate(cr, 1, model['model'], {})

    pool.get('ir.model.data')._process_end(cr, 1, package_todo)
    cr.commit()

def register_classes():
    module_list = get_modules()
    for package in create_graph(module_list):
        m = package.name
        logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:registering classes' % m)
        sys.stdout.flush()

        mod_path = get_module_path(m)
        if not os.path.isfile(mod_path+'.zip'):
            # XXX must restrict to only addons paths
            imp.load_module(m, *imp.find_module(m))
        else:
            import zipimport
            try:
                zimp = zipimport.zipimporter(mod_path+'.zip')
                zimp.load_module(m)
            except zipimport.ZipImportError:
                logger.notifyChannel('init', netsvc.LOG_ERROR, 'Couldn\'t find module %s' % m)

def load_modules(db, force_demo=False, status=None, update_module=False):
    if not status:
        status={}
    cr = db.cursor()
    force = []
    if force_demo:
        force.append('demo')
    if update_module:
        for module in tools.config['init']:
            cr.execute('update ir_module_module set state=%s where state=%s and name=%s', ('to install', 'uninstalled', module))
            cr.commit()
        cr.execute("select name from ir_module_module where state in ('installed', 'to install', 'to upgrade','to remove')")
    else:
        cr.execute("select name from ir_module_module where state in ('installed', 'to upgrade', 'to remove')")
    module_list = [name for (name,) in cr.fetchall()]
    graph = create_graph(module_list, force)
    report = tools.assertion_report()
    load_module_graph(cr, graph, status, report=report)
    if report.get_report():
        logger.notifyChannel('init', netsvc.LOG_INFO, 'assert:%s' % report)

    for kind in ('init', 'demo', 'update'):
        tools.config[kind]={}

    cr.commit()
    if update_module:
        cr.execute("select id,name from ir_module_module where state in ('to remove')")
        for mod_id, mod_name in cr.fetchall():
            pool = pooler.get_pool(cr.dbname)
            cr.execute('select model,res_id from ir_model_data where not noupdate and module=%s order by id desc', (mod_name,))
            for rmod,rid in cr.fetchall():
                #
                # TO BE Improved:
                #   I can not use the class_pool has _table could be defined in __init__
                #   and I can not use the pool has the module could not be loaded in the pool
                #
                uid = 1
                pool.get(rmod).unlink(cr, uid, [rid])
            cr.commit()
        #
        # TODO: remove menu without actions of childs
        #
        cr.execute('''delete from
                ir_ui_menu
            where
                (id not in (select parent_id from ir_ui_menu where parent_id is not null))
            and
                (id not in (select res_id from ir_values where model='ir.ui.menu'))
            and
                (id not in (select res_id from ir_model_data where model='ir.ui.menu'))''')

        cr.execute("update ir_module_module set state=%s where state in ('to remove')", ('uninstalled', ))
        cr.commit()
        pooler.restart_pool(cr.dbname)
    cr.close()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<commit_msg>[FIX] error when getting zip module names<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2008 Tiny SPRL (<http://tiny.be>). All Rights Reserved
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import os, sys, imp
import itertools
from sets import Set

import osv
import tools
import pooler

import netsvc
from osv import fields

import zipfile

logger = netsvc.Logger()

opj = os.path.join

_ad = os.path.abspath(opj(tools.config['root_path'], 'addons'))     # default addons path (base)
ad = os.path.abspath(tools.config['addons_path'])           # alternate addons path

sys.path.insert(1, _ad)
if ad != _ad:
    sys.path.insert(1, ad)

class Graph(dict):

    def addNode(self, name, deps):
        max_depth, father = 0, None
        for n in [Node(x, self) for x in deps]:
            if n.depth >= max_depth:
                father = n
                max_depth = n.depth
        if father:
            father.addChild(name)
        else:
            Node(name, self)

    def __iter__(self):
        level = 0
        done = Set(self.keys())
        while done:
            level_modules = [(name, module) for name, module in self.items() if module.depth==level]
            for name, module in level_modules:
                done.remove(name)
                yield module
            level += 1

class Singleton(object):

    def __new__(cls, name, graph):
        if name in graph:
            inst = graph[name]
        else:
            inst = object.__new__(cls)
            inst.name = name
            graph[name] = inst
        return inst

class Node(Singleton):

    def __init__(self, name, graph):
        self.graph = graph
        if not hasattr(self, 'childs'):
            self.childs = []
        if not hasattr(self, 'depth'):
            self.depth = 0

    def addChild(self, name):
        node = Node(name, self.graph)
        node.depth = self.depth + 1
        if node not in self.childs:
            self.childs.append(node)
        for attr in ('init', 'update', 'demo'):
            if hasattr(self, attr):
                setattr(node, attr, True)
        self.childs.sort(lambda x,y: cmp(x.name, y.name))

    def hasChild(self, name):
        return Node(name, self.graph) in self.childs or \
                bool([c for c in self.childs if c.hasChild(name)])

    def __setattr__(self, name, value):
        super(Singleton, self).__setattr__(name, value)
        if name in ('init', 'update', 'demo'):
            tools.config[name][self.name] = 1
            for child in self.childs:
                setattr(child, name, value)
        if name == 'depth':
            for child in self.childs:
                setattr(child, name, value + 1)

    def __iter__(self):
        return itertools.chain(iter(self.childs), *map(iter, self.childs))

    def __str__(self):
        return self._pprint()

    def _pprint(self, depth=0):
        s = '%s\n' % self.name
        for c in self.childs:
            s += '%s`-> %s' % ('   ' * depth, c._pprint(depth+1))
        return s

def get_module_path(module):
    """Return the path of the given module.
    """

    if os.path.exists(opj(ad, module)) or os.path.exists(opj(ad, '%s.zip' % module)):
        return opj(ad, module)

    if os.path.exists(opj(_ad, module)) or os.path.exists(opj(_ad, '%s.zip' % module)):
        return opj(_ad, module)

    logger.notifyChannel('init', netsvc.LOG_WARNING, 'addon:%s:module not found' % (module,))
    return False
    raise IOError, 'Module not found : %s' % module

def get_module_resource(module, *args):
    """Return the full path of a resource of the given module.

    @param module: the module
    @param args: the resource path components

    @return: absolute path to the resource
    """
    a = get_module_path(module)
    return a and opj(a, *args) or False

def get_modules():
    """Returns the list of module names
    """
    def listdir(dir):
        def clean(name):
            name = os.path.basename(name)
            if name[-4:] == '.zip':
                name = name[:-4]
            return name
        return map(clean, os.listdir(dir))

    return list(set(listdir(ad) + listdir(_ad)))

def create_graph(module_list, force=None):
    if not force:
        force=[]
    graph = Graph()
    packages = []

    for module in module_list:
        try:
            mod_path = get_module_path(module)
            if not mod_path:
                continue
        except IOError:
            continue
        terp_file = get_module_resource(module, '__terp__.py')
        if not terp_file: continue
        if os.path.isfile(terp_file) or zipfile.is_zipfile(mod_path+'.zip'):
            try:
                info = eval(tools.file_open(terp_file).read())
            except:
                logger.notifyChannel('init', netsvc.LOG_ERROR, 'addon:%s:eval file %s' % (module, terp_file))
                raise
            if info.get('installable', True):
                packages.append((module, info.get('depends', []), info))

    current,later = Set([p for p, dep, data in packages]), Set()
    while packages and current > later:
        package, deps, datas = packages[0]

        # if all dependencies of 'package' are already in the graph, add 'package' in the graph
        if reduce(lambda x,y: x and y in graph, deps, True):
            if not package in current:
                packages.pop(0)
                continue
            later.clear()
            current.remove(package)
            graph.addNode(package, deps)
            node = Node(package, graph)
            node.datas = datas
            for kind in ('init', 'demo', 'update'):
                if package in tools.config[kind] or 'all' in tools.config[kind] or kind in force:
                    setattr(node, kind, True)
        else:
            later.add(package)
            packages.append((package, deps, datas))
        packages.pop(0)

    for package in later:
        logger.notifyChannel('init', netsvc.LOG_ERROR, 'addon:%s:Unmet dependency' % package)

    return graph

def init_module_objects(cr, module_name, obj_list):
    pool = pooler.get_pool(cr.dbname)
    logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:creating or updating database tables' % module_name)
    for obj in obj_list:
        if hasattr(obj, 'init'):
            obj.init(cr)
        obj._auto_init(cr, {'module': module_name})
        cr.commit()

def load_module_graph(cr, graph, status=None, **kwargs):
    # **kwargs is passed directly to convert_xml_import
    if not status:
        status={}

    status = status.copy()
    package_todo = []
    statusi = 0
    for package in graph:
        status['progress'] = (float(statusi)+0.1)/len(graph)
        m = package.name
        logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s' % m)
        sys.stdout.flush()
        pool = pooler.get_pool(cr.dbname)
        modules = pool.instanciate(m, cr)

        cr.execute('select id from ir_module_module where name=%s', (m,))
        mid = int(cr.rowcount and cr.fetchone()[0] or 0)

        cr.execute('select state, demo from ir_module_module where id=%d', (mid,))
        (package_state, package_demo) = (cr.rowcount and cr.fetchone()) or ('uninstalled', False)
        idref = {}
        status['progress'] = (float(statusi)+0.4)/len(graph)
        if hasattr(package, 'init') or hasattr(package, 'update') or package_state in ('to install', 'to upgrade'):
            init_module_objects(cr, m, modules)
            for kind in ('init', 'update'):
                for filename in package.datas.get('%s_xml' % kind, []):
                    mode = 'update'
                    if hasattr(package, 'init') or package_state=='to install':
                        mode = 'init'
                    logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:loading %s' % (m, filename))
                    name, ext = os.path.splitext(filename)
                    if ext == '.csv':
                        tools.convert_csv_import(cr, m, os.path.basename(filename), tools.file_open(opj(m, filename)).read(), idref, mode=mode)
                    elif ext == '.sql':
                        queries = tools.file_open(opj(m, filename)).read().split(';')
                        for query in queries:
                            new_query = ' '.join(query.split())
                            if new_query:
                                cr.execute(new_query)
                    else:
                        tools.convert_xml_import(cr, m, tools.file_open(opj(m, filename)), idref, mode=mode, **kwargs)
            if hasattr(package, 'demo') or (package_demo and package_state != 'installed'):
                status['progress'] = (float(statusi)+0.75)/len(graph)
                for xml in package.datas.get('demo_xml', []):
                    name, ext = os.path.splitext(xml)
                    logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:loading %s' % (m, xml))
                    if ext == '.csv':
                        tools.convert_csv_import(cr, m, os.path.basename(xml), tools.file_open(opj(m, xml)).read(), idref, noupdate=True)
                    else:
                        tools.convert_xml_import(cr, m, tools.file_open(opj(m, xml)), idref, noupdate=True, **kwargs)
                cr.execute('update ir_module_module set demo=%s where id=%d', (True, mid))
            package_todo.append(package.name)
            cr.execute("update ir_module_module set state='installed' where state in ('to upgrade', 'to install') and id=%d", (mid,))

            cr.commit()

            # Update translations for all installed languages
            modobj = pool.get('ir.module.module')
            if modobj:
                modobj.update_translations(cr, 1, [mid], None)
                cr.commit()

        statusi+=1

    cr.execute("""select model,name from ir_model where id not in (select model_id from ir_model_access)""")
    for (model,name) in cr.fetchall():
        logger.notifyChannel('init', netsvc.LOG_WARNING, 'addon:object %s (%s) has no access rules!' % (model,name))

    pool = pooler.get_pool(cr.dbname)
    cr.execute('select * from ir_model where state=%s', ('manual',))
    for model in cr.dictfetchall():
        pool.get('ir.model').instanciate(cr, 1, model['model'], {})

    pool.get('ir.model.data')._process_end(cr, 1, package_todo)
    cr.commit()

def register_classes():
    module_list = get_modules()
    for package in create_graph(module_list):
        m = package.name
        logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:registering classes' % m)
        sys.stdout.flush()

        mod_path = get_module_path(m)
        if not os.path.isfile(mod_path+'.zip'):
            # XXX must restrict to only addons paths
            imp.load_module(m, *imp.find_module(m))
        else:
            import zipimport
            try:
                zimp = zipimport.zipimporter(mod_path+'.zip')
                zimp.load_module(m)
            except zipimport.ZipImportError:
                logger.notifyChannel('init', netsvc.LOG_ERROR, 'Couldn\'t find module %s' % m)

def load_modules(db, force_demo=False, status=None, update_module=False):
    if not status:
        status={}
    cr = db.cursor()
    force = []
    if force_demo:
        force.append('demo')
    if update_module:
        for module in tools.config['init']:
            cr.execute('update ir_module_module set state=%s where state=%s and name=%s', ('to install', 'uninstalled', module))
            cr.commit()
        cr.execute("select name from ir_module_module where state in ('installed', 'to install', 'to upgrade','to remove')")
    else:
        cr.execute("select name from ir_module_module where state in ('installed', 'to upgrade', 'to remove')")
    module_list = [name for (name,) in cr.fetchall()]
    graph = create_graph(module_list, force)
    report = tools.assertion_report()
    load_module_graph(cr, graph, status, report=report)
    if report.get_report():
        logger.notifyChannel('init', netsvc.LOG_INFO, 'assert:%s' % report)

    for kind in ('init', 'demo', 'update'):
        tools.config[kind]={}

    cr.commit()
    if update_module:
        cr.execute("select id,name from ir_module_module where state in ('to remove')")
        for mod_id, mod_name in cr.fetchall():
            pool = pooler.get_pool(cr.dbname)
            cr.execute('select model,res_id from ir_model_data where not noupdate and module=%s order by id desc', (mod_name,))
            for rmod,rid in cr.fetchall():
                #
                # TO BE Improved:
                #   I can not use the class_pool has _table could be defined in __init__
                #   and I can not use the pool has the module could not be loaded in the pool
                #
                uid = 1
                pool.get(rmod).unlink(cr, uid, [rid])
            cr.commit()
        #
        # TODO: remove menu without actions of childs
        #
        cr.execute('''delete from
                ir_ui_menu
            where
                (id not in (select parent_id from ir_ui_menu where parent_id is not null))
            and
                (id not in (select res_id from ir_values where model='ir.ui.menu'))
            and
                (id not in (select res_id from ir_model_data where model='ir.ui.menu'))''')

        cr.execute("update ir_module_module set state=%s where state in ('to remove')", ('uninstalled', ))
        cr.commit()
        pooler.restart_pool(cr.dbname)
    cr.close()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<|endoftext|>


<commit_before># -*- coding: utf8 -*-
"""
Copyright 2014 Ryan Brown <sb@ryansb.com>

This file is part of taskforge.

taskforge is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of
the License, or (at your option) any later version.

taskforge is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with taskforge.  If not, see <http://www.gnu.org/licenses/>.
"""

import re

import requests

from taskforge.plugin import PluginBase

"""Adapted from https://gist.github.com/uogbuji/705383"""
_URL_REGEX = re.compile(
    ur'(?i)\b((?:https?://)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+'
    ur'(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'".,<>?'
    ur'\xab\xbb\u201c\u201d\u2018\u2019]))')

class Shortener(PluginBase):
    _base = 'http://da.gd/'
    max_length = 24 # default longest non-shortened URL

    def __init__(self, **kwargs):
        self.max_length = kwargs.get('max_length',
                                     self.max_length)

        super(Shortener, self).__init__(**kwargs)

    def pre_run(self):
        pass

    def post_run(self):
        pass

    def _shorten_contents(self, text):
        """Returns a list of URLs in order of appearance

        Will not return if:
          - URL is already shortened
          - URL is shorter than it would be shortened
        """
        urls = [
            m[0] for m in _URL_REGEX.findall(text)
            if len(m[0]) > self.max_length and not m[0].startswith(self._base)
        ]

        for url in urls:
            if re.match('.*{}\.\.\..*'.format(url), text):
                continue
            shortened = self._shorten_url(url)
            self.log.debug("Shortened {} to {}".format(url, shortened))
            text = text.replace(url, shortened)

        return text

    def _shorten_url(self, url):
        r = requests.get(self._base + 's',
                         params={'url': url})
        if r.status_code != 200:
            return None
        return r.text.strip()

    def _shorten_urls(self, desc, annotations):
        """Given arbitrary text, shorten any url-like segments"""
        for index in range(len(annotations)):
            annotations[index] = self._shorten_contents(annotations[index])
        desc = self._shorten_contents(desc)
        return desc, annotations

    def process_task(self, task):
        newDesc, anno = self._shorten_urls(str(task['description']), task.get('annotations', []))
        results = task.update({
                'description': newDesc,
                'annotations': anno,
            })
        if any(results.values()):
            return task
        return None
<commit_msg>Don't update tasks with no annotations<commit_after># -*- coding: utf8 -*-
"""
Copyright 2014 Ryan Brown <sb@ryansb.com>

This file is part of taskforge.

taskforge is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of
the License, or (at your option) any later version.

taskforge is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with taskforge.  If not, see <http://www.gnu.org/licenses/>.
"""

import re

import requests

from taskforge.plugin import PluginBase

"""Adapted from https://gist.github.com/uogbuji/705383"""
_URL_REGEX = re.compile(
    ur'(?i)\b((?:https?://)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+'
    ur'(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'".,<>?'
    ur'\xab\xbb\u201c\u201d\u2018\u2019]))')

class Shortener(PluginBase):
    _base = 'http://da.gd/'
    max_length = 24 # default longest non-shortened URL

    def __init__(self, **kwargs):
        self.max_length = kwargs.get('max_length',
                                     self.max_length)

        super(Shortener, self).__init__(**kwargs)

    def pre_run(self):
        pass

    def post_run(self):
        pass

    def _shorten_contents(self, text):
        """Returns a list of URLs in order of appearance

        Will not return if:
          - URL is already shortened
          - URL is shorter than it would be shortened
        """
        urls = [
            m[0] for m in _URL_REGEX.findall(text)
            if len(m[0]) > self.max_length and not m[0].startswith(self._base)
        ]

        for url in urls:
            if re.match('.*{}\.\.\..*'.format(url), text):
                continue
            shortened = self._shorten_url(url)
            self.log.debug("Shortened {} to {}".format(url, shortened))
            text = text.replace(url, shortened)

        return text

    def _shorten_url(self, url):
        r = requests.get(self._base + 's',
                         params={'url': url})
        if r.status_code != 200:
            return None
        return r.text.strip()

    def _shorten_urls(self, desc, annotations):
        """Given arbitrary text, shorten any url-like segments"""
        for index in range(len(annotations)):
            annotations[index] = self._shorten_contents(annotations[index])
        desc = self._shorten_contents(desc)
        return desc, annotations

    def process_task(self, task):
        newDesc, anno = self._shorten_urls(str(task['description']), task.get('annotations', []))
        if anno:
            results = task.update({
                    'description': newDesc,
                    'annotations': anno,
                })
        else:
            results = task.update({
                    'description': newDesc,
            })
        if any(results.values()):
            return task
        return None
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2016 Andi Albrecht, albrecht.andi@gmail.com
#
# This module is part of python-sqlparse and is released under
# the BSD License: http://www.opensource.org/licenses/bsd-license.php

from sqlparse import sql, tokens as T
from sqlparse.compat import text_type
from sqlparse.utils import offset, indent

class AlignedIndentFilter(object):
    join_words = (r'((LEFT\s+|RIGHT\s+|FULL\s+)?'
                  r'(INNER\s+|OUTER\s+|STRAIGHT\s+)?|'
                  r'(CROSS\s+|NATURAL\s+)?)?JOIN\b')
    split_words = ('FROM',
                   join_words, 'ON',
                   'WHERE', 'AND', 'OR',
                   'GROUP', 'HAVING', 'LIMIT',
                   'ORDER', 'UNION', 'VALUES',
                   'SET', 'BETWEEN', 'EXCEPT')

    def __init__(self, char=' ', line_width=None):
        self.offset = 0
        self.indent = 0
        self.char = char
        self._max_kwd_len = len('select')

    def nl(self, offset=1):
        # offset = 1 represent a single space after SELECT
        offset = -len(offset) if not isinstance(offset, int) else offset
        # add two for the space and parens
        indent = self.indent * (2 + self._max_kwd_len)

        return sql.Token(T.Whitespace, '\n' + self.char * (
            self._max_kwd_len + offset + indent + self.offset))

    def _process_statement(self, tlist):
        if tlist.tokens[0].is_whitespace() and self.indent == 0:
            tlist.tokens.pop(0)

        # process the main query body
        self._process(sql.TokenList(tlist.tokens))

    def _process_parenthesis(self, tlist):
        # if this isn't a subquery, don't re-indent
        if tlist.token_next_by(m=(T.DML, 'SELECT')):
            with indent(self):
                tlist.insert_after(tlist[0], self.nl('SELECT'))
                # process the inside of the parantheses
                self._process_default(tlist)

            # de-indent last parenthesis
            tlist.insert_before(tlist[-1], self.nl())

    def _process_identifierlist(self, tlist):
        # columns being selected
        identifiers = list(tlist.get_identifiers())
        identifiers.pop(0)
        [tlist.insert_before(token, self.nl()) for token in identifiers]
        self._process_default(tlist)

    def _process_case(self, tlist):
        base_offset = len('case ')
        case_offset = len('when ')
        cases = tlist.get_cases(skip_ws=True)
        # align the end as well
        end_token = tlist.token_next_by(m=(T.Keyword, 'END'))
        cases.append((None, [end_token]))

        condition_width = max(
            len(' '.join(map(str, cond))) for cond, value in cases if cond)
        for i, (cond, value) in enumerate(cases):
            if cond is None:  # else or end
                stmt = value[0]
                line = value
            else:
                stmt = cond[0]
                line = cond + value

            if i > 0:
                tlist.insert_before(stmt, self.nl(
                    base_offset + case_offset - len(str(stmt))))
            if cond:
                ws = sql.Token(T.Whitespace, self.char * (
                    condition_width - len(' '.join(map(text_type, cond)))))
                tlist.insert_after(cond[-1], ws)

    def _next_token(self, tlist, idx=0):
        split_words = T.Keyword, self.split_words, True
        token = tlist.token_next_by(m=split_words, idx=idx)
        # treat "BETWEEN x and y" as a single statement
        if token and token.value.upper() == 'BETWEEN':
            token = self._next_token(tlist, token)
            if token and token.value.upper() == 'AND':
                token = self._next_token(tlist, token)
        return token

    def _split_kwds(self, tlist):
        token = self._next_token(tlist)
        while token:
            # joins are special case. only consider the first word as aligner
            if token.match(T.Keyword, self.join_words, regex=True):
                token_indent = token.value.split()[0]
            else:
                token_indent = text_type(token)
            tlist.insert_before(token, self.nl(token_indent))
            token = self._next_token(tlist, token)

    def _process_default(self, tlist):
        self._split_kwds(tlist)
        # process any sub-sub statements
        for sgroup in tlist.get_sublists():
            prev = tlist.token_prev(sgroup)
            # HACK: make "group/order by" work. Longer than max_len.
            offset_ = 3 if (prev and prev.match(T.Keyword, 'BY')) else 0
            with offset(self, offset_):
                self._process(sgroup)

    def _process(self, tlist):
        func_name = '_process_{cls}'.format(cls=type(tlist).__name__)
        func = getattr(self, func_name.lower(), self._process_default)
        func(tlist)

    def process(self, stmt):
        self._process(stmt)
        return stmt
<commit_msg>refactor process case<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2016 Andi Albrecht, albrecht.andi@gmail.com
#
# This module is part of python-sqlparse and is released under
# the BSD License: http://www.opensource.org/licenses/bsd-license.php

from sqlparse import sql, tokens as T
from sqlparse.compat import text_type
from sqlparse.utils import offset, indent

class AlignedIndentFilter(object):
    join_words = (r'((LEFT\s+|RIGHT\s+|FULL\s+)?'
                  r'(INNER\s+|OUTER\s+|STRAIGHT\s+)?|'
                  r'(CROSS\s+|NATURAL\s+)?)?JOIN\b')
    split_words = ('FROM',
                   join_words, 'ON',
                   'WHERE', 'AND', 'OR',
                   'GROUP', 'HAVING', 'LIMIT',
                   'ORDER', 'UNION', 'VALUES',
                   'SET', 'BETWEEN', 'EXCEPT')

    def __init__(self, char=' ', line_width=None):
        self.offset = 0
        self.indent = 0
        self.char = char
        self._max_kwd_len = len('select')

    def nl(self, offset=1):
        # offset = 1 represent a single space after SELECT
        offset = -len(offset) if not isinstance(offset, int) else offset
        # add two for the space and parens
        indent = self.indent * (2 + self._max_kwd_len)

        return sql.Token(T.Whitespace, '\n' + self.char * (
            self._max_kwd_len + offset + indent + self.offset))

    def _process_statement(self, tlist):
        if tlist.tokens[0].is_whitespace() and self.indent == 0:
            tlist.tokens.pop(0)

        # process the main query body
        self._process(sql.TokenList(tlist.tokens))

    def _process_parenthesis(self, tlist):
        # if this isn't a subquery, don't re-indent
        if tlist.token_next_by(m=(T.DML, 'SELECT')):
            with indent(self):
                tlist.insert_after(tlist[0], self.nl('SELECT'))
                # process the inside of the parantheses
                self._process_default(tlist)

            # de-indent last parenthesis
            tlist.insert_before(tlist[-1], self.nl())

    def _process_identifierlist(self, tlist):
        # columns being selected
        identifiers = list(tlist.get_identifiers())
        identifiers.pop(0)
        [tlist.insert_before(token, self.nl()) for token in identifiers]
        self._process_default(tlist)

    def _process_case(self, tlist):
        offset_ = len('case ') + len('when ')
        cases = tlist.get_cases(skip_ws=True)
        # align the end as well
        end_token = tlist.token_next_by(m=(T.Keyword, 'END'))
        cases.append((None, [end_token]))

        condition_width = [len(' '.join(map(text_type, cond))) if cond else 0
                           for cond, _ in cases]
        max_cond_width = max(condition_width)

        for i, (cond, value) in enumerate(cases):
            # cond is None when 'else or end'
            stmt = cond[0] if cond else value[0]

            if i > 0:
                tlist.insert_before(stmt, self.nl(
                    offset_ - len(text_type(stmt))))
            if cond:
                ws = sql.Token(T.Whitespace, self.char * (
                    max_cond_width - condition_width[i]))
                tlist.insert_after(cond[-1], ws)

    def _next_token(self, tlist, idx=0):
        split_words = T.Keyword, self.split_words, True
        token = tlist.token_next_by(m=split_words, idx=idx)
        # treat "BETWEEN x and y" as a single statement
        if token and token.value.upper() == 'BETWEEN':
            token = self._next_token(tlist, token)
            if token and token.value.upper() == 'AND':
                token = self._next_token(tlist, token)
        return token

    def _split_kwds(self, tlist):
        token = self._next_token(tlist)
        while token:
            # joins are special case. only consider the first word as aligner
            if token.match(T.Keyword, self.join_words, regex=True):
                token_indent = token.value.split()[0]
            else:
                token_indent = text_type(token)
            tlist.insert_before(token, self.nl(token_indent))
            token = self._next_token(tlist, token)

    def _process_default(self, tlist):
        self._split_kwds(tlist)
        # process any sub-sub statements
        for sgroup in tlist.get_sublists():
            prev = tlist.token_prev(sgroup)
            # HACK: make "group/order by" work. Longer than max_len.
            offset_ = 3 if (prev and prev.match(T.Keyword, 'BY')) else 0
            with offset(self, offset_):
                self._process(sgroup)

    def _process(self, tlist):
        func_name = '_process_{cls}'.format(cls=type(tlist).__name__)
        func = getattr(self, func_name.lower(), self._process_default)
        func(tlist)

    def process(self, stmt):
        self._process(stmt)
        return stmt
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import re
import sys
import json
import logging
from optparse import OptionParser

__doc__ = """
Script to create LCA for contigs from seperate Protein and rRNA (feature) LCAs.
Features mapped to contigs based on ID prefix:
    feature ID = PREFIX_start_stop_stand
    contig ID = PREFIX
LCA file format, TSV, same for inputs and outputs:
    md5 list, feature id, identity list, length list, evalue list, lca string, depth of lca (1-8)"""

# logging
LOG_FORMAT = '[%(asctime)-15s] [%(levelname)-5s] %(message)s'
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
logger = logging.getLogger()

# regex
id_re = re.compile(r"^(\S+?)_\d+_\d+_[-|+]$") # .group(1) == contig name

def process_line(line):
    parts = line.strip().split('\t')
    if len(parts) < 7:
        return "", None
    (md5, frag, ident, length, e_val, lca, lvl) = parts[:7]
    if not (frag and md5 and lca):
        return "", None
    id_match = id_re.match(frag)
    if not rid_match:
        return "", None
    return rid_match.group(1), [ md5.split(';'), frag, ident.split(';'), length.split(';'), e_val.split(';'), lca.split(';'), lvl ]

def merge_rows(cid, rows):
    md5, ident, length, e_val = (set() for i in range(4))
    lca = ["-"] * 8
    lvl = 0
    for row in rows:
        md5.update(row[0])
        ident.update(row[2])
        length.update(row[3])
        e_val.update(row[4])
    lcamatrix = [row[5] for row in rows]
    lcarotate = zip(*lcamatrix)
    for i, x in enumerate(lcarotate):
        if all_equal(x):
            lca[i] = x[0]
            lvl = i + 1
        else:
            break
    return [ list(md5), cid, list(ident), list(length), list(e_val), lca, lvl ]

def all_equal(iterable):
    "Returns True if all the elements are equal to each other"
    g = groupby(iterable)
    return next(g, True) and not next(g, False)

def print_row(hdl, row):
    for r in row:
        if isinstance(r, list):
            r = ";".join(r)
    hdl.write("\t".join(row)+"\n")

usage = "usage: %prog [options]\n" + __doc__
def main(args):
    parser = OptionParser(usage=usage)
    parser.add_option('--in_rna', dest="in_rna", default=None, help="input file: expanded rna lca")
    parser.add_option('--in_prot', dest="in_prot", default=None, help="input file: expanded protein lca")
    parser.add_option('--output', dest="output", default=None, help="output file: expanded contig lca")
    parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="Print informational messages.")

    (opts, args) = parser.parse_args()
    if not (opts.in_rna and os.path.isfile(opts.in_rna)):
        logger.error("missing required input rna lca file")
        return 1
    if not (opts.in_prot and os.path.isfile(opts.in_prot)):
        logger.error("missing required input protein lca file")
        return 1
    if not opts.output:
        logger.error("missing required output file")
        return 1

    rna_contigs = set()
    ohdl = open(opts.output, 'w')
    rhdl = open(opts.in_rna, 'rU')

    # create contig LCAs for RNA features first, take precidence over protein features
    if opts.verbose:
        print "Reading file %s ... "%(opts.in_rna)

    # get first parsable line
    prev = ""
    row  = []
    data = []
    while prev == "":
        firstline = rhdl.readline()
        prev, row = process_line(firstline)

    data.append(row)
    rctg = 0
    rrna = 1

    # process remaining lines
    for line in rhdl:
        cid, row = process_line(line)
        if cid == "":
            continue
        if cid != prev:
            # new contig found, process old
            mrow = merge_rows(prev, data)
            rna_contigs.add(prev)
            print_row(ohdl, mrow)
            # reset
            prev = cid
            data = []
            rctg += 1
        data.append(row)
        rrna += 1

    # process last contig
    if len(data) > 0:
        mrow = merge_rows(cid, data)
        rna_contigs.add(cid)
        print_row(ohdl, mrow)
        rctg += 1

    rhdl.close()
    if opts.verbose:
        print "Done: %d contigs with %d rRNAs processed"%(rctg, rrna)

    phdl = open(opts.in_prot, 'rU')
    if opts.verbose:
        "Reading file %s ... "%(opts.in_prot)

    # get first parsable line
    prev = ""
    row  = []
    data = []
    while prev == "":
        firstline = phdl.readline()
        prev, row = process_line(firstline)

    data.append(row)
    pctg = 0
    prot = 1

    # process remaining lines
    for line in phdl:
        cid, row = process_line(line)
        if (cid == "") or (cid in rna_contigs):
            # skip those found with rnas
            continue
        if cid != prev:
            # new contig found, process old
            mrow = merge_rows(prev, data)
            print_row(ohdl, mrow)
            # reset
            prev = cid
            data = []
            pctg += 1
        data.append(row)
        prot += 1

    # process last contig
    if len(data) > 0:
        mrow = merge_rows(cid, data)
        print_row(ohdl, mrow)
        prot += 1

    phdl.close()
    ohdl.close()
    if opts.verbose:
        print "Done: %d contigs with %d proteins processed"%(pctg, prot)

    return 0

if __name__ == "__main__":
    sys.exit(main(sys.argv))
<commit_msg>fix variable name<commit_after>#!/usr/bin/env python

import os
import re
import sys
import json
import logging
from optparse import OptionParser

__doc__ = """
Script to create LCA for contigs from seperate Protein and rRNA (feature) LCAs.
Features mapped to contigs based on ID prefix:
    feature ID = PREFIX_start_stop_stand
    contig ID = PREFIX
LCA file format, TSV, same for inputs and outputs:
    md5 list, feature id, identity list, length list, evalue list, lca string, depth of lca (1-8)"""

# logging
LOG_FORMAT = '[%(asctime)-15s] [%(levelname)-5s] %(message)s'
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
logger = logging.getLogger()

# regex
id_re = re.compile(r"^(\S+?)_\d+_\d+_[-|+]$") # .group(1) == contig name

def process_line(line):
    parts = line.strip().split('\t')
    if len(parts) < 7:
        return "", None
    (md5, frag, ident, length, e_val, lca, lvl) = parts[:7]
    if not (frag and md5 and lca):
        return "", None
    id_match = id_re.match(frag)
    if not id_match:
        return "", None
    return id_match.group(1), [ md5.split(';'), frag, ident.split(';'), length.split(';'), e_val.split(';'), lca.split(';'), lvl ]

def merge_rows(cid, rows):
    md5, ident, length, e_val = (set() for i in range(4))
    lca = ["-"] * 8
    lvl = 0
    for row in rows:
        md5.update(row[0])
        ident.update(row[2])
        length.update(row[3])
        e_val.update(row[4])
    lcamatrix = [row[5] for row in rows]
    lcarotate = zip(*lcamatrix)
    for i, x in enumerate(lcarotate):
        if all_equal(x):
            lca[i] = x[0]
            lvl = i + 1
        else:
            break
    return [ list(md5), cid, list(ident), list(length), list(e_val), lca, lvl ]

def all_equal(iterable):
    "Returns True if all the elements are equal to each other"
    g = groupby(iterable)
    return next(g, True) and not next(g, False)

def print_row(hdl, row):
    for r in row:
        if isinstance(r, list):
            r = ";".join(r)
    hdl.write("\t".join(row)+"\n")

usage = "usage: %prog [options]\n" + __doc__
def main(args):
    parser = OptionParser(usage=usage)
    parser.add_option('--in_rna', dest="in_rna", default=None, help="input file: expanded rna lca")
    parser.add_option('--in_prot', dest="in_prot", default=None, help="input file: expanded protein lca")
    parser.add_option('--output', dest="output", default=None, help="output file: expanded contig lca")
    parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="Print informational messages.")

    (opts, args) = parser.parse_args()
    if not (opts.in_rna and os.path.isfile(opts.in_rna)):
        logger.error("missing required input rna lca file")
        return 1
    if not (opts.in_prot and os.path.isfile(opts.in_prot)):
        logger.error("missing required input protein lca file")
        return 1
    if not opts.output:
        logger.error("missing required output file")
        return 1

    rna_contigs = set()
    ohdl = open(opts.output, 'w')
    rhdl = open(opts.in_rna, 'rU')

    # create contig LCAs for RNA features first, take precidence over protein features
    if opts.verbose:
        print "Reading file %s ... "%(opts.in_rna)

    # get first parsable line
    prev = ""
    row  = []
    data = []
    while prev == "":
        firstline = rhdl.readline()
        prev, row = process_line(firstline)

    data.append(row)
    rctg = 0
    rrna = 1

    # process remaining lines
    for line in rhdl:
        cid, row = process_line(line)
        if cid == "":
            continue
        if cid != prev:
            # new contig found, process old
            mrow = merge_rows(prev, data)
            rna_contigs.add(prev)
            print_row(ohdl, mrow)
            # reset
            prev = cid
            data = []
            rctg += 1
        data.append(row)
        rrna += 1

    # process last contig
    if len(data) > 0:
        mrow = merge_rows(cid, data)
        rna_contigs.add(cid)
        print_row(ohdl, mrow)
        rctg += 1

    rhdl.close()
    if opts.verbose:
        print "Done: %d contigs with %d rRNAs processed"%(rctg, rrna)

    phdl = open(opts.in_prot, 'rU')
    if opts.verbose:
        "Reading file %s ... "%(opts.in_prot)

    # get first parsable line
    prev = ""
    row  = []
    data = []
    while prev == "":
        firstline = phdl.readline()
        prev, row = process_line(firstline)

    data.append(row)
    pctg = 0
    prot = 1

    # process remaining lines
    for line in phdl:
        cid, row = process_line(line)
        if (cid == "") or (cid in rna_contigs):
            # skip those found with rnas
            continue
        if cid != prev:
            # new contig found, process old
            mrow = merge_rows(prev, data)
            print_row(ohdl, mrow)
            # reset
            prev = cid
            data = []
            pctg += 1
        data.append(row)
        prot += 1

    # process last contig
    if len(data) > 0:
        mrow = merge_rows(cid, data)
        print_row(ohdl, mrow)
        prot += 1

    phdl.close()
    ohdl.close()
    if opts.verbose:
        print "Done: %d contigs with %d proteins processed"%(pctg, prot)

    return 0

if __name__ == "__main__":
    sys.exit(main(sys.argv))
<|endoftext|>