<commit_before># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from dlrn.db import Commit
from dlrn.drivers.pkginfo import PkgInfoDriver
from dlrn.repositories import getdistrobranch
from dlrn.repositories import getsourcebranch
from dlrn.repositories import refreshrepo
from dlrn.utils import run_external_preprocess

import csv
import logging
import os
import re
import sh
import shutil

from distroinfo import info
from distroinfo import query
from six.moves.urllib.request import urlopen

logger = logging.getLogger("dlrn-downstream-driver")

def fail_req_attr_missing(attr_name, package):
    raise Exception(
        "Missing required attribute '%s' "
        "for package: %s (project: %s)" % (
            attr_name, package['project'], package['name']))

def fail_req_config_missing(opt_name):
    raise Exception(
        "Missing required config option '%s' "
        "for dlrn.drivers.downstream driver." % opt_name)

class DownstreamInfoDriver(PkgInfoDriver):
    DRIVER_CONFIG = {
        'downstream_driver': {
            'rdoinfo_repo': {'name': 'repo'},
            'info_files': {'type': 'list'},
            'versions_url': {},
            'downstream_distro_branch': {},
            'downstream_tag': {},
            'downstream_distgit_key': {},
            'use_upstream_spec': {'type': 'boolean'},
            'downstream_spec_replace_list': {'type': 'list'},
            'cache_dir': {},
        }
    }

    def __init__(self, *args, **kwargs):
        super(DownstreamInfoDriver, self).__init__(*args, **kwargs)
        self.distroinfo_path = None

    def getpackages(self, **kwargs):
        """Valid parameters:

        :param local_info_repo: local rdoinfo repo to use instead of fetching
                                the default one using distroinfo.
        :param tags: release tags to use (mitaka, newton, etc).
        """
        local_info_repo = kwargs.get('local_info_repo')
        tags = kwargs.get('tags')

        info_files = self.config_options.info_files
        if not info_files:
            fail_req_config_missing('info_file')

        inforepo = None
        if local_info_repo:
            inforepo = info.DistroInfo(
                info_files=info_files,
                local_info=local_info_repo,
                cache_base_path=self.config_options.cache_dir)
            # NOTE(jpena): in general, info_files will only contain one file,
            # but it supports multiple... In that case, we will have a comma
            # separated list of URLs
            self.distroinfo_path = "%s/%s" % (local_info_repo.rstrip('/'),
                                              info_files[0])
            for extra_file in info_files[1:]:
                self.distroinfo_path += ",%s/%s" % (
                    local_info_repo.rstrip('/'))

        elif self.config_options.rdoinfo_repo:
            inforepo = info.DistroInfo(
                info_files=info_files,
                remote_git_info=self.config_options.rdoinfo_repo,
                cache_base_path=self.config_options.cache_dir)
            rdoinfo_repo = self.config_options.rdoinfo_repo
            self.distroinfo_path = "%s/%s" % (rdoinfo_repo.rstrip('/'),
                                              info_files[0])
            for extra_file in info_files[1:]:
                self.distroinfo_path += ",%s/%s" % (
                    rdoinfo_repo.rstrip('/'), extra_file)

        else:
            fail_req_config_missing('repo')
        pkginfo = inforepo.get_info(apply_tag=tags)

        self.packages = pkginfo["packages"]
        if tags:
            # FIXME allow list of tags?
            self.packages = query.filter_pkgs(self.packages, {'tags': tags})

        if self.config_options.downstream_tag:
            # filter out packages missing parameters with
            # downstream_tag
            downstream_tag = self.config_options.downstream_tag
            self.packages = query.filter_pkgs(
                self.packages,
                {'tags': downstream_tag})
        return self.packages

    def _getversions(self):
        """Fetch 'versions.csv'

        from versions_url config option and return the contained data as
        a dict with package name as a key.
        """
        versions_url = self.config_options.versions_url
        if not versions_url:
            fail_req_config_missing('versions_url')

        # return versions.csv as a dict with package name as a key
        vers = {}
        r = urlopen(versions_url)
        content = [x.decode('utf-8') for x in r.readlines()]
        # first line is headers
        for row in csv.reader(content[1:]):
            vers[row[0]] = row[1:]
        return vers

    def _fetch_reference_commit(self):
        """Fetch the commit.yaml file used for this run

        """
        versions_url = self.config_options.versions_url
        commit_url = re.sub('/versions.csv$', '/commit.yaml', versions_url)
        try:
            r = urlopen(commit_url)
            content = [x.decode('utf-8') for x in r.readlines()]
            return content
        except Exception:
            # We do not want to fail on this
            return None

    def _write_reference_commit(self, directory):
        """Write the commit.yaml file

        Used for this run in directory as reference_commit.yaml

        :param: directory where the file will be stored
        """

        if self.ref_commit_yaml is None:
            return
        path = os.path.join(directory, 'reference_commit.yaml')
        with open(path, 'w') as fp:
            for line in self.ref_commit_yaml:
                fp.write(line)

    def _transform_spec(self, directory):
        """Transform based on rules, basically a crude sed implementation

        :param directory: directory to search the spec on
        """
        for f in os.listdir(directory):
            if f.endswith('.spec'):
                specpath = os.path.join(directory, f)
                with open(specpath, "r") as fp:
                    contents = fp.readlines()
                with open(specpath, "w") as fp:
                    for line in contents:
                        for rule in self.config_options.\
                            downstream_spec_replace_list:
                            src = rule.split('/')[0]
                            dst = rule.split('/')[1]
                            line = re.sub(src, dst, line)
                        fp.write(line)

    def getinfo(self, **kwargs):
        project = kwargs.get('project')
        package = kwargs.get('package')
        local = kwargs.get('local')
        dev_mode = kwargs.get('dev_mode')

        datadir = self.config_options.datadir
        repo = package['upstream']
        distgit_attr = self.config_options.downstream_distgit_key or 'distgit'
        distro = package.get(distgit_attr)
        if not distro:
            fail_req_attr_missing(distgit_attr, package)
        distro_dir = self._distgit_clone_dir(package['name'])
        distro_dir_full = self.distgit_dir(package['name'])
        distro_branch = self.config_options.downstream_distro_branch
        if not distro_branch:
            fail_req_config_missing('downstream_distro_branch')
        source_branch = getsourcebranch(package)
        versions = self._getversions()

        self.ref_commit_yaml = self._fetch_reference_commit()

        ups_distro = package['master-distgit']
        ups_distro_dir = self._upstream_distgit_clone_dir(package['name'])
        ups_distro_dir_full = self.upstream_distgit_dir(package['name'])
        ups_distro_branch = getdistrobranch(package)

        # only process packages present in versions.csv
        if package['name'] not in versions:
            logger.warning('Package %s not present in %s - skipping.' % (
                package['name'], self.config_options.versions_url))
            return []
        version = versions[package['name']]

        dt_distro = 0  # In this driver we do not care about dt_distro

        if dev_mode is False:
            try:
                distro_branch, extended_hash, dt_extended = refreshrepo(
                    distro, distro_dir, distro_branch, local=local,
                    full_path=distro_dir_full)
                # extract distro_hash from versions.csv
                distro_hash = version[3]
                # Also download upstream distgit
                _, _, _ = refreshrepo(
                    ups_distro, ups_distro_dir, distro_hash,
                    local=local, full_path=ups_distro_dir_full)

            except Exception:
                # The error was already logged by refreshrepo, and we want
                # to avoid halting the whole run because this distgit repo
                # failed, so return an empty list
                return []
        else:
            distro_hash = "dev"
            extended_hash = "dev"
            dt_extended = 0
            if not os.path.isdir(distro_dir):
                # We should fail in this case, since we are running
                # in dev mode, so no try/except
                refreshrepo(distro, distro_dir, distro_branch, local=local,
                            full_path=distro_dir_full)
            if not os.path.isdir(ups_distro_dir):
                refreshrepo(ups_distro, ups_distro_dir, ups_distro_branch,
                            local=local, full_path=ups_distro_dir_full)

        # repo is usually a string, but if it contains more then one entry we
        # git clone into a project subdirectory
        repos = [repo]
        if isinstance(repo, list):
            repos = repo
        project_toprocess = []
        for repo in repos:
            repo_dir = os.path.join(datadir, project)
            if len(repos) > 1:
                repo_dir = os.path.join(repo_dir, os.path.split(repo)[1])
            try:
                source_branch, _, _ = refreshrepo(repo, repo_dir,
                                                  source_branch,
                                                  local=local)
            except Exception:
                # The error was already logged by refreshrepo, and the only
                # side-effect is that we are not adding this commit to the
                # list of commits to be processed, so we can ignore it and
                # move on to the next repo
                continue
            dt = version[5]
            commit_hash = version[1]
            commit = Commit(dt_commit=float(dt), project_name=project,
                            type='rpm',
                            commit_hash=commit_hash, repo_dir=repo_dir,
                            distro_hash=distro_hash, dt_distro=dt_distro,
                            extended_hash=extended_hash,
                            dt_extended=dt_extended,
                            distgit_dir=self.distgit_dir(package['name']),
                            commit_branch=source_branch)
            project_toprocess.append(commit)

        return project_toprocess

    def preprocess(self, **kwargs):
        package_name = kwargs.get('package_name')
        commit_hash = kwargs.get('commit_hash')
        distgit_dir = self.distgit_dir(package_name)
        ups_distro_dir_full = self.upstream_distgit_dir(package_name)
        distro_dir_full = self.distgit_dir(package_name)
        source_dir = "%s/%s" % (self.config_options.datadir, package_name)

        # In this case, we will copy the upstream distgit into downstream
        # distgit, then transform spec
        if self.config_options.use_upstream_spec:
            # Copy upstream distgit to downstream distgit
            for f in os.listdir(ups_distro_dir_full):
                # skip hidden files
                if not f.startswith('.'):
                    shutil.copy(os.path.join(ups_distro_dir_full, f),
                                distro_dir_full)

            if len(self.config_options.downstream_spec_replace_list) > 0:
                self._transform_spec(distro_dir_full)

        # If we have a jinja2 spec template, run pre-processing
        preprocess_needed = False
        for f in os.listdir(distgit_dir):
            if f.endswith('.spec.j2'):
                # We have a template here, so we have to preprocess
                preprocess_needed = True
                break
        if preprocess_needed:
            logger.info('Pre-processing template at %s' % distgit_dir)
            renderspec = sh.renderspec.bake(_cwd=distgit_dir,
                                            _tty_out=False, _timeout=3600)
            renderspec('--spec-style', 'fedora', '--epoch',
                       '../../epoch/fedora.yaml')

        for custom_preprocess in self.config_options.custom_preprocess:
            if custom_preprocess != '':
                run_external_preprocess(
                    cmdline=custom_preprocess,
                    pkgname=package_name,
                    distgit=distgit_dir,
                    upstream_distgit=ups_distro_dir_full,
                    distroinfo=self.distroinfo_path,
                    source_dir=source_dir,
                    commit_hash=commit_hash)
        return

    def distgit_dir(self, package_name):
        datadir = self.config_options.datadir
        # Find extra directory inside it, if needed
        extra_dir = '/'
        for package in self.packages:
            if package['name'] == package_name:
                if 'distgit-path' in package:
                    extra_dir = package['distgit-path']
                    break
        return os.path.join(datadir, package_name + "_distro",
                            extra_dir.lstrip('/'))

    def _distgit_clone_dir(self, package_name):
        datadir = self.config_options.datadir
        return os.path.join(datadir, package_name + "_distro")

    def upstream_distgit_dir(self, package_name):
        datadir = self.config_options.datadir
        # Find extra directory inside it, if needed
        extra_dir = '/'
        for package in self.packages:
            if package['name'] == package_name:
                if 'distgit-path' in package:
                    extra_dir = package['distgit-path']
                    break
        return os.path.join(datadir, package_name + "_distro_upstream",
                            extra_dir.lstrip('/'))

    def _upstream_distgit_clone_dir(self, package_name):
        datadir = self.config_options.datadir
        return os.path.join(datadir, package_name + "_distro_upstream")
<commit_msg>Fix reference commit writes in the downstream driver<commit_after># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from dlrn.db import Commit
from dlrn.drivers.pkginfo import PkgInfoDriver
from dlrn.repositories import getdistrobranch
from dlrn.repositories import getsourcebranch
from dlrn.repositories import refreshrepo
from dlrn.utils import run_external_preprocess

import csv
import logging
import os
import re
import sh
import shutil

from distroinfo import info
from distroinfo import query
from six.moves.urllib.request import urlopen

logger = logging.getLogger("dlrn-downstream-driver")

def fail_req_attr_missing(attr_name, package):
    raise Exception(
        "Missing required attribute '%s' "
        "for package: %s (project: %s)" % (
            attr_name, package['project'], package['name']))

def fail_req_config_missing(opt_name):
    raise Exception(
        "Missing required config option '%s' "
        "for dlrn.drivers.downstream driver." % opt_name)

class DownstreamInfoDriver(PkgInfoDriver):
    DRIVER_CONFIG = {
        'downstream_driver': {
            'rdoinfo_repo': {'name': 'repo'},
            'info_files': {'type': 'list'},
            'versions_url': {},
            'downstream_distro_branch': {},
            'downstream_tag': {},
            'downstream_distgit_key': {},
            'use_upstream_spec': {'type': 'boolean'},
            'downstream_spec_replace_list': {'type': 'list'},
            'cache_dir': {},
        }
    }

    def __init__(self, *args, **kwargs):
        super(DownstreamInfoDriver, self).__init__(*args, **kwargs)
        self.distroinfo_path = None

    def getpackages(self, **kwargs):
        """Valid parameters:

        :param local_info_repo: local rdoinfo repo to use instead of fetching
                                the default one using distroinfo.
        :param tags: release tags to use (mitaka, newton, etc).
        """
        local_info_repo = kwargs.get('local_info_repo')
        tags = kwargs.get('tags')

        info_files = self.config_options.info_files
        if not info_files:
            fail_req_config_missing('info_file')

        inforepo = None
        if local_info_repo:
            inforepo = info.DistroInfo(
                info_files=info_files,
                local_info=local_info_repo,
                cache_base_path=self.config_options.cache_dir)
            # NOTE(jpena): in general, info_files will only contain one file,
            # but it supports multiple... In that case, we will have a comma
            # separated list of URLs
            self.distroinfo_path = "%s/%s" % (local_info_repo.rstrip('/'),
                                              info_files[0])
            for extra_file in info_files[1:]:
                self.distroinfo_path += ",%s/%s" % (
                    local_info_repo.rstrip('/'))

        elif self.config_options.rdoinfo_repo:
            inforepo = info.DistroInfo(
                info_files=info_files,
                remote_git_info=self.config_options.rdoinfo_repo,
                cache_base_path=self.config_options.cache_dir)
            rdoinfo_repo = self.config_options.rdoinfo_repo
            self.distroinfo_path = "%s/%s" % (rdoinfo_repo.rstrip('/'),
                                              info_files[0])
            for extra_file in info_files[1:]:
                self.distroinfo_path += ",%s/%s" % (
                    rdoinfo_repo.rstrip('/'), extra_file)

        else:
            fail_req_config_missing('repo')
        pkginfo = inforepo.get_info(apply_tag=tags)

        self.packages = pkginfo["packages"]
        if tags:
            # FIXME allow list of tags?
            self.packages = query.filter_pkgs(self.packages, {'tags': tags})

        if self.config_options.downstream_tag:
            # filter out packages missing parameters with
            # downstream_tag
            downstream_tag = self.config_options.downstream_tag
            self.packages = query.filter_pkgs(
                self.packages,
                {'tags': downstream_tag})
        return self.packages

    def _getversions(self):
        """Fetch 'versions.csv'

        from versions_url config option and return the contained data as
        a dict with package name as a key.
        """
        versions_url = self.config_options.versions_url
        if not versions_url:
            fail_req_config_missing('versions_url')

        # return versions.csv as a dict with package name as a key
        vers = {}
        r = urlopen(versions_url)
        content = [x.decode('utf-8') for x in r.readlines()]
        # first line is headers
        for row in csv.reader(content[1:]):
            vers[row[0]] = row[1:]
        return vers

    def _fetch_reference_commit(self):
        """Fetch the commit.yaml file used for this run

        """
        versions_url = self.config_options.versions_url
        commit_url = re.sub('/versions.csv$', '/commit.yaml', versions_url)
        try:
            r = urlopen(commit_url)
            content = [x.decode('utf-8') for x in r.readlines()]
            return content
        except Exception:
            # We do not want to fail on this
            return None

    def _write_reference_commit(self, directory):
        """Write the commit.yaml file

        Used for this run in directory as reference_commit.yaml

        :param: directory where the file will be stored
        """

        ref_commit_yaml = self._fetch_reference_commit()

        if ref_commit_yaml is None:
            return
        path = os.path.join(directory, 'reference_commit.yaml')
        with open(path, 'w') as fp:
            for line in ref_commit_yaml:
                fp.write(line)

    def _transform_spec(self, directory):
        """Transform based on rules, basically a crude sed implementation

        :param directory: directory to search the spec on
        """
        for f in os.listdir(directory):
            if f.endswith('.spec'):
                specpath = os.path.join(directory, f)
                with open(specpath, "r") as fp:
                    contents = fp.readlines()
                with open(specpath, "w") as fp:
                    for line in contents:
                        for rule in self.config_options.\
                            downstream_spec_replace_list:
                            src = rule.split('/')[0]
                            dst = rule.split('/')[1]
                            line = re.sub(src, dst, line)
                        fp.write(line)

    def getinfo(self, **kwargs):
        project = kwargs.get('project')
        package = kwargs.get('package')
        local = kwargs.get('local')
        dev_mode = kwargs.get('dev_mode')

        datadir = self.config_options.datadir
        repo = package['upstream']
        distgit_attr = self.config_options.downstream_distgit_key or 'distgit'
        distro = package.get(distgit_attr)
        if not distro:
            fail_req_attr_missing(distgit_attr, package)
        distro_dir = self._distgit_clone_dir(package['name'])
        distro_dir_full = self.distgit_dir(package['name'])
        distro_branch = self.config_options.downstream_distro_branch
        if not distro_branch:
            fail_req_config_missing('downstream_distro_branch')
        source_branch = getsourcebranch(package)
        versions = self._getversions()

        ups_distro = package['master-distgit']
        ups_distro_dir = self._upstream_distgit_clone_dir(package['name'])
        ups_distro_dir_full = self.upstream_distgit_dir(package['name'])
        ups_distro_branch = getdistrobranch(package)

        # only process packages present in versions.csv
        if package['name'] not in versions:
            logger.warning('Package %s not present in %s - skipping.' % (
                package['name'], self.config_options.versions_url))
            return []
        version = versions[package['name']]

        dt_distro = 0  # In this driver we do not care about dt_distro

        if dev_mode is False:
            try:
                distro_branch, extended_hash, dt_extended = refreshrepo(
                    distro, distro_dir, distro_branch, local=local,
                    full_path=distro_dir_full)
                # extract distro_hash from versions.csv
                distro_hash = version[3]
                # Also download upstream distgit
                _, _, _ = refreshrepo(
                    ups_distro, ups_distro_dir, distro_hash,
                    local=local, full_path=ups_distro_dir_full)

            except Exception:
                # The error was already logged by refreshrepo, and we want
                # to avoid halting the whole run because this distgit repo
                # failed, so return an empty list
                return []
        else:
            distro_hash = "dev"
            extended_hash = "dev"
            dt_extended = 0
            if not os.path.isdir(distro_dir):
                # We should fail in this case, since we are running
                # in dev mode, so no try/except
                refreshrepo(distro, distro_dir, distro_branch, local=local,
                            full_path=distro_dir_full)
            if not os.path.isdir(ups_distro_dir):
                refreshrepo(ups_distro, ups_distro_dir, ups_distro_branch,
                            local=local, full_path=ups_distro_dir_full)

        # repo is usually a string, but if it contains more then one entry we
        # git clone into a project subdirectory
        repos = [repo]
        if isinstance(repo, list):
            repos = repo
        project_toprocess = []
        for repo in repos:
            repo_dir = os.path.join(datadir, project)
            if len(repos) > 1:
                repo_dir = os.path.join(repo_dir, os.path.split(repo)[1])
            try:
                source_branch, _, _ = refreshrepo(repo, repo_dir,
                                                  source_branch,
                                                  local=local)
            except Exception:
                # The error was already logged by refreshrepo, and the only
                # side-effect is that we are not adding this commit to the
                # list of commits to be processed, so we can ignore it and
                # move on to the next repo
                continue
            dt = version[5]
            commit_hash = version[1]
            commit = Commit(dt_commit=float(dt), project_name=project,
                            type='rpm',
                            commit_hash=commit_hash, repo_dir=repo_dir,
                            distro_hash=distro_hash, dt_distro=dt_distro,
                            extended_hash=extended_hash,
                            dt_extended=dt_extended,
                            distgit_dir=self.distgit_dir(package['name']),
                            commit_branch=source_branch)
            project_toprocess.append(commit)

        return project_toprocess

    def preprocess(self, **kwargs):
        package_name = kwargs.get('package_name')
        commit_hash = kwargs.get('commit_hash')
        distgit_dir = self.distgit_dir(package_name)
        ups_distro_dir_full = self.upstream_distgit_dir(package_name)
        distro_dir_full = self.distgit_dir(package_name)
        source_dir = "%s/%s" % (self.config_options.datadir, package_name)

        # In this case, we will copy the upstream distgit into downstream
        # distgit, then transform spec
        if self.config_options.use_upstream_spec:
            # Copy upstream distgit to downstream distgit
            for f in os.listdir(ups_distro_dir_full):
                # skip hidden files
                if not f.startswith('.'):
                    shutil.copy(os.path.join(ups_distro_dir_full, f),
                                distro_dir_full)

            if len(self.config_options.downstream_spec_replace_list) > 0:
                self._transform_spec(distro_dir_full)

        # If we have a jinja2 spec template, run pre-processing
        preprocess_needed = False
        for f in os.listdir(distgit_dir):
            if f.endswith('.spec.j2'):
                # We have a template here, so we have to preprocess
                preprocess_needed = True
                break
        if preprocess_needed:
            logger.info('Pre-processing template at %s' % distgit_dir)
            renderspec = sh.renderspec.bake(_cwd=distgit_dir,
                                            _tty_out=False, _timeout=3600)
            renderspec('--spec-style', 'fedora', '--epoch',
                       '../../epoch/fedora.yaml')

        for custom_preprocess in self.config_options.custom_preprocess:
            if custom_preprocess != '':
                run_external_preprocess(
                    cmdline=custom_preprocess,
                    pkgname=package_name,
                    distgit=distgit_dir,
                    upstream_distgit=ups_distro_dir_full,
                    distroinfo=self.distroinfo_path,
                    source_dir=source_dir,
                    commit_hash=commit_hash)
        return

    def distgit_dir(self, package_name):
        datadir = self.config_options.datadir
        # Find extra directory inside it, if needed
        extra_dir = '/'
        for package in self.packages:
            if package['name'] == package_name:
                if 'distgit-path' in package:
                    extra_dir = package['distgit-path']
                    break
        return os.path.join(datadir, package_name + "_distro",
                            extra_dir.lstrip('/'))

    def _distgit_clone_dir(self, package_name):
        datadir = self.config_options.datadir
        return os.path.join(datadir, package_name + "_distro")

    def upstream_distgit_dir(self, package_name):
        datadir = self.config_options.datadir
        # Find extra directory inside it, if needed
        extra_dir = '/'
        for package in self.packages:
            if package['name'] == package_name:
                if 'distgit-path' in package:
                    extra_dir = package['distgit-path']
                    break
        return os.path.join(datadir, package_name + "_distro_upstream",
                            extra_dir.lstrip('/'))

    def _upstream_distgit_clone_dir(self, package_name):
        datadir = self.config_options.datadir
        return os.path.join(datadir, package_name + "_distro_upstream")
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import argparse
import pkg_resources
import sys

from pre_commit import color
from pre_commit.commands.autoupdate import autoupdate
from pre_commit.commands.clean import clean
from pre_commit.commands.install_uninstall import install
from pre_commit.commands.install_uninstall import uninstall
from pre_commit.commands.run import run
from pre_commit.error_handler import error_handler
from pre_commit.runner import Runner

def main(argv=None):
    argv = argv if argv is not None else sys.argv[1:]
    parser = argparse.ArgumentParser()

    # http://stackoverflow.com/a/8521644/812183
    parser.add_argument(
        '-V', '--version',
        action='version',
        version='%(prog)s {0}'.format(
            pkg_resources.get_distribution('pre-commit').version
        )
    )

    subparsers = parser.add_subparsers(dest='command')

    install_parser = subparsers.add_parser(
        'install', help='Intall the pre-commit script.',
    )
    install_parser.add_argument(
        '-f', '--overwrite', action='store_true',
        help='Overwrite existing hooks / remove migration mode.',
    )
    install_parser.add_argument(
        '--install-hooks', action='store_true',
        help=(
            'Whether to install hook environments for all environments '
            'in the config file.'
        ),
    )

    subparsers.add_parser('uninstall', help='Uninstall the pre-commit script.')

    subparsers.add_parser('clean', help='Clean out pre-commit files.')

    subparsers.add_parser('autoupdate', help='Auto-update hooks config.')

    run_parser = subparsers.add_parser('run', help='Run hooks.')
    run_parser.add_argument('hook', nargs='?', help='A single hook-id to run')
    run_parser.add_argument(
        '--all-files', '-a', action='store_true', default=False,
        help='Run on all the files in the repo.  Implies --no-stash.',
    )
    run_parser.add_argument(
        '--color', default='auto', type=color.use_color,
        help='Whether to use color in output.  Defaults to `auto`',
    )
    run_parser.add_argument(
        '--no-stash', default=False, action='store_true',
        help='Use this option to prevent auto stashing of unstaged files.',
    )
    run_parser.add_argument(
        '--verbose', '-v', action='store_true', default=False,
    )

    help = subparsers.add_parser(
        'help', help='Show help for a specific command.'
    )
    help.add_argument('help_cmd', nargs='?', help='Command to show help for.')

    # Argparse doesn't really provide a way to use a `default` subparser
    if len(argv) == 0:
        argv = ['run']
    args = parser.parse_args(argv)

    if args.command == 'help':
        if args.help_cmd:
            parser.parse_args([args.help_cmd, '--help'])
        else:
            parser.parse_args(['--help'])

    with error_handler():
        runner = Runner.create()

        if args.command == 'install':
            return install(
                runner, overwrite=args.overwrite, hooks=args.install_hooks,
            )
        elif args.command == 'uninstall':
            return uninstall(runner)
        elif args.command == 'clean':
            return clean(runner)
        elif args.command == 'autoupdate':
            return autoupdate(runner)
        elif args.command == 'run':
            return run(runner, args)
        else:
            raise NotImplementedError(
                'Command {0} not implemented.'.format(args.command)
            )

        raise AssertionError(
            'Command {0} failed to exit with a returncode'.format(args.command)
        )

if __name__ == '__main__':
    exit(main())
<commit_msg>Update autoupdate help message.<commit_after>from __future__ import unicode_literals

import argparse
import pkg_resources
import sys

from pre_commit import color
from pre_commit.commands.autoupdate import autoupdate
from pre_commit.commands.clean import clean
from pre_commit.commands.install_uninstall import install
from pre_commit.commands.install_uninstall import uninstall
from pre_commit.commands.run import run
from pre_commit.error_handler import error_handler
from pre_commit.runner import Runner

def main(argv=None):
    argv = argv if argv is not None else sys.argv[1:]
    parser = argparse.ArgumentParser()

    # http://stackoverflow.com/a/8521644/812183
    parser.add_argument(
        '-V', '--version',
        action='version',
        version='%(prog)s {0}'.format(
            pkg_resources.get_distribution('pre-commit').version
        )
    )

    subparsers = parser.add_subparsers(dest='command')

    install_parser = subparsers.add_parser(
        'install', help='Install the pre-commit script.',
    )
    install_parser.add_argument(
        '-f', '--overwrite', action='store_true',
        help='Overwrite existing hooks / remove migration mode.',
    )
    install_parser.add_argument(
        '--install-hooks', action='store_true',
        help=(
            'Whether to install hook environments for all environments '
            'in the config file.'
        ),
    )

    subparsers.add_parser('uninstall', help='Uninstall the pre-commit script.')

    subparsers.add_parser('clean', help='Clean out pre-commit files.')

    subparsers.add_parser(
        'autoupdate',
        help="Auto-update pre-commit config to the latest repos' versions.",
    )

    run_parser = subparsers.add_parser('run', help='Run hooks.')
    run_parser.add_argument('hook', nargs='?', help='A single hook-id to run')
    run_parser.add_argument(
        '--all-files', '-a', action='store_true', default=False,
        help='Run on all the files in the repo.  Implies --no-stash.',
    )
    run_parser.add_argument(
        '--color', default='auto', type=color.use_color,
        help='Whether to use color in output.  Defaults to `auto`',
    )
    run_parser.add_argument(
        '--no-stash', default=False, action='store_true',
        help='Use this option to prevent auto stashing of unstaged files.',
    )
    run_parser.add_argument(
        '--verbose', '-v', action='store_true', default=False,
    )

    help = subparsers.add_parser(
        'help', help='Show help for a specific command.'
    )
    help.add_argument('help_cmd', nargs='?', help='Command to show help for.')

    # Argparse doesn't really provide a way to use a `default` subparser
    if len(argv) == 0:
        argv = ['run']
    args = parser.parse_args(argv)

    if args.command == 'help':
        if args.help_cmd:
            parser.parse_args([args.help_cmd, '--help'])
        else:
            parser.parse_args(['--help'])

    with error_handler():
        runner = Runner.create()

        if args.command == 'install':
            return install(
                runner, overwrite=args.overwrite, hooks=args.install_hooks,
            )
        elif args.command == 'uninstall':
            return uninstall(runner)
        elif args.command == 'clean':
            return clean(runner)
        elif args.command == 'autoupdate':
            return autoupdate(runner)
        elif args.command == 'run':
            return run(runner, args)
        else:
            raise NotImplementedError(
                'Command {0} not implemented.'.format(args.command)
            )

        raise AssertionError(
            'Command {0} failed to exit with a returncode'.format(args.command)
        )

if __name__ == '__main__':
    exit(main())
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import os
import json
import configparser
import requests

url = 'http://tdp.me/v1'

class Config_Opts(object):
    """
    Config stuff go here?
    """

    def __init__(self, api_key=None):
        self.api_key = api_key

def get_tdp_api_key():
    """
    Look for configuration and parse it for the API key. Return True and store
    the key in class Config_Opts if the key is found, or return False.
    """
    api_key = None

    if 'TDP_CONFIG' in os.environ:
        paths = [os.environ['TDP_CONFIG']]
    else:
        paths = ['/etc/tdp.cfg', '~/.tdp']

    paths = [os.path.expandvars(p) for p in paths]
    paths = [os.path.expanduser(p) for p in paths]

    cfg = configparser.RawConfigParser()
    cfg.read(paths)

    if 'TDP_API_KEY' in os.environ:
        api_key = os.environ['TDP_API_KEY']
    elif cfg.has_section('Credentials'):
        api_key = cfg.get('Credentials', 'api_key')

    if api_key:
        Config_Opts(api_key)
        return True
    else:
        return False

headers = {'X-Access-Token': api_key}

<commit_msg>> implying system cfg loljoke<commit_after>#!/usr/bin/env python3

import os
import json
import configparser
import requests

url = 'http://tdp.me/v1'

class Config_Opts(object):
    """
    Config stuff go here?
    """

    def __init__(self, api_key=None):
        self.api_key = api_key

def get_tdp_api_key():
    """
    Look for configuration and parse it for the API key. Return True and store
    the key in class Config_Opts if the key is found, or return False.
    """
    api_key = None

    if 'TDP_CONFIG' in os.environ:
        paths = [os.environ['TDP_CONFIG']]
    else:
        paths = ['~/.tdp']

    paths = [os.path.expandvars(p) for p in paths]
    paths = [os.path.expanduser(p) for p in paths]

    cfg = configparser.RawConfigParser()
    cfg.read(paths)

    if 'TDP_API_KEY' in os.environ:
        api_key = os.environ['TDP_API_KEY']
    elif cfg.has_section('Credentials'):
        api_key = cfg.get('Credentials', 'api_key')

    if api_key:
        Config_Opts(api_key)
        return True
    else:
        return False

headers = {'X-Access-Token': api_key}

<|endoftext|>


<commit_before>from os import makedirs, path
import shutil
import sublime
import sublime_plugin

SUBLIME_ROOT = path.join(sublime.packages_path(), '../')
THEME_EXTENDER_FOLDER = 'Packages/User/Theme Extender/'
THEME_EXTENDER_FULL_FOLDER = path.join(SUBLIME_ROOT, 'Packages/User/Theme Extender/')

class ThemeExtender():
    @classmethod
    def guarantee_theme_extender_folder(cls):
        if not path.exists(THEME_EXTENDER_FULL_FOLDER):
            makedirs(THEME_EXTENDER_FULL_FOLDER)

class ThemeExtenderExtendThemeCommand(sublime_plugin.WindowCommand):
    def run(self):
        """Load existing theme, open an extension for it, enter extending mode"""
        # Load our settings and color scheme
        settings = sublime.load_settings('Preferences.sublime-settings')
        scheme_filepath = settings.get('color_scheme')
        scheme_full_filepath = path.join(SUBLIME_ROOT, scheme_filepath)

        # Determine the filename and its parts
        # e.g. 'Packages/Monokai Extended/Monokai Extended Bright.tmTheme' -> 'Monokai Extended Bright', '.tmTheme'
        # TODO: Move theme filepath resolution into class method
        scheme_filename = path.basename(scheme_filepath)
        [scheme_filename_root, scheme_filename_ext] = path.splitext(scheme_filename)

        # Generate our new filepath
        # e.g. 'Packages/User/ThemeExtender/Monokai Extended Bright.extended.tmTheme'
        extended_filepath = path.join(THEME_EXTENDER_FOLDER,
            scheme_filename_root + '.extended' + scheme_filename_ext)
        extended_full_filepath = path.join(SUBLIME_ROOT, extended_filepath)

        # If the theme was already in the Theme Extender folder, reset it to the current scheme
        if scheme_filepath.startswith(THEME_EXTENDER_FOLDER):
            extended_filepath = scheme_filepath
            extended_full_filepath = scheme_full_filepath

        # If the ThemeExtender folder doesn't already exist, create it
        ThemeExtender.guarantee_theme_extender_folder()

        # If the color scheme doesn't exist
        if not path.exists(extended_full_filepath):
            # Copy over the existing theme to the new location
            shutil.copyfile(scheme_full_filepath, extended_full_filepath)

            # Point the settings to the extended theme and save
            settings.set('color_scheme', extended_filepath)
            sublime.save_settings('Preferences.sublime-settings')

        # Open a buffer to the file
        # DEV: If it is new, the buffer will save it automagically
        # e.g. 'Packages/User/ThemeExtender/Monokai Extended Bright.extended.tmTheme.plist'
        extension_filepath = path.join(THEME_EXTENDER_FOLDER,
            scheme_filename_root + '.extended' + scheme_filename_ext + '.plist')
        extension_full_filepath = path.join(SUBLIME_ROOT, extension_filepath)
        self.window.open_file(extension_full_filepath)
<commit_msg>Added in listener<commit_after>from os import makedirs, path
import shutil
import sublime
import sublime_plugin

SUBLIME_ROOT = path.join(sublime.packages_path(), '../')
THEME_EXTENDER_FOLDER = 'Packages/User/Theme Extender/'
THEME_EXTENDER_FULL_FOLDER = path.join(SUBLIME_ROOT, 'Packages/User/Theme Extender/')

class ThemeExtender():
    @classmethod
    def guarantee_theme_extender_folder(cls):
        if not path.exists(THEME_EXTENDER_FULL_FOLDER):
            makedirs(THEME_EXTENDER_FULL_FOLDER)

class ThemeExtenderExtendThemeCommand(sublime_plugin.WindowCommand):
    def run(self):
        """Load existing theme, open an extension for it, enter extending mode"""
        # Load our settings and color scheme
        settings = sublime.load_settings('Preferences.sublime-settings')
        scheme_filepath = settings.get('color_scheme')
        scheme_full_filepath = path.join(SUBLIME_ROOT, scheme_filepath)

        # Determine the filename and its parts
        # e.g. 'Packages/Monokai Extended/Monokai Extended Bright.tmTheme' -> 'Monokai Extended Bright', '.tmTheme'
        # TODO: Move theme filepath resolution into class method
        scheme_filename = path.basename(scheme_filepath)
        [scheme_filename_root, scheme_filename_ext] = path.splitext(scheme_filename)

        # Generate our new filepath
        # e.g. 'Packages/User/ThemeExtender/Monokai Extended Bright.extended.tmTheme'
        extended_filepath = path.join(THEME_EXTENDER_FOLDER,
            scheme_filename_root + '.extended' + scheme_filename_ext)
        extended_full_filepath = path.join(SUBLIME_ROOT, extended_filepath)

        # If the theme was already in the Theme Extender folder, reset it to the current scheme
        if scheme_filepath.startswith(THEME_EXTENDER_FOLDER):
            extended_filepath = scheme_filepath
            extended_full_filepath = scheme_full_filepath

        # If the ThemeExtender folder doesn't already exist, create it
        ThemeExtender.guarantee_theme_extender_folder()

        # If the color scheme doesn't exist
        if not path.exists(extended_full_filepath):
            # Copy over the existing theme to the new location
            shutil.copyfile(scheme_full_filepath, extended_full_filepath)

            # Point the settings to the extended theme and save
            settings.set('color_scheme', extended_filepath)
            sublime.save_settings('Preferences.sublime-settings')

        # Open a buffer to the file
        # DEV: If it is new, the buffer will save it automagically
        # e.g. 'Packages/User/ThemeExtender/Monokai Extended Bright.extended.tmTheme.plist'
        extension_filepath = path.join(THEME_EXTENDER_FOLDER,
            scheme_filename_root + '.extended' + scheme_filename_ext + '.plist')
        extension_full_filepath = path.join(SUBLIME_ROOT, extension_filepath)
        self.window.open_file(extension_full_filepath)

class ThemeExtenderListener(sublime_plugin.EventListener):
    def on_post_save(self, view):
        print 'haaaai'
<|endoftext|>


<commit_before>from __future__ import print_function

from PIL import Image
import os
import json

filelist = []
output = {}
for (dirpath, dirnames, filenames) in os.walk("sprites"):
    if dirnames == []:
        output[dirpath[8:]+"/ number of frames"] = len(filenames)
    for filename in filenames:
        if filename.endswith('.png'):
            filelist.append(os.sep.join([dirpath, filename]))

for filepath in filelist:
    if not "/ui/" in filepath:
        orig_im = Image.open(filepath)
        orig_im.convert("RGBA")
        w, h = orig_im.size
        scaled_im = Image.new("RGBA", (w+2, h+2))
        scaled_im.paste(orig_im, (1, 1))

        scaled_pixels = scaled_im.load()
        im = Image.new("RGBA", (w+2, h+2))
        pixels = im.load()

        for x in range(0, w+2):
            for y in range(0, h+2):
                # If pixel is transparent
                if scaled_pixels[x, y][3] == 0:
                    isborder = False
                    for i, j in ([-1, 0], [1, 0], [0, -1], [0, 1]):
                        if scaled_pixels[max(0, min(w+1, x+i)), max(0, min(h+1, y+j))][3] != 0:
                            isborder = True
                            break
                    if isborder:
                        # Make red
                        pixels[x, y] = (225, 17, 17)
                else:
                    pixels[x, y] = (0, 0, 0, 0)

        im.save(filepath.replace(".", "_enemy."))

        orig_im = Image.open(filepath)
        orig_im.convert("RGBA")
        w, h = orig_im.size
        scaled_im = Image.new("RGBA", (w+2, h+2))
        scaled_im.paste(orig_im, (1, 1))

        scaled_pixels = scaled_im.load()
        im = Image.new("RGBA", (w+2, h+2))
        pixels = im.load()

        for x in range(0, w+2):
            for y in range(0, h+2):
                # If pixel is transparent
                if scaled_pixels[x, y][3] == 0:
                    isborder = False
                    for i, j in ([-1, 0], [1, 0], [0, -1], [0, 1]):
                        if scaled_pixels[max(0, min(w+1, x+i)), max(0, min(h+1, y+j))][3] != 0:
                            isborder = True
                            break
                    if isborder:
                        # Make blue
                        pixels[x, y] = (0, 180, 255)
                else:
                    pixels[x, y] = (0, 0, 0, 0)

        im.save(filepath.replace(".", "_friend."))

filelist = []
for (dirpath, dirnames, filenames) in os.walk("sprites"):
    for filename in filenames:
        if filename.endswith('.png'):
            filelist.append(os.sep.join([dirpath, filename]))

for filepath in filelist:
    im = Image.open(filepath)
    im.convert("RGBA")
    w, h = im.size
    r = 0
    l = w
    t = h
    b = 0

    pix = im.load()
    for x in range(w):
        for y in range(h):
            if pix[x, y][3] != 0:
                if l > x:
                    l = x
                if r < x:
                    r = x
                if t > y:
                    t = y
                if b < y:
                    b = y

    output[filepath[8:]] = [w/2-l, h/2-t]
    if "run" in filepath:
        print(filepath, [w/2-l, h/2-t])
    im2 = im.crop((l, t, r+1, b+1))
    im2.save(filepath)

with open("sprites/spritedata.json", "w") as f:
    json.dump(output, f)
<commit_msg>Removed useless debug message.<commit_after>from __future__ import print_function

from PIL import Image
import os
import json

filelist = []
output = {}
for (dirpath, dirnames, filenames) in os.walk("sprites"):
    if dirnames == []:
        output[dirpath[8:]+"/ number of frames"] = len(filenames)
    for filename in filenames:
        if filename.endswith('.png'):
            filelist.append(os.sep.join([dirpath, filename]))

for filepath in filelist:
    if not "/ui/" in filepath:
        orig_im = Image.open(filepath)
        orig_im.convert("RGBA")
        w, h = orig_im.size
        scaled_im = Image.new("RGBA", (w+2, h+2))
        scaled_im.paste(orig_im, (1, 1))

        scaled_pixels = scaled_im.load()
        im = Image.new("RGBA", (w+2, h+2))
        pixels = im.load()

        for x in range(0, w+2):
            for y in range(0, h+2):
                # If pixel is transparent
                if scaled_pixels[x, y][3] == 0:
                    isborder = False
                    for i, j in ([-1, 0], [1, 0], [0, -1], [0, 1]):
                        if scaled_pixels[max(0, min(w+1, x+i)), max(0, min(h+1, y+j))][3] != 0:
                            isborder = True
                            break
                    if isborder:
                        # Make red
                        pixels[x, y] = (225, 17, 17)
                else:
                    pixels[x, y] = (0, 0, 0, 0)

        im.save(filepath.replace(".", "_enemy."))

        orig_im = Image.open(filepath)
        orig_im.convert("RGBA")
        w, h = orig_im.size
        scaled_im = Image.new("RGBA", (w+2, h+2))
        scaled_im.paste(orig_im, (1, 1))

        scaled_pixels = scaled_im.load()
        im = Image.new("RGBA", (w+2, h+2))
        pixels = im.load()

        for x in range(0, w+2):
            for y in range(0, h+2):
                # If pixel is transparent
                if scaled_pixels[x, y][3] == 0:
                    isborder = False
                    for i, j in ([-1, 0], [1, 0], [0, -1], [0, 1]):
                        if scaled_pixels[max(0, min(w+1, x+i)), max(0, min(h+1, y+j))][3] != 0:
                            isborder = True
                            break
                    if isborder:
                        # Make blue
                        pixels[x, y] = (0, 180, 255)
                else:
                    pixels[x, y] = (0, 0, 0, 0)

        im.save(filepath.replace(".", "_friend."))

filelist = []
for (dirpath, dirnames, filenames) in os.walk("sprites"):
    for filename in filenames:
        if filename.endswith('.png'):
            filelist.append(os.sep.join([dirpath, filename]))

for filepath in filelist:
    im = Image.open(filepath)
    im.convert("RGBA")
    w, h = im.size
    r = 0
    l = w
    t = h
    b = 0

    pix = im.load()
    for x in range(w):
        for y in range(h):
            if pix[x, y][3] != 0:
                if l > x:
                    l = x
                if r < x:
                    r = x
                if t > y:
                    t = y
                if b < y:
                    b = y

    output[filepath[8:]] = [w/2-l, h/2-t]
    im2 = im.crop((l, t, r+1, b+1))
    im2.save(filepath)

with open("sprites/spritedata.json", "w") as f:
    json.dump(output, f)
<|endoftext|>


<commit_before># SecretStorage module for Python
# Access passwords using the SecretService DBus API
# Author: Dmitry Shachnev, 2013
# License: BSD

"""This module provides some utility functions, but these shouldn't
normally be used by external applications."""

import dbus
from secretstorage.defines import *
from secretstorage.exceptions import ItemNotFoundException

class InterfaceWrapper(dbus.Interface):
	"""Wraps ``dbus.Interface`` class and replaces some D-Bus exceptions
	with :doc:`SecretStorage exceptions <exceptions>`."""

	def catch_errors(self, function_in):
		def function_out(*args, **kwargs):
			try:
				return function_in(*args, **kwargs)
			except dbus.exceptions.DBusException as e:
				if e.get_dbus_name() == DBUS_UNKNOWN_METHOD:
					raise ItemNotFoundException('Item does not exist!')
				if e.get_dbus_name() == DBUS_NO_SUCH_OBJECT:
					raise ItemNotFoundException(e.get_dbus_message())
				raise
		return function_out

	def __getattr__(self, attribute):
		result = dbus.Interface.__getattr__(self, attribute)
		if callable(result):
			result = self.catch_errors(result)
		return result

def open_session(bus):
	"""Returns a new Secret Service session."""
	service_obj = bus.get_object(SECRETS, SS_PATH)
	service_iface = dbus.Interface(service_obj, SS_PREFIX+'Service')
	return service_iface.OpenSession('plain', '', signature='sv')[1]

def format_secret(secret, session):
	"""Formats `secret` to make possible to pass it to the
	Secret Service API."""
	if not isinstance(secret, bytes):
		secret = secret.encode('utf-8')
	return dbus.Struct((session, '', dbus.ByteArray(secret), 'text/plain'))

def exec_prompt(bus, prompt, callback):
	"""Executes the given `prompt`, when complete calls `callback`
	function with two arguments: a boolean representing whether the
	operation was dismissed and a list of unlocked item paths. A main
	loop should be running and registered for this function to work."""
	prompt_obj = bus.get_object(SECRETS, prompt)
	prompt_iface = dbus.Interface(prompt_obj, SS_PREFIX+'Prompt')
	prompt_iface.Prompt('')
	def new_callback(dismissed, unlocked):
		if isinstance(unlocked, dbus.Array):
			unlocked = list(unlocked)
		callback(bool(dismissed), unlocked)
	prompt_iface.connect_to_signal('Completed', new_callback)

def exec_prompt_async_glib(bus, prompt):
	"""Like :func:`exec_prompt`, but asynchronous (uses loop from GLib
	API). Retuns (*dismissed*, *unlocked*) tuple."""
	from gi.repository import GObject
	loop = GObject.MainLoop()
	result = []
	def callback(dismissed, unlocked):
		result.append(dismissed)
		result.append(unlocked)
		loop.quit()
	exec_prompt(bus, prompt, callback)
	loop.run()
	return result[0], result[1]

def exec_prompt_async_qt(bus, prompt):
	"""Like :func:`exec_prompt`, but asynchronous (uses loop from PyQt4
	API). Retuns (*dismissed*, *unlocked*) tuple."""
	from PyQt4.QtCore import QCoreApplication
	app = QCoreApplication([])
	result = []
	def callback(dismissed, unlocked):
		result.append(dismissed)
		result.append(unlocked)
		loop.quit()
	exec_prompt(bus, prompt, callback)
	app.exec_()
	return result[0], result[1]

def to_unicode(string):
	"""Converts D-Bus string to unicode string."""
	try:
		# For Python 2
		return unicode(string)
	except NameError:
		# For Python 3
		return str(string)
<commit_msg>util: fix typos in docstrings<commit_after># SecretStorage module for Python
# Access passwords using the SecretService DBus API
# Author: Dmitry Shachnev, 2013
# License: BSD

"""This module provides some utility functions, but these shouldn't
normally be used by external applications."""

import dbus
from secretstorage.defines import *
from secretstorage.exceptions import ItemNotFoundException

class InterfaceWrapper(dbus.Interface):
	"""Wraps ``dbus.Interface`` class and replaces some D-Bus exceptions
	with :doc:`SecretStorage exceptions <exceptions>`."""

	def catch_errors(self, function_in):
		def function_out(*args, **kwargs):
			try:
				return function_in(*args, **kwargs)
			except dbus.exceptions.DBusException as e:
				if e.get_dbus_name() == DBUS_UNKNOWN_METHOD:
					raise ItemNotFoundException('Item does not exist!')
				if e.get_dbus_name() == DBUS_NO_SUCH_OBJECT:
					raise ItemNotFoundException(e.get_dbus_message())
				raise
		return function_out

	def __getattr__(self, attribute):
		result = dbus.Interface.__getattr__(self, attribute)
		if callable(result):
			result = self.catch_errors(result)
		return result

def open_session(bus):
	"""Returns a new Secret Service session."""
	service_obj = bus.get_object(SECRETS, SS_PATH)
	service_iface = dbus.Interface(service_obj, SS_PREFIX+'Service')
	return service_iface.OpenSession('plain', '', signature='sv')[1]

def format_secret(secret, session):
	"""Formats `secret` to make possible to pass it to the
	Secret Service API."""
	if not isinstance(secret, bytes):
		secret = secret.encode('utf-8')
	return dbus.Struct((session, '', dbus.ByteArray(secret), 'text/plain'))

def exec_prompt(bus, prompt, callback):
	"""Executes the given `prompt`, when complete calls `callback`
	function with two arguments: a boolean representing whether the
	operation was dismissed and a list of unlocked item paths. A main
	loop should be running and registered for this function to work."""
	prompt_obj = bus.get_object(SECRETS, prompt)
	prompt_iface = dbus.Interface(prompt_obj, SS_PREFIX+'Prompt')
	prompt_iface.Prompt('')
	def new_callback(dismissed, unlocked):
		if isinstance(unlocked, dbus.Array):
			unlocked = list(unlocked)
		callback(bool(dismissed), unlocked)
	prompt_iface.connect_to_signal('Completed', new_callback)

def exec_prompt_async_glib(bus, prompt):
	"""Like :func:`exec_prompt`, but asynchronous (uses loop from GLib
	API). Returns (*dismissed*, *unlocked*) tuple."""
	from gi.repository import GObject
	loop = GObject.MainLoop()
	result = []
	def callback(dismissed, unlocked):
		result.append(dismissed)
		result.append(unlocked)
		loop.quit()
	exec_prompt(bus, prompt, callback)
	loop.run()
	return result[0], result[1]

def exec_prompt_async_qt(bus, prompt):
	"""Like :func:`exec_prompt`, but asynchronous (uses loop from PyQt4
	API). Returns (*dismissed*, *unlocked*) tuple."""
	from PyQt4.QtCore import QCoreApplication
	app = QCoreApplication([])
	result = []
	def callback(dismissed, unlocked):
		result.append(dismissed)
		result.append(unlocked)
		loop.quit()
	exec_prompt(bus, prompt, callback)
	app.exec_()
	return result[0], result[1]

def to_unicode(string):
	"""Converts D-Bus string to unicode string."""
	try:
		# For Python 2
		return unicode(string)
	except NameError:
		# For Python 3
		return str(string)
<|endoftext|>


<commit_before># Copyright 2015 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import time

import six

from .globals import current_queue, task_context

class Retry(Exception):
    """When raised within a task, the task will be re-queued"""
    pass

QUEUED = 'queued'
RETRYING = 'retrying'
FINISHED = 'finished'
FAILED = 'failed'
STARTED = 'started'

class Task(object):
    def __init__(self, id, f, args, kwargs):
        self.id = id
        self.f = f
        self.args = args
        self.kwargs = kwargs
        self.retries = 0
        self.reset()

    def reset(self):
        self.status = QUEUED
        self.result = None
        self.retires = 0

    def retry(self):
        self.status = RETRYING
        self.retries += 1

    def start(self):
        self.status = STARTED
        self.result = None

    def finish(self, result):
        self.status = FINISHED
        self.result = result

    def fail(self, exception):
        self.status = FAILED
        self.result = exception

    def summary(self):
        return '{id}: {f.__name__}({args}, {kwargs}) -> {result} ({status})'\
            .format(**self.__dict__)

    def execute(self, queue=None):
        if not queue:
            queue = current_queue

        with self.task_context():
            self.start()
            # Notify the storage that this task has been started.
            queue.storage.put_task(self)

            try:
                result = self.call_func()
                self.finish(result)
            except Retry:
                # Task raised Retry, so re-enqueue the task to run again later.
                self.retry()
                queue.enqueue_task(self)
            except Exception as e:
                self.fail(e)
            finally:
                # Record success, failure, or retry in the storage.
                queue.storage.put_task(self)

    def call_func(self):
        return self.f(*self.args, **self.kwargs)

    def task_context(self):
        """
        Returns a context manager that sets this task as the current_task
        global. Similar to flask's app.request_context. This is used by the
        workers to make the global available inside of task functions.
        """
        return task_context(self)

    __call__ = execute

    def __str__(self):
        return '<Task {}>'.format(self.summary())

class TaskResult(object):
    """Similar to concurrent.futures.Future, this class can be used to get the
    result of a task.
    """
    def __init__(self, task_id, queue=None):
        self.task_id = task_id

        if not queue:
            queue = current_queue

        self.storage = queue.storage

    def get_task(self):
        return self.storage.get_task(self.task_id)

    def result(self, timeout=None):
        """Gets the result of the task.

        Arguments:
            timeout: Maximum seconds to wait for a result before raising a
                TimeoutError. If set to None, this will wait forever. If the
                queue doesn't store results and timeout is None, this call will
                never return.
        """
        start = time.time()
        while True:
            task = self.get_task()
            if not task or task.status not in (FINISHED, FAILED):
                if not timeout:
                    continue
                elif time.time() - start < timeout:
                    continue
                else:
                    raise TimeoutError()
            if task.status == FINISHED:
                return task.result
            if task.status == FAILED:
                raise task.result

if not six.PY3:
    class TimeoutError(Exception):
        """The operation exceeded the given deadline."""
        pass
<commit_msg>Fixing python 3 tests<commit_after># Copyright 2015 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import time

from .globals import current_queue, task_context

class Retry(Exception):
    """When raised within a task, the task will be re-queued"""
    pass

class TimeoutError(Exception):
    """The operation exceeded the given deadline."""
    pass

QUEUED = 'queued'
RETRYING = 'retrying'
FINISHED = 'finished'
FAILED = 'failed'
STARTED = 'started'

class Task(object):
    def __init__(self, id, f, args, kwargs):
        self.id = id
        self.f = f
        self.args = args
        self.kwargs = kwargs
        self.retries = 0
        self.reset()

    def reset(self):
        self.status = QUEUED
        self.result = None
        self.retires = 0

    def retry(self):
        self.status = RETRYING
        self.retries += 1

    def start(self):
        self.status = STARTED
        self.result = None

    def finish(self, result):
        self.status = FINISHED
        self.result = result

    def fail(self, exception):
        self.status = FAILED
        self.result = exception

    def summary(self):
        return '{id}: {f.__name__}({args}, {kwargs}) -> {result} ({status})'\
            .format(**self.__dict__)

    def execute(self, queue=None):
        if not queue:
            queue = current_queue

        with self.task_context():
            self.start()
            # Notify the storage that this task has been started.
            queue.storage.put_task(self)

            try:
                result = self.call_func()
                self.finish(result)
            except Retry:
                # Task raised Retry, so re-enqueue the task to run again later.
                self.retry()
                queue.enqueue_task(self)
            except Exception as e:
                self.fail(e)
            finally:
                # Record success, failure, or retry in the storage.
                queue.storage.put_task(self)

    def call_func(self):
        return self.f(*self.args, **self.kwargs)

    def task_context(self):
        """
        Returns a context manager that sets this task as the current_task
        global. Similar to flask's app.request_context. This is used by the
        workers to make the global available inside of task functions.
        """
        return task_context(self)

    __call__ = execute

    def __str__(self):
        return '<Task {}>'.format(self.summary())

class TaskResult(object):
    """Similar to concurrent.futures.Future, this class can be used to get the
    result of a task.
    """
    def __init__(self, task_id, queue=None):
        self.task_id = task_id

        if not queue:
            queue = current_queue

        self.storage = queue.storage

    def get_task(self):
        return self.storage.get_task(self.task_id)

    def result(self, timeout=None):
        """Gets the result of the task.

        Arguments:
            timeout: Maximum seconds to wait for a result before raising a
                TimeoutError. If set to None, this will wait forever. If the
                queue doesn't store results and timeout is None, this call will
                never return.
        """
        start = time.time()
        while True:
            task = self.get_task()
            if not task or task.status not in (FINISHED, FAILED):
                if not timeout:
                    continue
                elif time.time() - start < timeout:
                    continue
                else:
                    raise TimeoutError()
            if task.status == FINISHED:
                return task.result
            if task.status == FAILED:
                raise task.result
<|endoftext|>


<commit_before>"""

http://blast.ncbi.nlm.nih.gov/blastcgihelp.shtml
http://www.ncbi.nlm.nih.gov/toolkit/doc/book/ch_demo/?rendertype=table&id=ch_demo.T5#ch_demo.id1_fetch.html_ref_fasta

"""
from collections import OrderedDict
import pandas as pd
import pylab

__all__ = ["FASTA", "MultiFASTA"]

"""
json file like in ensembl:
    molecule (e.g., dna)
    id =

    {u'id': u'contig::ABBA01004489.1:1:100:1',
             u'molecule': u'dna',
              u'seq': u'ctgtact'}
"""

class Sequence(str):
    def __init__(self):
        self.description = None

    def complement(self):
        pass

    def reverse(self):
        pass

class MultiFASTA(object):
    """Class to manipulate several several FASTA items

    Here, we load some FASTA using UniProt web service::

        >>> from biokit.io import MultiFASTA
        >>> mf = MultiFASTA()
        >>> mf.load_fasta("P43408")
        >>> mf.load_fasta("P21318")

    You can then get back to your accession entries as follows ::

        >>> mf.ids
        ['P43408', 'P21318']

    And the sequences in the same order can be accessed::

        >>> len(mf)
        2

    Each FASTA is stored in :attr:`fasta`, which is a dictionary where each
    values is an instance of :class:`FASTA`::

        >>> print(mf._fasta["P43408"].fasta)
        >sp|P43408|KADA_METIG Adenylate kinase OS=Methanotorris igneus GN=adkA PE=1 SV=2
        MKNKVVVVTGVPGVGGTTLTQKTIEKLKEEGIEYKMVNFGTVMFEVAKEEGLVEDRDQMR
        KLDPDTQKRIQKLAGRKIAEMAKESNVIVDTHSTVKTPKGYLAGLPIWVLEELNPDIIVI
        VETSSDEILMRRLGDATRNRDIELTSDIDEHQFMNRCAAMAYGVLTGATVKIIKNRDGLL
        DKAVEELISVLK

    The most convenient way to access to all data is to use the dataframe attribute::

        >>> mf.df.Sequence

    .. plot::
        :width: 50%
        :include-source:

        >>> from bioservices.apps import MultiFASTA
        >>> f = MultiFASTA()
        >>> f.load_fasta(["P43403", "P43410"])
        >>> f.df.Size.hist()

    """
    def __init__(self, verbose=True):
        # fetch the sequence using this attribute
        self.verbose = verbose
        self._fasta_fetcher = FASTA()

        # an ordered dictionary to store the fasta contents
        self._fasta = OrderedDict()

    def __len__(self):
        return len(self._fasta)

    def _get_fasta(self):
        return self._fasta
    fasta = property(_get_fasta, doc="Returns all FASTA instances ")

    def _get_ids(self):
        return [f for f in self._fasta.keys()]
    ids = property(_get_ids, doc="returns list of keys/accession identifiers")

    def load_fasta(self, ids):
        """Loads a single FASTA file into the dictionary

        """
        if isinstance(ids, str):
            ids = [ids]

        for id_ in ids:
            self._fasta_fetcher.load(id_)
            # create a new instance of FASTA and save fasta data
            f = FASTA()
            f._fasta = self._fasta_fetcher._fasta[:]

            # append in the ordered dictionary
            self._fasta[id_] = f
            if self.verbose:
                print("%s loaded" % id_)

    def save_fasta(self, filename):
        """Save all FASTA into a file"""
        fh = open(filename, "w")
        for f in self._fasta.values():
            fh.write(f.fasta)
        fh.close()

    def read_fasta(self, filename):
        """Load several FASTA from a filename"""
        fh = open(filename, "r")
        data = fh.read()
        fh.close()

        # we split according to ">2 character
        for thisfasta in data.split(">")[1:]:
            f = FASTA()
            f._fasta = f._interpret(thisfasta)
            print f._fasta
            print f.accession
            try: dummy = f.accession, self.ids # just to try
            except:pass
            if f.accession is not None and f.accession not in self.ids:
                self._fasta[f.accession] = f
            else:
                if self.verbose:
                    print("Accession %s is already in the ids list or could not be interpreted. skipped" % str(f.accession))
            return

    def _get_df(self):
        df =  pd.concat([self.fasta[id_].df for id_ in self.fasta.keys()])
        df.reset_index(inplace=True)
        return df
    df = property(_get_df)

    def hist_size(self, **kargs):
        self.df.Size.hist(**kargs)
        pylab.title("Histogram length of the sequences")
        pylab.xlabel("Length")

class FASTA(object):
    """Dedicated class to manipulates FASTA sequence(s)

    Here is a FASTA file example::

        >sp|P43408|KADA_METIG Adenylate kinase OS=Methanotorris igneus GN=adkA PE=1 SV=2
        MKNKVVVVTGVPGVGGTTLTQKTIEKLKEEGIEYKMVNFGTVMFEVAKEEGLVEDRDQMR
        KLDPDTQKRIQKLAGRKIAEMAKESNVIVDTHSTVKTPKGYLAGLPIWVLEELNPDIIVI
        VETSSDEILMRRLGDATRNRDIELTSDIDEHQFMNRCAAMAYGVLTGATVKIIKNRDGLL
        DKAVEELISVLK

    Comments can be added with the ; character. Note also that length of the sequence
    is wrapped to 80 characters in general.

    The format is made of a header and a sequence. Any FASTA can be read
    and the pair of header/sequence retrieved from the :attr:`sequence` and
    :attr:`header` attributes. However, headers differ from one database to
    another one (see table below)

    Wikipedia. A multiple sequence FASTA format would be obtained by concatenating several single sequence FASTA files. This does not imply a contradiction with the format as only the first line in a FASTA file may start with a ";" or ">", hence forcing all subsequent sequences to start with a ">" in order to be taken as different ones (and further forcing the exclusive reservation of ">" for the sequence definition line). Thus, the examples above may as well be taken as a multisequence file if taken together.

    You can read a FASTA sequence from a local file, a string or download one
    from UniProt using BioServices::

    .. doctest::

        >>> from biokit.io.fasta import FASTA
        >>> f = FASTA()
        >>> f.load("P43403")
        >>> acc = f.accession    # the accession (P43403)
        >>> fasta = f.fasta      # raw FASTA string
        >>> seq = f.sequence     # the sequence itself
        >>> header = f.header    # the header itself
        >>> identifier = f.identifier

    You can also get a dataframe also using Pandas library.::

        >>> f.df

    The columns stored in the dataframe encompase:

        * **Accession** that is taken from the header (e.g., P43403 from uniprot)
        * **Sequence**, a copy of the FASTA sequence
        * **Size**,  the length of the sequence.
        * **Database**, the database type found in the header (e.g., sp for
          SWISS-PROT; see below for a list of database and their header format).
        * Some column such as **Organism** are filled only for some database
        * **Identififers** is the begining of the header.

    .. seealso:: :class:`MultiFASTA` for multi FASTA manipulation.

    List of identifiers corresponding to different databases.

    ================================= ====================================
    ================================= ====================================
    GenBank                           gi|gi-number|gb|accession|locus
    GenBank                           gb|accession|locus
    GenInfo integrated database       gi|integer
    EMBL Data Library                 gi|gi-number|emb|accession|locus
    EMBL Data Library                 emb|accession|locus
    DDBJ, DNA Database of Japan       gi|gi-number|dbj|accession|locus
    DDBJ, DNA Database of Japan       dbj|accession|locus
    NBRF PIR                          pir||entry
    PIR                               pir|accession|name
    Protein Research Foundation       prf||name
    PRF                               prf|accession|name
    SWISS-PROT                        sp|accession|name
    Brookhaven Protein Data Bank (1)  pdb|entry|chain
    Brookhaven Protein Data Bank (2)  entry:chain|PDBID|CHAIN|SEQUENCE
    Patents                           pat|country|number
    patent                            pat|country|patent|sequence
    pre-grant patent                  pgp|country|application-number|seq-number
    GenInfo Backbone Id               bbs|number
    GenInfo Backbone moltype          bbm|number
    GenInfo import ID                 gim|integer
    General database identifier       gnl|database|identifier
    NCBI Reference Sequence           ref|accession|locus
    RefSeq 2                          ref|accession|name
    Local Sequence identifier         lcl|identifier
    PDB                               pdb|entry|chain
    DDBJ                              dbj|accession|locus
    Default                           |accession|name
    ================================= ====================================

    identifier is mix of integers and characters

    gnl allows databases not included in this list to use the same
    identifying syntax.

    The :meth::`load_fasta` relies on UniProt service.
    """

    known_dbtypes = ["sp", "gi"]
    def __init__(self):
        self._fasta = None

    def _get_fasta(self):
        return self._fasta
    fasta = property(_get_fasta, doc="returns FASTA content")

    # for all types
    def _get_sequence(self):
        if self.fasta:
            return "".join(self.fasta.split("\n")[1:])
        else:
            raise ValueError("You need to load a fasta sequence first using get_fasta or read_fasta")
    sequence = property(_get_sequence, doc="returns the sequence only")

    # for all types
    def _get_header(self):
        if self.fasta:
            return self.fasta.split("\n")[0]
        else:
            raise ValueError("You need to load a fasta sequence first using get_fasta or read_fasta")
    header = property(_get_header, doc="returns header only")

    def _get_dbtype(self):
        dbtype = self.header.split("|")[0].replace(">", "")

        return dbtype
    dbtype = property(_get_dbtype)

    # for all types
    def _get_identifier(self):
        return self.header.split(" ")[0]
    identifier = property(_get_identifier)

    def _get_entry(self):
        return self.header.split("|")[2].split(" ")[0]
    entry = property(_get_entry, doc="returns entry only")

    # swiss prot only
    def _get_accession(self):
        if self.dbtype == "sp":
            #header = self.header
            return self.identifier.split("|")[1]
        elif self.dbtype == "gi":
            return self.identifier.split("|")[1]
        else:
            return self.identifier
    accession = property(_get_accession)

    # swiss prot only
    def _get_name_sp(self):
        if self.dbtype == "sp":
            header = self.header
            return header.split(" ")[0].split("|")[2]
    name = property(_get_name_sp)

    def _get_df(self):
        df = pd.DataFrame({
            "Identifiers": [self.identifier],
            "Accession": [self.accession],
            "Entry": [self.entry],
            "Database": [self.dbtype],
            "Organism": [self.organism],
            "PE": [self.PE],
            "SV": [self.SV],
            "Sequence": [self.sequence],
            "Header": [self.header],
            "Size": [len(self.sequence)]})
        return df
    df = property(_get_df)

    def _get_info_from_header(self, prefix):
        if prefix not in self.header:
            return None
        # finds the prefix
        index = self.header.index(prefix+"=")
        # remove it
        name = self.header[index:][3:]
        # figure out if there is anothe = sign to split the string
        # otherwise, the prefix we looked for is the last one anyway
        if "=" in name:
            name = name.split("=")[0]
            # here each = sign in FASTA is preceded by 2 characters that we must remove
            name = name[0:-2]
            name = name.strip()
        else:
            name = name.strip()
        return name

    def _get_gene_name(self):
        return self._get_info_from_header("GN")
    gene_name = property(_get_gene_name,
        doc="returns gene name from GN keyword found in the header if any")

    def _get_organism(self):
        return self._get_info_from_header("OS")
    organism = property(_get_organism,
        doc="returns organism from OS keyword found in the header if any")

    def _get_PE(self):
        pe = self._get_info_from_header("PE")
        if pe is not None:
            return int(pe)
    PE = property(_get_PE,
        doc="returns PE keyword found in the header if any")

    def _get_SV(self):
        sv = self._get_info_from_header("SV")
        if sv is not None:
            return int(sv)
    SV = property(_get_SV,
        doc="returns SV keyword found in the header if any")

    def __str__(self):
        str_ = self.fasta
        return str_

    def load(self, id_):
        self.load_fasta(id_)

    def load_fasta(self, id_):
        """Fetches FASTA from uniprot and loads into attribute :attr:`fasta`

        :param str id_: a given uniprot identifier
        :returns: nothing

        .. note:: same as :meth:`get_fasta` but returns nothing
        """
        # save fasta into attributes fasta
        from bioservices import UniProt
        u = UniProt(verbose=False)
        try:
            res = u.retrieve(id_, frmt="fasta")
            # some entries in uniprot are valid but obsolet and return empty string
            if res == "":
                raise Exception
            self._fasta = res[:]
        except:
            pass

    def save_fasta(self, filename):
        """Save FASTA file into a filename

        :param str data: the FASTA contents
        :param str filename: where to save it
        """
        if self._fasta == None:
            raise ValueError("No fasta was read or downloaded. Nothing to save.")

        fh = open(filename, "w")
        fh.write(self._fasta)
        fh.close()

    def read_fasta(self, filename):
        """Reads a FASTA file and loads it

        Type::

            >>> f = FASTA()
            >>> f.read_fasta(filename)
            >>> f.fasta

        :return: nothing

        .. warning:: If more than one FASTA is contained in the file, an error is raised
        """
        fh = open(filename, "r")
        data = fh.read()
        fh.close()

        # Is there more than one sequence ?
        data = data.split(">")[1:]
        if len(data)>1 or len(data)==0:
            raise ValueError("""Only one sequence expected to be found. Found %s. Please use MultiFASTA class instead""" % len(data))

        self._data = data
        if data.count(">sp|")>1:
            raise ValueError("""It looks like your FASTA file contains more than
            one FASTA. You must use MultiFASTA class instead""")

        self._fasta = data[:]
        self._fasta = self._fasta[0]
        if self.dbtype not in self.known_dbtypes:
            print("Only sp and gi header are recognised so far but sequence and header are loaded")

    def _interpret(self, data):
        # cleanup the data in case of empty spaces or \n characters
        return data

    def check(self):
        """Before submitting a request, any numerical digits in the query
        sequence should either be removed or replaced by appropriate letter
        codes (e.g., N for unknown nucleic acid residue or X for unknown
        amino acid residue). The nucleic acid codes supported are:

         For those programs that use amino acid query sequences (BLASTP
         and TBLASTN), the accepted amino acid codes are:
         A, B, C, D, E, F, G, H, I, K, L, M, N, P, Q, R, S, T, U, V, W, Y, Z, X, *, -

         * is for translation stop and - for gap of indeterminate length

        """
        pass

<commit_msg>fixing whitespace error<commit_after>"""

http://blast.ncbi.nlm.nih.gov/blastcgihelp.shtml
http://www.ncbi.nlm.nih.gov/toolkit/doc/book/ch_demo/?rendertype=table&id=ch_demo.T5#ch_demo.id1_fetch.html_ref_fasta

"""
from collections import OrderedDict
import pandas as pd
import pylab

__all__ = ["FASTA", "MultiFASTA"]

"""
json file like in ensembl:
    molecule (e.g., dna)
    id =

    {u'id': u'contig::ABBA01004489.1:1:100:1',
             u'molecule': u'dna',
              u'seq': u'ctgtact'}
"""

class Sequence(str):
    def __init__(self):
        self.description = None

    def complement(self):
        pass

    def reverse(self):
        pass

class MultiFASTA(object):
    """Class to manipulate several several FASTA items

    Here, we load some FASTA using UniProt web service::

        >>> from biokit.io import MultiFASTA
        >>> mf = MultiFASTA()
        >>> mf.load_fasta("P43408")
        >>> mf.load_fasta("P21318")

    You can then get back to your accession entries as follows ::

        >>> mf.ids
        ['P43408', 'P21318']

    And the sequences in the same order can be accessed::

        >>> len(mf)
        2

    Each FASTA is stored in :attr:`fasta`, which is a dictionary where each
    values is an instance of :class:`FASTA`::

        >>> print(mf._fasta["P43408"].fasta)
        >sp|P43408|KADA_METIG Adenylate kinase OS=Methanotorris igneus GN=adkA PE=1 SV=2
        MKNKVVVVTGVPGVGGTTLTQKTIEKLKEEGIEYKMVNFGTVMFEVAKEEGLVEDRDQMR
        KLDPDTQKRIQKLAGRKIAEMAKESNVIVDTHSTVKTPKGYLAGLPIWVLEELNPDIIVI
        VETSSDEILMRRLGDATRNRDIELTSDIDEHQFMNRCAAMAYGVLTGATVKIIKNRDGLL
        DKAVEELISVLK

    The most convenient way to access to all data is to use the dataframe attribute::

        >>> mf.df.Sequence

    .. plot::
        :width: 50%
        :include-source:

        >>> from bioservices.apps import MultiFASTA
        >>> f = MultiFASTA()
        >>> f.load_fasta(["P43403", "P43410"])
        >>> f.df.Size.hist()

    """
    def __init__(self, verbose=True):
        # fetch the sequence using this attribute
        self.verbose = verbose
        self._fasta_fetcher = FASTA()

        # an ordered dictionary to store the fasta contents
        self._fasta = OrderedDict()

    def __len__(self):
        return len(self._fasta)

    def _get_fasta(self):
        return self._fasta
    fasta = property(_get_fasta, doc="Returns all FASTA instances ")

    def _get_ids(self):
        return [f for f in self._fasta.keys()]
    ids = property(_get_ids, doc="returns list of keys/accession identifiers")

    def load_fasta(self, ids):
        """Loads a single FASTA file into the dictionary

        """
        if isinstance(ids, str):
            ids = [ids]

        for id_ in ids:
            self._fasta_fetcher.load(id_)
            # create a new instance of FASTA and save fasta data
            f = FASTA()
            f._fasta = self._fasta_fetcher._fasta[:]

            # append in the ordered dictionary
            self._fasta[id_] = f
            if self.verbose:
                print("%s loaded" % id_)

    def save_fasta(self, filename):
        """Save all FASTA into a file"""
        fh = open(filename, "w")
        for f in self._fasta.values():
            fh.write(f.fasta)
        fh.close()

    def read_fasta(self, filename):
        """Load several FASTA from a filename"""
        fh = open(filename, "r")
        data = fh.read()
        fh.close()

        # we split according to ">2 character
        for thisfasta in data.split(">")[1:]:
            f = FASTA()
            f._fasta = f._interpret(thisfasta)
            print f._fasta
            print f.accession
            try: dummy = f.accession, self.ids # just to try
            except:pass
            if f.accession is not None and f.accession not in self.ids:
                self._fasta[f.accession] = f
            else:
                if self.verbose:
                    print("Accession %s is already in the ids list or could not be interpreted. skipped" % str(f.accession))
        return

    def _get_df(self):
        df =  pd.concat([self.fasta[id_].df for id_ in self.fasta.keys()])
        df.reset_index(inplace=True)
        return df
    df = property(_get_df)

    def hist_size(self, **kargs):
        self.df.Size.hist(**kargs)
        pylab.title("Histogram length of the sequences")
        pylab.xlabel("Length")

class FASTA(object):
    """Dedicated class to manipulates FASTA sequence(s)

    Here is a FASTA file example::

        >sp|P43408|KADA_METIG Adenylate kinase OS=Methanotorris igneus GN=adkA PE=1 SV=2
        MKNKVVVVTGVPGVGGTTLTQKTIEKLKEEGIEYKMVNFGTVMFEVAKEEGLVEDRDQMR
        KLDPDTQKRIQKLAGRKIAEMAKESNVIVDTHSTVKTPKGYLAGLPIWVLEELNPDIIVI
        VETSSDEILMRRLGDATRNRDIELTSDIDEHQFMNRCAAMAYGVLTGATVKIIKNRDGLL
        DKAVEELISVLK

    Comments can be added with the ; character. Note also that length of the sequence
    is wrapped to 80 characters in general.

    The format is made of a header and a sequence. Any FASTA can be read
    and the pair of header/sequence retrieved from the :attr:`sequence` and
    :attr:`header` attributes. However, headers differ from one database to
    another one (see table below)

    Wikipedia. A multiple sequence FASTA format would be obtained by concatenating several single sequence FASTA files. This does not imply a contradiction with the format as only the first line in a FASTA file may start with a ";" or ">", hence forcing all subsequent sequences to start with a ">" in order to be taken as different ones (and further forcing the exclusive reservation of ">" for the sequence definition line). Thus, the examples above may as well be taken as a multisequence file if taken together.

    You can read a FASTA sequence from a local file, a string or download one
    from UniProt using BioServices::

    .. doctest::

        >>> from biokit.io.fasta import FASTA
        >>> f = FASTA()
        >>> f.load("P43403")
        >>> acc = f.accession    # the accession (P43403)
        >>> fasta = f.fasta      # raw FASTA string
        >>> seq = f.sequence     # the sequence itself
        >>> header = f.header    # the header itself
        >>> identifier = f.identifier

    You can also get a dataframe also using Pandas library.::

        >>> f.df

    The columns stored in the dataframe encompase:

        * **Accession** that is taken from the header (e.g., P43403 from uniprot)
        * **Sequence**, a copy of the FASTA sequence
        * **Size**,  the length of the sequence.
        * **Database**, the database type found in the header (e.g., sp for
          SWISS-PROT; see below for a list of database and their header format).
        * Some column such as **Organism** are filled only for some database
        * **Identififers** is the begining of the header.

    .. seealso:: :class:`MultiFASTA` for multi FASTA manipulation.

    List of identifiers corresponding to different databases.

    ================================= ====================================
    ================================= ====================================
    GenBank                           gi|gi-number|gb|accession|locus
    GenBank                           gb|accession|locus
    GenInfo integrated database       gi|integer
    EMBL Data Library                 gi|gi-number|emb|accession|locus
    EMBL Data Library                 emb|accession|locus
    DDBJ, DNA Database of Japan       gi|gi-number|dbj|accession|locus
    DDBJ, DNA Database of Japan       dbj|accession|locus
    NBRF PIR                          pir||entry
    PIR                               pir|accession|name
    Protein Research Foundation       prf||name
    PRF                               prf|accession|name
    SWISS-PROT                        sp|accession|name
    Brookhaven Protein Data Bank (1)  pdb|entry|chain
    Brookhaven Protein Data Bank (2)  entry:chain|PDBID|CHAIN|SEQUENCE
    Patents                           pat|country|number
    patent                            pat|country|patent|sequence
    pre-grant patent                  pgp|country|application-number|seq-number
    GenInfo Backbone Id               bbs|number
    GenInfo Backbone moltype          bbm|number
    GenInfo import ID                 gim|integer
    General database identifier       gnl|database|identifier
    NCBI Reference Sequence           ref|accession|locus
    RefSeq 2                          ref|accession|name
    Local Sequence identifier         lcl|identifier
    PDB                               pdb|entry|chain
    DDBJ                              dbj|accession|locus
    Default                           |accession|name
    ================================= ====================================

    identifier is mix of integers and characters

    gnl allows databases not included in this list to use the same
    identifying syntax.

    The :meth::`load_fasta` relies on UniProt service.
    """

    known_dbtypes = ["sp", "gi"]
    def __init__(self):
        self._fasta = None

    def _get_fasta(self):
        return self._fasta
    fasta = property(_get_fasta, doc="returns FASTA content")

    # for all types
    def _get_sequence(self):
        if self.fasta:
            return "".join(self.fasta.split("\n")[1:])
        else:
            raise ValueError("You need to load a fasta sequence first using get_fasta or read_fasta")
    sequence = property(_get_sequence, doc="returns the sequence only")

    # for all types
    def _get_header(self):
        if self.fasta:
            return self.fasta.split("\n")[0]
        else:
            raise ValueError("You need to load a fasta sequence first using get_fasta or read_fasta")
    header = property(_get_header, doc="returns header only")

    def _get_dbtype(self):
        dbtype = self.header.split("|")[0].replace(">", "")

        return dbtype
    dbtype = property(_get_dbtype)

    # for all types
    def _get_identifier(self):
        return self.header.split(" ")[0]
    identifier = property(_get_identifier)

    def _get_entry(self):
        return self.header.split("|")[2].split(" ")[0]
    entry = property(_get_entry, doc="returns entry only")

    # swiss prot only
    def _get_accession(self):
        if self.dbtype == "sp":
            #header = self.header
            return self.identifier.split("|")[1]
        elif self.dbtype == "gi":
            return self.identifier.split("|")[1]
        else:
            return self.identifier
    accession = property(_get_accession)

    # swiss prot only
    def _get_name_sp(self):
        if self.dbtype == "sp":
            header = self.header
            return header.split(" ")[0].split("|")[2]
    name = property(_get_name_sp)

    def _get_df(self):
        df = pd.DataFrame({
            "Identifiers": [self.identifier],
            "Accession": [self.accession],
            "Entry": [self.entry],
            "Database": [self.dbtype],
            "Organism": [self.organism],
            "PE": [self.PE],
            "SV": [self.SV],
            "Sequence": [self.sequence],
            "Header": [self.header],
            "Size": [len(self.sequence)]})
        return df
    df = property(_get_df)

    def _get_info_from_header(self, prefix):
        if prefix not in self.header:
            return None
        # finds the prefix
        index = self.header.index(prefix+"=")
        # remove it
        name = self.header[index:][3:]
        # figure out if there is anothe = sign to split the string
        # otherwise, the prefix we looked for is the last one anyway
        if "=" in name:
            name = name.split("=")[0]
            # here each = sign in FASTA is preceded by 2 characters that we must remove
            name = name[0:-2]
            name = name.strip()
        else:
            name = name.strip()
        return name

    def _get_gene_name(self):
        return self._get_info_from_header("GN")
    gene_name = property(_get_gene_name,
        doc="returns gene name from GN keyword found in the header if any")

    def _get_organism(self):
        return self._get_info_from_header("OS")
    organism = property(_get_organism,
        doc="returns organism from OS keyword found in the header if any")

    def _get_PE(self):
        pe = self._get_info_from_header("PE")
        if pe is not None:
            return int(pe)
    PE = property(_get_PE,
        doc="returns PE keyword found in the header if any")

    def _get_SV(self):
        sv = self._get_info_from_header("SV")
        if sv is not None:
            return int(sv)
    SV = property(_get_SV,
        doc="returns SV keyword found in the header if any")

    def __str__(self):
        str_ = self.fasta
        return str_

    def load(self, id_):
        self.load_fasta(id_)

    def load_fasta(self, id_):
        """Fetches FASTA from uniprot and loads into attribute :attr:`fasta`

        :param str id_: a given uniprot identifier
        :returns: nothing

        .. note:: same as :meth:`get_fasta` but returns nothing
        """
        # save fasta into attributes fasta
        from bioservices import UniProt
        u = UniProt(verbose=False)
        try:
            res = u.retrieve(id_, frmt="fasta")
            # some entries in uniprot are valid but obsolet and return empty string
            if res == "":
                raise Exception
            self._fasta = res[:]
        except:
            pass

    def save_fasta(self, filename):
        """Save FASTA file into a filename

        :param str data: the FASTA contents
        :param str filename: where to save it
        """
        if self._fasta == None:
            raise ValueError("No fasta was read or downloaded. Nothing to save.")

        fh = open(filename, "w")
        fh.write(self._fasta)
        fh.close()

    def read_fasta(self, filename):
        """Reads a FASTA file and loads it

        Type::

            >>> f = FASTA()
            >>> f.read_fasta(filename)
            >>> f.fasta

        :return: nothing

        .. warning:: If more than one FASTA is contained in the file, an error is raised
        """
        fh = open(filename, "r")
        data = fh.read()
        fh.close()

        # Is there more than one sequence ?
        data = data.split(">")[1:]
        if len(data)>1 or len(data)==0:
            raise ValueError("""Only one sequence expected to be found. Found %s. Please use MultiFASTA class instead""" % len(data))

        self._data = data
        if data.count(">sp|")>1:
            raise ValueError("""It looks like your FASTA file contains more than
            one FASTA. You must use MultiFASTA class instead""")

        self._fasta = data[:]
        self._fasta = self._fasta[0]
        if self.dbtype not in self.known_dbtypes:
            print("Only sp and gi header are recognised so far but sequence and header are loaded")

    def _interpret(self, data):
        # cleanup the data in case of empty spaces or \n characters
        return data

    def check(self):
        """Before submitting a request, any numerical digits in the query
        sequence should either be removed or replaced by appropriate letter
        codes (e.g., N for unknown nucleic acid residue or X for unknown
        amino acid residue). The nucleic acid codes supported are:

         For those programs that use amino acid query sequences (BLASTP
         and TBLASTN), the accepted amino acid codes are:
         A, B, C, D, E, F, G, H, I, K, L, M, N, P, Q, R, S, T, U, V, W, Y, Z, X, *, -

         * is for translation stop and - for gap of indeterminate length

        """
        pass

<|endoftext|>