<commit_before>import pytest

from hs_file_types.models import ModelInstanceLogicalFile

# TODO: implement tests for linking model instance aggregation to model program aggregation using the
# FK (executed_by) attribute of model instance aggregation
# The following tests cases for linking needs to be implemented:
# 1. linking aggregations within the same resource
#     a. linking one mp aggregation to one mi aggregation
#     b. linking one mp aggregation to two mi aggregations
# 2. linking aggregations that do not belong to the same resource
# 3. test linking is not allowed if user does not have access to the resource that contains the mp aggregation
# 4. test that mi aggregation does not get deleted if the linked mp aggregation gets deleted
# 5. test that mi aggregation does not get deleted if the resource containing the linked mp aggregation gets deleted

@pytest.mark.django_db(transaction=True)
def test_set_metadata(composite_resource_with_mi_aggregation, mock_irods):
    """Test that we can store all metadata items for a model instance aggregation"""

    res, user = composite_resource_with_mi_aggregation
    # mi_aggr = next(res.logical_files)
    mi_aggr = ModelInstanceLogicalFile.objects.first()
    assert isinstance(mi_aggr, ModelInstanceLogicalFile)

    # test extra metadata
    assert not mi_aggr.metadata.extra_metadata
    extra_meta = {'key1': 'value 1', 'key2': 'value 2'}
    mi_aggr.metadata.extra_metadata = extra_meta
    mi_aggr.metadata.save()
    assert mi_aggr.metadata.extra_metadata == extra_meta

    # test keywords
    assert not mi_aggr.metadata.keywords
    keywords = ['kw-1', 'kw-2']
    mi_aggr.metadata.keywords = keywords
    mi_aggr.metadata.save()
    assert mi_aggr.metadata.keywords == keywords

    # test coverage metadata
    assert not mi_aggr.metadata.coverages.all()
    value_dict = {'name': 'Name for period coverage', 'start': '1/1/2000', 'end': '12/12/2012'}
    temp_cov = mi_aggr.metadata.create_element('coverage', type='period', value=value_dict)
    assert temp_cov.value['name'] == 'Name for period coverage'
    assert temp_cov.value['start'] == '1/1/2000'
    assert temp_cov.value['end'] == '12/12/2012'
    assert mi_aggr.metadata.coverages.all().count() == 1

    value_dict = {'east': '56.45678', 'north': '12.6789', 'units': 'Decimal degree'}
    spatial_cov = mi_aggr.metadata.create_element('coverage', type='point', value=value_dict)
    assert spatial_cov.value['projection'] == 'WGS 84 EPSG:4326'
    assert spatial_cov.value['units'] == 'Decimal degree'
    assert spatial_cov.value['north'] == 12.6789
    assert spatial_cov.value['east'] == 56.45678
    assert mi_aggr.metadata.coverages.all().count() == 2
    # test model output metadata
    assert not mi_aggr.metadata.has_model_output
    mi_aggr.metadata.has_model_output = True
    mi_aggr.metadata.save()
    # test setting metadata json
    assert not mi_aggr.metadata.metadata_json
    # set mi metadata json from the content of the following file
    schema_file_path = 'pytest/assets/mi_metadata.json'
    with open(schema_file_path, 'r') as file_obj:
        meta_json = file_obj.read()
    assert len(meta_json) > 0
    mi_aggr.metadata.metadata_json = meta_json
    mi_aggr.metadata.save()
    mi_aggr = ModelInstanceLogicalFile.objects.first()
    assert mi_aggr.metadata.metadata_json

<commit_msg>[#3519] adding tests for linking model instance to model program (executed_by)<commit_after>import os

import pytest
from django.core.exceptions import PermissionDenied
from django.core.files.uploadedfile import UploadedFile

from hs_access_control.models import PrivilegeCodes
from hs_core.hydroshare import add_file_to_resource
from hs_file_types.models import ModelInstanceLogicalFile, ModelProgramLogicalFile

@pytest.mark.django_db(transaction=True)
def test_link_model_aggregations_same_resource(composite_resource_with_mi_aggregation, mock_irods):
    """Test that we can link one model instance aggregation to one model program aggregation within the same resource"""

    res, user = composite_resource_with_mi_aggregation
    assert ModelInstanceLogicalFile.objects.count() == 1
    mi_aggr = ModelInstanceLogicalFile.objects.first()
    # check that mi_aggr is not related to any model program aggregation
    assert mi_aggr.metadata.executed_by is None
    # create a model program aggregation
    file_path = 'pytest/assets/logan.vrt'
    upload_folder = None
    file_to_upload = UploadedFile(file=open(file_path, 'rb'),
                                  name=os.path.basename(file_path))

    res_file = add_file_to_resource(
        res, file_to_upload, folder=upload_folder, check_target_folder=True
    )

    assert ModelProgramLogicalFile.objects.count() == 0
    # set file to model program aggregation type
    ModelProgramLogicalFile.set_file_type(res, user, res_file.id)
    assert ModelProgramLogicalFile.objects.count() == 1
    mp_aggr = ModelProgramLogicalFile.objects.first()
    # link model instance aggregation to model program aggregation
    mi_aggr.set_link_to_model_program(user=user, model_prog_aggr=mp_aggr)
    mi_aggr = ModelInstanceLogicalFile.objects.first()
    # check that mi_aggr is related to model program aggregation
    assert mi_aggr.metadata.executed_by is not None

@pytest.mark.django_db(transaction=True)
def test_link_model_aggregations_different_resources(composite_resource_with_mi_aggregation,
                                                     composite_resource_2_with_mp_aggregation, mock_irods):
    """Test that we can link one model instance aggregation in one resource to one model program aggregation
    in another resource provided the user doing the link operation has at edit permission on the resource
    containing the model instance aggregation and view permission on the resource containing the model program
    aggregation"""

    mi_res, mi_user = composite_resource_with_mi_aggregation
    mp_res, mp_user = composite_resource_2_with_mp_aggregation
    assert mi_res != mp_res
    assert mi_user != mp_user

    authorized = mi_user.uaccess.can_change_resource(mi_res)
    assert authorized
    assert ModelInstanceLogicalFile.objects.count() == 1
    mi_aggr = ModelInstanceLogicalFile.objects.first()
    assert ModelProgramLogicalFile.objects.count() == 1
    mp_aggr = ModelProgramLogicalFile.objects.first()
    # check that mi_aggr is not related to any model program aggregation
    assert mi_aggr.metadata.executed_by is None
    # test that when the user does not have view access to the mp_res, trying to link the
    # 2 aggregations should fail
    authorized = mi_user.uaccess.can_view_resource(mp_res)
    assert not authorized
    with pytest.raises(PermissionDenied):
        mi_aggr.set_link_to_model_program(user=mi_user, model_prog_aggr=mp_aggr)
    # give mi_user view access to mp_res
    mp_user.uaccess.share_resource_with_user(mp_res, mi_user, PrivilegeCodes.VIEW)
    authorized = mi_user.uaccess.can_view_resource(mp_res)
    assert authorized
    # now we should be able to link model instance aggregation to model program aggregation
    mi_aggr.set_link_to_model_program(user=mi_user, model_prog_aggr=mp_aggr)
    mi_aggr = ModelInstanceLogicalFile.objects.first()
    # check that mi_aggr is related to model program aggregation
    assert mi_aggr.metadata.executed_by is not None
    # need to delete this resource as the fixture does not delete it
    mp_res.delete()

@pytest.mark.django_db(transaction=True)
def test_model_instance_on_model_program_delete(composite_resource_with_mi_aggregation, mock_irods):
    """Test that when we remove/delete a model program aggregation that the linked model instance aggregation does not
    get deleted"""

    res, user = composite_resource_with_mi_aggregation
    assert ModelInstanceLogicalFile.objects.count() == 1
    mi_aggr = ModelInstanceLogicalFile.objects.first()
    # check that mi_aggr is not related to any model program aggregation
    assert mi_aggr.metadata.executed_by is None
    # create a model program aggregation
    file_path = 'pytest/assets/logan.vrt'
    upload_folder = None
    file_to_upload = UploadedFile(file=open(file_path, 'rb'),
                                  name=os.path.basename(file_path))

    res_file = add_file_to_resource(
        res, file_to_upload, folder=upload_folder, check_target_folder=True
    )

    assert ModelProgramLogicalFile.objects.count() == 0
    # set file to model program aggregation type
    ModelProgramLogicalFile.set_file_type(res, user, res_file.id)
    assert ModelProgramLogicalFile.objects.count() == 1
    mp_aggr = ModelProgramLogicalFile.objects.first()
    # link model instance aggregation to model program aggregation
    mi_aggr.set_link_to_model_program(user=user, model_prog_aggr=mp_aggr)
    mi_aggr = ModelInstanceLogicalFile.objects.first()
    # check that mi_aggr is related to model program aggregation
    assert mi_aggr.metadata.executed_by is not None
    # remove mp_aggregation
    mp_aggr.remove_aggregation()
    assert ModelProgramLogicalFile.objects.count() == 0
    assert ModelInstanceLogicalFile.objects.count() == 1
    mi_aggr = ModelInstanceLogicalFile.objects.first()
    # check that mi_aggr is not related to any model program aggregation
    assert mi_aggr.metadata.executed_by is None

@pytest.mark.django_db(transaction=True)
def test_model_instance_on_model_program_resource_delete(composite_resource_with_mi_aggregation,
                                                         composite_resource_2_with_mp_aggregation, mock_irods):
    """Test that when we delete a resource containing a model program aggregation linked to a model
    instance aggregation, the model instance aggregation doesn't get deleted"""

    mi_res, mi_user = composite_resource_with_mi_aggregation
    mp_res, mp_user = composite_resource_2_with_mp_aggregation
    assert mi_res != mp_res
    assert mi_user != mp_user

    authorized = mi_user.uaccess.can_change_resource(mi_res)
    assert authorized
    assert ModelInstanceLogicalFile.objects.count() == 1
    mi_aggr = ModelInstanceLogicalFile.objects.first()
    assert ModelProgramLogicalFile.objects.count() == 1
    mp_aggr = ModelProgramLogicalFile.objects.first()
    # check that mi_aggr is not related to any model program aggregation
    assert mi_aggr.metadata.executed_by is None
    # give mi_user view access to mp_res
    mp_user.uaccess.share_resource_with_user(mp_res, mi_user, PrivilegeCodes.VIEW)
    authorized = mi_user.uaccess.can_view_resource(mp_res)
    assert authorized
    # now we should be able to link model instance aggregation to model program aggregation
    mi_aggr.set_link_to_model_program(user=mi_user, model_prog_aggr=mp_aggr)
    mi_aggr = ModelInstanceLogicalFile.objects.first()
    # check that mi_aggr is related to model program aggregation
    assert mi_aggr.metadata.executed_by is not None
    # delete mp_resource
    mp_res.delete()
    assert ModelProgramLogicalFile.objects.count() == 0
    # check that mi_aggr is not related to any model program aggregation
    mi_aggr = ModelInstanceLogicalFile.objects.first()
    assert mi_aggr.metadata.executed_by is None

@pytest.mark.django_db(transaction=True)
def test_set_metadata(composite_resource_with_mi_aggregation, mock_irods):
    """Test that we can store all metadata items for a model instance aggregation"""

    res, user = composite_resource_with_mi_aggregation
    # mi_aggr = next(res.logical_files)
    mi_aggr = ModelInstanceLogicalFile.objects.first()

    # test extra metadata
    assert not mi_aggr.metadata.extra_metadata
    extra_meta = {'key1': 'value 1', 'key2': 'value 2'}
    mi_aggr.metadata.extra_metadata = extra_meta
    mi_aggr.metadata.save()
    assert mi_aggr.metadata.extra_metadata == extra_meta

    # test keywords
    assert not mi_aggr.metadata.keywords
    keywords = ['kw-1', 'kw-2']
    mi_aggr.metadata.keywords = keywords
    mi_aggr.metadata.save()
    assert mi_aggr.metadata.keywords == keywords

    # test coverage metadata
    assert not mi_aggr.metadata.coverages.all()
    value_dict = {'name': 'Name for period coverage', 'start': '1/1/2000', 'end': '12/12/2012'}
    temp_cov = mi_aggr.metadata.create_element('coverage', type='period', value=value_dict)
    assert temp_cov.value['name'] == 'Name for period coverage'
    assert temp_cov.value['start'] == '1/1/2000'
    assert temp_cov.value['end'] == '12/12/2012'
    assert mi_aggr.metadata.coverages.all().count() == 1

    value_dict = {'east': '56.45678', 'north': '12.6789', 'units': 'Decimal degree'}
    spatial_cov = mi_aggr.metadata.create_element('coverage', type='point', value=value_dict)
    assert spatial_cov.value['projection'] == 'WGS 84 EPSG:4326'
    assert spatial_cov.value['units'] == 'Decimal degree'
    assert spatial_cov.value['north'] == 12.6789
    assert spatial_cov.value['east'] == 56.45678
    assert mi_aggr.metadata.coverages.all().count() == 2
    # test model output metadata
    assert not mi_aggr.metadata.has_model_output
    mi_aggr.metadata.has_model_output = True
    mi_aggr.metadata.save()
    # test setting metadata json
    assert not mi_aggr.metadata.metadata_json
    # set mi metadata json from the content of the following file
    schema_file_path = 'pytest/assets/mi_metadata.json'
    with open(schema_file_path, 'r') as file_obj:
        meta_json = file_obj.read()
    assert len(meta_json) > 0
    mi_aggr.metadata.metadata_json = meta_json
    mi_aggr.metadata.save()
    mi_aggr = ModelInstanceLogicalFile.objects.first()
    assert mi_aggr.metadata.metadata_json

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import requests

from copy import deepcopy
from monolithe.lib.utils.printer import Printer
from .managers import TaskManager

from monolithe.lib.utils.vsdk import VSDKUtils
from monolithe.lib.utils.urls import URLUtils
from monolithe.lib.utils.parse import ParsingUtils

API_DOCS = '/api-docs'
SCHEMA_FILEPATH = '/schema'
SWAGGER_APIS = 'apis'
SWAGGER_APIVERSION = 'apiVersion'
SWAGGER_PATH = 'path'
SPEC_EXTENSION = '.spec'

class SwaggerParser(object):
    """ Parser for Swagger description file.

        Works for both local and remote files depending on the option that are passed

    """

    def __init__(self, vsdurl, path, apiversion):
        """ Initializes the parser

            Args:
                vsdurl (string): url to the vsd
                path (string): path to a local schema folder
                apiversion (float): api version to use
        """
        self.vsdurl = vsdurl
        self.path = path
        self.apiversion = apiversion

    def run(self, filters=[]):
        """ Run the parser to grab all swagger resources

        """
        if URLUtils.is_url(self.vsdurl):
            base_path = '%s%s' % (VSDKUtils.get_vsd_url(self.vsdurl, self.apiversion), SCHEMA_FILEPATH)
            function_name = '_get_swagger_resource'
        else:
            base_path = self.path  # No need to compute a version here
            function_name = '_read_swagger_resource'

        models = self._run(base_path=base_path, function_name=function_name, filters=filters)

        return ParsingUtils.order(models)

    # Utilities

    def _run(self, base_path, function_name, filters):
        """ Run the server on the remote vsd

        """
        func = getattr(self, function_name)
        api_docs = func(base_path=base_path, path=API_DOCS)

        if SWAGGER_APIS not in api_docs or len(api_docs[SWAGGER_APIS]) == 0:
            Printer.raiseError("No apis information found in api-docs")

        if SWAGGER_APIVERSION not in api_docs:
            Printer.raiseError("No api version found in api-docs")

        if self.apiversion is None:
            self.apiversion = VSDKUtils.get_float_version(api_docs[SWAGGER_APIVERSION])

        filters = ParsingUtils.reverse_filters(filters)

        models = dict()
        task_manager = TaskManager()

        for api in api_docs[SWAGGER_APIS]:
            path = api[SWAGGER_PATH]
            task_manager.start_task(method=self._get_resource, base_path=base_path, path=path, function_name=function_name, results=models, filters=filters)

        task_manager.wait_until_exit()

        return models

    def _get_resource(self, base_path, path, function_name, results=dict(), filters=[]):
        """ Grab resource information

            Args:
                base_path: the base path where to grab information
                path: the path where to grab information
                results: the dictionary to fill with all information
                filters: list of filters

            Returns:
                Fills result dictionary
        """

        (package, model_name) = URLUtils.split_package_path(path)

        model_name = ParsingUtils.get_correct_name(model_name)
        rest_name = VSDKUtils.get_singular_name(model_name.lower())

        if len(filters) > 0 and not ParsingUtils.have_similar_strings(rest_name, filters):
            return

        func = getattr(self, function_name)
        swagger_resource = func(base_path=base_path, path=path)
        swagger_resource['package'] = package

        if model_name == 'Metadata':
            # Sadly I have to split this particular entity myself :(
            self._add_metadata_copies(resource=swagger_resource, results=results)
        else:
            results[model_name] = swagger_resource

    def _add_metadata_copies(self, resource, results):
        """ Metadata is splitted into:
            * Metadata
            * Global Metadata
            * Aggregate Metadata
        """
        metadata_info = deepcopy(resource)
        global_metadata_info = deepcopy(resource)
        aggregate_metadata_info = deepcopy(resource)

        metadata_info['apis'] = []
        global_metadata_info['apis'] = []
        aggregate_metadata_info['apis'] = []

        metadata_info['models']['Metadata']['id'] = u'Metadata'

        global_metadata_info['models']['GlobalMetadata'] = global_metadata_info['models']['Metadata']
        global_metadata_info['models'].pop('Metadata')
        global_metadata_info['models']['GlobalMetadata']['id'] = u'GlobalMetadata'

        aggregate_metadata_info['models']['AggregateMetadata'] = aggregate_metadata_info['models']['Metadata']
        aggregate_metadata_info['models'].pop('Metadata')
        aggregate_metadata_info['models']['AggregateMetadata']['id'] = u'AggregateMetadata'

        for api in resource['apis']:
            api_copy = deepcopy(api)
            if 'aggregatemetadatas' in api['path']:
                aggregate_metadata_info['apis'].append(api_copy)
            elif 'globalmetadatas' in api['path']:
                global_metadata_info['apis'].append(api_copy)
            else:
                metadata_info['apis'].append(api_copy)

        results['Metadata'] = metadata_info
        results['GlobalMetadata'] = global_metadata_info
        results['AggregateMetadata'] = aggregate_metadata_info

    # Specific methods

    def _get_swagger_resource(self, base_path, path):
        """ Make an HTTP request to retrieve information

        """
        url = '%s%s' % (base_path, path)

        try:  # Ugly hack due to Java issue: http://mvjira.mv.usa.alcatel.com/browse/VSD-546
            response = requests.get(url=url, verify=False)
        except requests.exceptions.SSLError:
            response = requests.get(url=url, verify=False)

        if response.status_code != 200:
            Printer.raiseError("[HTTP %s] An error occured while trying to retrieve information at url: %s" % (response.status_code, url))

        data = None
        try:
            data = response.json()
        except:
            Printer.raiseError("Could not load properly json from %s" % url)

        return data

    def _read_swagger_resource(self, base_path, path):
        """ Read the file to retrieve information

        """
        filepath = '%s%s' % (base_path, path)
        return ParsingUtils.parseJSON(filepath)

class SpecificationParser(object):
    """ Parse specifications directory

    """

    @classmethod
    def run(cls, directory):
        """ Grab all specification in given directory and return a dictionary of specs.

            Returns:
                A dictionary of specification

        """

        if not os.path.isdir(directory):
            Printer.raiseError("[File Path] Not a directory %s" % (directory))

        specs = dict()
        filenames = []
        for filename in os.listdir(directory):
            if filename.endswith(SPEC_EXTENSION):
                filenames.append(filename)
                filepath = '%s/%s' % (directory, filename)

                data = ParsingUtils.parseJSON(filepath)
                name = data['model']['entityName']

                specs[name] = data

        Printer.success('Parsed %s specifications' % len(filenames))
        return ParsingUtils.order(specs)
<commit_msg>use new APIs<commit_after># -*- coding: utf-8 -*-

import os
import requests

from copy import deepcopy
from monolithe.lib.utils.printer import Printer
from .managers import TaskManager

from monolithe.lib.utils.vsdk import VSDKUtils
from monolithe.lib.utils.urls import URLUtils
from monolithe.lib.utils.parse import ParsingUtils

API_DOCS = '/api-docs'
SCHEMA_FILEPATH = '/schema'
SWAGGER_APIS = 'apis'
SWAGGER_APIVERSION = 'apiVersion'
SWAGGER_PATH = 'path'
SPEC_EXTENSION = '.spec'

class SwaggerParser(object):
    """ Parser for Swagger description file.

        Works for both local and remote files depending on the option that are passed

    """

    def __init__(self, vsdurl, path, apiversion):
        """ Initializes the parser

            Args:
                vsdurl (string): url to the vsd
                path (string): path to a local schema folder
                apiversion (float): api version to use
        """
        self.vsdurl = vsdurl
        self.path = path
        self.apiversion = apiversion

    def run(self, filters=[]):
        """ Run the parser to grab all swagger resources

        """
        if URLUtils.is_url(self.vsdurl):
            base_path = '%s%s' % (VSDKUtils.get_vsd_url(self.vsdurl, self.apiversion), SCHEMA_FILEPATH)
            function_name = '_get_swagger_resource'
        else:
            base_path = self.path  # No need to compute a version here
            function_name = '_read_swagger_resource'

        models = self._run(base_path=base_path, function_name=function_name, filters=filters)

        return ParsingUtils.order(models)

    # Utilities

    def _run(self, base_path, function_name, filters):
        """ Run the server on the remote vsd

        """
        func = getattr(self, function_name)
        api_docs = func(base_path=base_path, path=API_DOCS)

        if SWAGGER_APIS not in api_docs or len(api_docs[SWAGGER_APIS]) == 0:
            Printer.raiseError("No apis information found in api-docs")

        if SWAGGER_APIVERSION not in api_docs:
            Printer.raiseError("No api version found in api-docs")

        if self.apiversion is None:
            self.apiversion = VSDKUtils.get_float_version(api_docs[SWAGGER_APIVERSION])

        filters = ParsingUtils.reverse_filters(filters)

        models = dict()
        task_manager = TaskManager()

        for api in api_docs[SWAGGER_APIS]:
            path = api[SWAGGER_PATH]
            task_manager.start_task(method=self._get_resource, base_path=base_path, path=path, function_name=function_name, results=models, filters=filters)

        task_manager.wait_until_exit()

        return models

    def _get_resource(self, base_path, path, function_name, results=dict(), filters=[]):
        """ Grab resource information

            Args:
                base_path: the base path where to grab information
                path: the path where to grab information
                results: the dictionary to fill with all information
                filters: list of filters

            Returns:
                Fills result dictionary
        """

        (package, model_name) = URLUtils.split_package_path(path)

        model_name = ParsingUtils.get_correct_name(model_name)
        rest_name = VSDKUtils.get_singular_name(model_name.lower())

        if len(filters) > 0 and not ParsingUtils.have_similar_strings(rest_name, filters):
            return

        func = getattr(self, function_name)
        swagger_resource = func(base_path=base_path, path=path)
        swagger_resource['package'] = package

        if model_name == 'Metadata':
            # Sadly I have to split this particular entity myself :(
            self._add_metadata_copies(resource=swagger_resource, results=results)
        else:
            results[model_name] = swagger_resource

    def _add_metadata_copies(self, resource, results):
        """ Metadata is splitted into:
            * Metadata
            * Global Metadata
            * Aggregate Metadata
        """
        metadata_info = deepcopy(resource)
        global_metadata_info = deepcopy(resource)
        aggregate_metadata_info = deepcopy(resource)

        metadata_info['apis'] = []
        global_metadata_info['apis'] = []
        aggregate_metadata_info['apis'] = []

        metadata_info['models']['Metadata']['id'] = u'Metadata'

        global_metadata_info['models']['GlobalMetadata'] = global_metadata_info['models']['Metadata']
        global_metadata_info['models'].pop('Metadata')
        global_metadata_info['models']['GlobalMetadata']['id'] = u'GlobalMetadata'

        aggregate_metadata_info['models']['AggregateMetadata'] = aggregate_metadata_info['models']['Metadata']
        aggregate_metadata_info['models'].pop('Metadata')
        aggregate_metadata_info['models']['AggregateMetadata']['id'] = u'AggregateMetadata'

        for api in resource['apis']:
            api_copy = deepcopy(api)
            if 'aggregatemetadatas' in api['path']:
                aggregate_metadata_info['apis'].append(api_copy)
            elif 'globalmetadatas' in api['path']:
                global_metadata_info['apis'].append(api_copy)
            else:
                metadata_info['apis'].append(api_copy)

        results['Metadata'] = metadata_info
        results['GlobalMetadata'] = global_metadata_info
        results['AggregateMetadata'] = aggregate_metadata_info

    # Specific methods

    def _get_swagger_resource(self, base_path, path):
        """ Make an HTTP request to retrieve information

        """
        url = '%s%s' % (base_path, path)

        try:  # Ugly hack due to Java issue: http://mvjira.mv.usa.alcatel.com/browse/VSD-546
            response = requests.get(url=url, verify=False)
        except requests.exceptions.SSLError:
            response = requests.get(url=url, verify=False)

        if response.status_code != 200:
            Printer.raiseError("[HTTP %s] An error occured while trying to retrieve information at url: %s" % (response.status_code, url))

        data = None
        try:
            data = response.json()
        except:
            Printer.raiseError("Could not load properly json from %s" % url)

        return data

    def _read_swagger_resource(self, base_path, path):
        """ Read the file to retrieve information

        """
        filepath = '%s%s' % (base_path, path)
        return ParsingUtils.parseFile(filepath)

class SpecificationParser(object):
    """ Parse specifications directory

    """

    @classmethod
    def run(cls, directory):
        """ Grab all specification in given directory and return a dictionary of specs.

            Returns:
                A dictionary of specification

        """

        if not os.path.isdir(directory):
            Printer.raiseError("[File Path] Not a directory %s" % (directory))

        specs = dict()
        filenames = []
        for filename in os.listdir(directory):
            if filename.endswith(SPEC_EXTENSION):
                filenames.append(filename)
                filepath = '%s/%s' % (directory, filename)

                data = ParsingUtils.parseFile(filepath)
                name = data['model']['entityName']

                specs[name] = data

        Printer.success('Parsed %s specifications' % len(filenames))
        return ParsingUtils.order(specs)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import logging

import threading

from evdev import ecodes, InputDevice, list_devices

from select import select

logger = logging.getLogger(__name__)

class InputThread(threading.Thread):
    def __init__(self):
        super(InputThread, self).__init__()
        self.name = "Input Thread"

        self._stop = threading.Event()
        self.devices_by_fn = {}
        self.devices_by_fd = {}
        self.handlers_by_fd = {}

    def stop(self):
        self._stop.set()
        logger.debug("Thread {0} is asked to stop".format(self.name))

    def register_handler(self, handler):
        if handler.device_fn in self.devices_by_fn:
            device = self.devices_by_fn[handler.device_fn]
        else:
            device = InputDevice(handler.device_fn)
            self.devices_by_fn[handler.device_fn] = device
            self.devices_by_fd[device.fd] = device
            self.handlers_by_fd[device.fd] = []

        # Check if device has needed event
        capabilities = device.capabilities()
        if handler.event_type in capabilities:
            for event_code in handler.event_codes:
                if event_code in capabilities[handler.event_type]:
                    self.handlers_by_fd[device.fd].append(handler)
                else:
                    logger.warning('Event {0} not found in input device "{1}"'.format(
                        ecodes.bytype[handler.event_type][handler.event_code], device.name))
                    return False
        else:
            logger.warning(
                'Input device "{1}" has no capability {0}'.format(ecodes.EV[handler.event_type], device.name))
            return False

        return True

    def register_handlers(self, handlers):
        for handler in handlers:
            self.register_handler(handler)

    def run(self):
        while not self._stop.isSet():
            r, w, x = select(self.devices_by_fd, [], [], 10)
            for fd in r:
                for event in self.devices_by_fd[fd].read():
                    for handler in self.handlers_by_fd[fd]:
                        handler.check(event)
        logger.debug("Thread {0} stopped".format(self.name))

class Handler(object):
    def __init__(self, device_fn, event_type, event_codes):
        self.device_fn = device_fn

        if event_type in ecodes.ecodes:
            self.event_type = ecodes.ecodes[event_type]
        else:
            logger.error('Event type {0} unknown'.format(event_type))

        self.event_codes = list()
        for event_code in event_codes:
            if event_code in ecodes.ecodes:
                self.event_codes.append(ecodes.ecodes[event_code])
            else:
                logger.error('Event {0} not found for {1} events'.format(event_code, event_type))

    def check(self, event):
        if self.event_type == event.type:
            if event.code in self.event_codes:
                self.handle(event, ecodes.ecodes[event.code])

    def handle(self, event, keycode):
        pass
<commit_msg>Don't use ecodes<commit_after>from __future__ import unicode_literals

import logging

import threading

from evdev import ecodes, InputDevice, list_devices

from select import select

logger = logging.getLogger(__name__)

class InputThread(threading.Thread):
    def __init__(self):
        super(InputThread, self).__init__()
        self.name = "Input Thread"

        self._stop = threading.Event()
        self.devices_by_fn = {}
        self.devices_by_fd = {}
        self.handlers_by_fd = {}

    def stop(self):
        self._stop.set()
        logger.debug("Thread {0} is asked to stop".format(self.name))

    def register_handler(self, handler):
        if handler.device_fn in self.devices_by_fn:
            device = self.devices_by_fn[handler.device_fn]
        else:
            device = InputDevice(handler.device_fn)
            self.devices_by_fn[handler.device_fn] = device
            self.devices_by_fd[device.fd] = device
            self.handlers_by_fd[device.fd] = []

        # Check if device has needed event
        capabilities = device.capabilities()
        if handler.event_type in capabilities:
            for event_code in handler.event_codes:
                if event_code in capabilities[handler.event_type]:
                    self.handlers_by_fd[device.fd].append(handler)
                else:
                    logger.warning('Event {0} not found in input device "{1}"'.format(
                        ecodes.bytype[handler.event_type][handler.event_code], device.name))
                    return False
        else:
            logger.warning(
                'Input device "{1}" has no capability {0}'.format(ecodes.EV[handler.event_type], device.name))
            return False

        return True

    def register_handlers(self, handlers):
        for handler in handlers:
            self.register_handler(handler)

    def run(self):
        while not self._stop.isSet():
            r, w, x = select(self.devices_by_fd, [], [], 10)
            for fd in r:
                for event in self.devices_by_fd[fd].read():
                    for handler in self.handlers_by_fd[fd]:
                        handler.check(event)
        logger.debug("Thread {0} stopped".format(self.name))

class Handler(object):
    def __init__(self, device_fn, event_type, event_codes):
        self.device_fn = device_fn

        if event_type in ecodes.ecodes:
            self.event_type = ecodes.ecodes[event_type]
        else:
            logger.error('Event type {0} unknown'.format(event_type))

        self.event_codes = list()
        for event_code in event_codes:
            if event_code in ecodes.ecodes:
                self.event_codes.append(ecodes.ecodes[event_code])
            else:
                logger.error('Event {0} not found for {1} events'.format(event_code, event_type))

    def check(self, event):
        if self.event_type == event.type:
            if event.code in self.event_codes:
                self.handle(event, event.code)

    def handle(self, event, keycode):
        pass
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2016 by Kevin Ha
"""
Calculates motif scores for all PWMs in a given set of sequences in FASTA
format

Requires python 2.7+

This tool was motivated by the RNA RBP motif scanning tool from CISBP-RNA:
http://cisbp-rna.ccbr.utoronto.ca/TFTools.php
"""

import sys
import time
import glob
import fileinput
import os
import os.path
import warnings
import argparse
import ast
from collections import defaultdict
import multiprocessing
import pandas as pd
from itertools import izip, repeat
from BioAddons.Alphabet import *
from BioAddons.motifs import matrix
from BioAddons.SeqStruct import SeqStruct
from Bio import motifs, SeqIO
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from Bio.Alphabet import RNAAlphabet, IUPAC

__version__ = 'v0.7.2'

def getoptions():
    desc = "Scan sequence for motif binding sites. Results sent to STDOUT."
    parser = argparse.ArgumentParser(description=desc, version=__version__)
    parser.add_argument('fastafiles', metavar='FASTA', nargs='*',
                        help="Input sequence and structure FASTA files")
    parser.add_argument('-d', '--pfm_dir', dest="pfm_dir",
                        help="Directory of PFMs or a single file [%(default)s]")
    parser.add_argument('-p', '--pseudocount', type=float,
                        dest="pseudocount", default=0,
                        help="Pseudocount for normalizing PFM. [%(default)s]")
    #parser.add_argument('-r', '--rbpinfo', type='string', dest='rbpinfo',
        #default=os.path.dirname(os.path.abspath(__file__)) +
        #"/db/RBP_Information_all_motifs.txt",
        #help="RBP info for adding meta data to results. [%(default)s]")
    parser.add_argument('-t', '--type', dest='seqtype',
                        choices=['DNA', 'RNA', 'SS', 'RNASS'], default="RNA",
                        help=("Alphabet of PFM (DNA|RNA|SS for "
                              "ContextualSecondaryStructure). "
                              "[%(default)s]"))
    parser.add_argument('-m', '--minscore', type=float, dest='minscore',
                        default=6,
                        help="Minimum score for motif hits. [%(default)s]")
    parser.add_argument('-s', '--seq', dest='testseq', default=None,
                        help=("Supply a test sequence to scan. FASTA files "
                              " will be ignored."))
    parser.add_argument('-c', '--cores', type=int, default=8, dest="cores",
                        help="Number of processing cores [%(default)s]")
    #parser.add_argument('-x', '--excel', action="store_true", dest="excel",
            #default=False,
            #help="Format the RBP_ID column with =HYPERLINK(url) for " +
            #"import into Excel [%(default)s]")
    parser.add_argument('-u', '--uniformbg', action="store_true",
                        default=False, dest="uniform_background",
                        help=("Use uniform background for calculating "
                              "log-odds [%(default)s]. Default is to compute "
                              "background from input sequences. This option "
                              "is mutually exclusive with -B."))
    parser.add_argument('-B', '--bgonly', action="store_true", default=False,
                        dest="bgonly",
                        help=("Compute background probabilities from input "
                              "sequences (STDOUT) and exit. Useful for "
                              "getting background probabilities from a "
                              "superset of sequences. Then, these values can "
                              "be subsequently supplied using -b. "
                              "[%(default)s]"))
    parser.add_argument('-b', '--bg', default=None, dest="custom_background",
                        help=("Load file of pre-computed background "
                              "probabilities"))
    parser.add_argument('-x', '--debug', action="store_true", default=False,
                        dest="debug",
                        help=("Turn on debug mode  "
                              "(aka disable parallelization) [%(default)s]"))
    args = parser.parse_args()

    if not (args.pfm_dir or args.bgonly):
        parser.error("Must specify the PFM directory with -d")

    if args.uniform_background is True and args.custom_background is not None:
        parser.error("You cannot set uniform and custom background options "
                     "at the same time\n")

    if len(args.fastafiles) == 2:
        args.seqtype = 'RNASS'

    if args.seqtype == 'SS':
        args.alphabet = ContextualSecondaryStructure()
    elif args.seqtype == 'RNA':
        args.alphabet = IUPAC.IUPACUnambiguousRNA()
    elif args.seqtype == 'RNASS':
        args.alphabet = ContextualSequenceSecondaryStructure()
    else:
        args.alphabet = IUPAC.IUPACUnambiguousDNA()

    return args

def batch_iterator(iterator, batch_size):
    """Returns lists of length batch_size.

    This can be used on any iterator, for example to batch up
    SeqRecord objects from Bio.SeqIO.parse(...), or to batch
    Alignment objects from Bio.AlignIO.parse(...), or simply
    lines from a file handle.

    This is a generator function, and it returns lists of the
    entries from the supplied iterator.  Each list will have
    batch_size entries, although the final list may be shorter.

    Source: http://biopython.org/wiki/Split_large_file
    """
    entry = True  # Make sure we loop once
    while entry:
        batch = []
        while len(batch) < batch_size:
            try:
                entry = iterator.next()
            except StopIteration:
                entry = None
            if entry is None:
                # End of file
                break
            batch.append(entry)
        if batch:
            yield batch

def load_motifs(dbdir, *args):
    """
    Load all motifs from given directory. Will look for *.txt files
    """
    motifs_set = {}
    print >> sys.stderr, "Loading motifs ",
    tic = time.time()
    if os.path.isfile(dbdir):
        mfiles = [dbdir]
    else:
        mfiles = glob.glob(dbdir + "/*.txt")

    for mfile in mfiles:
        try:
            motif_id = os.path.splitext(os.path.basename(mfile))[0]
            motifs_set[motif_id] = pfm2pssm(mfile, *args)
        except ValueError:
            print >> sys.stderr, "\nFailed to load motif %s" % mfile
        except KeyError:
            print >> sys.stderr, "\nFailed to load motif %s" % mfile
            print >> sys.stderr, "Check that you are using the correct --type"
            raise
        except:
            print "Unexpected error:", sys.exc_info()[0]
            raise
        print >> sys.stderr, "\b.",
        sys.stderr.flush()
    toc = time.time()
    print >> sys.stderr, "done in %0.2f seconds!" % (float(toc - tic))
    print >> sys.stderr, "Found %d motifs" % len(motifs_set)
    return motifs_set

def pfm2pssm(pfm_file, pseudocount, alphabet, background=None):
    """
    Convert load PFM and convert it to PSSM (take the log_odds)
    """
    pfm = pd.read_table(pfm_file)
    pfm = pfm.drop(pfm.columns[0], 1).to_dict(orient='list')
    pfm = motifs.Motif(alphabet=alphabet, counts=pfm)
    pfm = pfm.counts.normalize(pseudocount)

    # Can optionally add background, but for now assuming uniform probability
    pssm = pfm.log_odds(background=background)

    pssm = matrix.ExtendedPositionSpecificScoringMatrix(pssm.alphabet, pssm)

    return pssm

def preprocess_seq(seqrec, alphabet):
    """Pre-process the SeqRecord by setting the alphabet and performing
    transcription if necessary.

    Return Seq object
    """
    if not isinstance(seqrec, SeqRecord):
        raise TypeError("SeqRecord object must be supplied")

    if isinstance(alphabet, IUPAC.IUPACAmbiguousRNA) and \
        not isinstance(seqrec.seq.alphabet, RNAAlphabet):
        # If RNA alphabet is specified and input sequences are in DNA, we need
        # to transcribe them to RNA
        try:
            seq = seqrec.seq.transcribe()
            seq.alphabet = alphabet
            seq = seq.upper()
        except:
            raise
    else:
        seq = seqrec.seq

    ## If strand is specified, reverse-complement the sequence
    #strand_match = re.search(r'strand=([+-])', seqrec.description)
    #if strand_match and strand_match.group(1) == "-":
        #seq = seq.reverse_complement()

    return seq

def collect(motif_hits, seqtype):
    """ Finalize results into a DataFrame for output
    """
    columns=['Motif_ID', 'Start', 'End', 'Sequence', 'LogOdds']

    if seqtype == 'RNASS':
        columns.append('RNA_Sequence')
        columns.append('Structure_Sequence')

    # Create DataFrame from motif hits
    hits = pd.DataFrame(motif_hits, columns=columns)

    # Merge metadata with hits
    return hits.sort_values(['Start', 'Motif_ID'])

def scan(pssm, seq, minscore, motif_id):
    results = []
    for position, score in pssm.search(seq, threshold=minscore, both=False):
        end_position = position + len(pssm.consensus)

        fragment = seq[position:end_position]
        #if isinstance(seq.alphabet, IUPAC.IUPACUnambiguousRNA):
            #fragment = fragment.transcribe()

        values = [motif_id,
                  position + 1, end_position,
                  str(fragment),
                  round(score, 3)]
        if isinstance(seq, SeqStruct):
            values.extend(seq.reverse_convert(position, end_position))
        results.append(values)
    return results

def scan_all(seqrecord, *args):
    """
    Scan seq for all motifs in pssms
    """
    pssms = args[0]
    opts = args[1]
    hits = []

    seq = preprocess_seq(seqrecord, opts.alphabet)
    for motif_id, pssm in pssms.iteritems():
        results = scan(pssm, seq, opts.minscore, motif_id)
        hits.extend(results)

    # Collect results
    final = collect(hits, opts.seqtype)
    return final

def _scan_all_star(a_b):
    return scan_all(*a_b)

def generate_seqstruct(seqi, structi):
    """For RNASS, combined sequence and structure to generate a new
    SeqRecord with alphabet ContextualSequenceSecondaryStructure
    """
    for seqrec, structrec in zip(seqi, structi):
        rna = preprocess_seq(seqrec, IUPAC.IUPACUnambiguousRNA())
        yield SeqRecord(SeqStruct(rna, structrec.seq),
                        id=seqrec.id, name=seqrec.name,
                        description=seqrec.description)

def parse_sequences(fastas, alphabet):
    """Load FASTA sequence and return SeqRecord iterator
    """
    fin1 = fileinput.input(fastas[0],
                           openhook=fileinput.hook_compressed)
    seqiter = SeqIO.parse(fin1, 'fasta')

    if len(fastas) == 2:
        fin2 = fileinput.input(fastas[1],
                               openhook=fileinput.hook_compressed)
        structiter = SeqIO.parse(fin2, 'fasta')
        seqiter = generate_seqstruct(seqiter, structiter)

    return seqiter

def compute_background(fastas, alphabet, verbose=True):
    """Compute background probabiilities from all input sequences
    """
    print >> sys.stderr, "Calculating background probabilities..."
    content = defaultdict(int)
    total = len(alphabet.letters)       # add psuedocount for each letter
    seqiter = parse_sequences(fastas, alphabet)

    for seqrecord in seqiter:
        seqobj = preprocess_seq(seqrecord, alphabet)
        for letter in alphabet.letters:
            amount = seqobj.count(letter)
            content[letter] += amount
            total += amount
    pct_sum = 0

    for letter, count in content.iteritems():
        content[letter] = (float(count) + 1) / total    # add pseudocount
        if content[letter] <= 0.05:
            warnings.warn("Letter %s has low content: %0.2f" % (letter, content[letter]), Warning)
        pct_sum += content[letter]

    if verbose: print >> sys.stderr, dict(content)
    assert abs(1.0 - pct_sum) < 0.0001, "Background sums to %f" % pct_sum
    return content

def main():
    args = getoptions()
    tic = time.time()
    final = pd.DataFrame()
    count = 0

    # Calculate sequence background from input
    if args.uniform_background or args.testseq is not None:
        print >> sys.stderr, "Using uniform background probabilities"
        bg = None
    else:
        if args.custom_background is not None:
            print >> sys.stderr, ("Reading custom background probabilities "
                                  "from %s" % args.custom_background)
            # load custom background
            # http://stackoverflow.com/a/11027069
            with open(args.custom_background, 'r') as fin:
                bg = fin.read()
                bg = ast.literal_eval(bg)
                print >> sys.stderr, dict(bg)
        else:
            bg = compute_background(args.fastafiles, args.alphabet, args.cores)

            if args.bgonly:
                # Print background probabilities and quit
                print dict(bg)
                sys.exit()

    # Load PFMs
    pssms = load_motifs(args.pfm_dir, args.pseudocount, args.alphabet, bg)

    if args.testseq is not None:
        #seq = _set_seq(SeqRecord(Seq(args.testseq)), args.alphabet)
        final = scan_all(SeqRecord(Seq(args.testseq)), pssms, args)
        final['Sequence_ID'] = 'testseq'
        final['Description'] = ''
        count += 1
    else:
        print >> sys.stderr, "Scanning sequences "

        results = []
        seq_iter = parse_sequences(args.fastafiles, args.alphabet)

        if args.debug:
            for seqrecord in seq_iter:
                hits = scan_all(seqrecord, pssms, args)
                hits['Sequence_ID'] = seqrecord.id
                hits['Description'] = seqrecord.description
                results.append(hits)
                count += 1
        else:
            p = multiprocessing.Pool(args.cores)
            for i, batch in enumerate(batch_iterator(seq_iter, 500)):
                batch_results = p.map(_scan_all_star, izip(batch,
                                                           repeat(pssms),
                                                           repeat(args)
                                                          )
                                     )

                # Process each result
                for j, hits in enumerate(batch_results):
                    if hits is None:
                        continue
                    hits['Sequence_ID'] = batch[j].id
                    hits['Description'] = batch[j].description
                    count += 1
                    results.append(hits)
            p.close()

        if len(results) != 0:
            final = pd.concat(results)

    cols = final.columns.tolist()
    cols = cols[-2:] + cols[:-2]
    final[cols].to_csv(sys.stdout, sep="\t", index=False)
    toc = time.time()

    runtime = float(toc - tic)
    if runtime > 60:
        print >> sys.stderr, "Done in %0.4f minutes!" % (runtime / 60)
    else:
        print >> sys.stderr, "Done in %0.4f seconds!" % (runtime)
    print >> sys.stderr, "Processed %d sequences" % count

if __name__ == '__main__':
    main()
<commit_msg>Remove need for SeqStruct and its arbitrary alphabet<commit_after>#!/usr/bin/env python
# Copyright 2016-2017 by Kevin Ha
"""
Calculates motif scores for all PFMs in a given set of sequences in FASTA
format

Requires python 2.7+

This tool was motivated by the RNA RBP motif scanning tool from CISBP-RNA:
http://cisbp-rna.ccbr.utoronto.ca/TFTools.php
"""

import sys
import time
import fileinput
import os
import os.path
import warnings
import argparse
import ast
from collections import defaultdict
import multiprocessing
import pandas as pd
from itertools import izip, repeat
from BioAddons.Alphabet import ContextualSecondaryStructure
from BioAddons.motifs import matrix
from Bio import motifs, SeqIO
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from Bio.Alphabet import RNAAlphabet, IUPAC

__version__ = 'v0.8.0'

def getoptions():
    desc = "Scan sequence for motif binding sites. Results sent to STDOUT."
    parser = argparse.ArgumentParser(description=desc, version=__version__)
    parser.add_argument('fastafiles', metavar='FASTA', nargs='*',
                        help="Input sequence and structure FASTA files")
    pfm_grp = parser.add_argument_group("PFM options")
    pfm_grp.add_argument('-p', '--pfm_seq', dest="pfm_seq", type=str,
                        help="Sequence PFM")
    pfm_grp.add_argument('-q', '--pfm_struct', dest="pfm_struct", type=str,
                        help="Structure PFM")

    parser.add_argument('-C', '--pseudocount', type=float,
                        dest="pseudocount", default=0,
                        help="Pseudocount for normalizing PFM. [%(default)s]")
    parser.add_argument('-m', '--minscore', type=float, dest='minscore',
                        default=6,
                        help="Minimum score for motif hits. [%(default)s]")
    parser.add_argument('-t', '--testseq', dest='testseq', default=None,
                        help=("Supply a test sequence to scan. FASTA files "
                              "will be ignored. Can supply sequence and "
                              "structure as single string separated by comma."))
    parser.add_argument('-c', '--cores', type=int, default=8, dest="cores",
                        help="Number of processing cores [%(default)s]")

    bg_grp = parser.add_argument_group('Background frequency options')
    bg_grp.add_argument('-u', '--uniformbg', action="store_true",
                        default=False, dest="uniform_background",
                        help=("Use uniform background for calculating "
                              "log-odds [%(default)s]. Default is to compute "
                              "background from input sequences. This option "
                              "is mutually exclusive with -B."))
    bg_grp.add_argument('-g', '--bgonly', action="store_true", default=False,
                        dest="bgonly",
                        help=("Compute background probabilities from input "
                              "sequences (STDOUT) and exit. Useful for "
                              "getting background probabilities from a "
                              "superset of sequences. Then, these values can "
                              "be subsequently supplied using -b. "
                              "[%(default)s]"))
    bg_grp.add_argument('-b', '--bg_seq', default=None, dest="bg_seq",
                        help=("Load file of pre-computed background "
                              "probabilities for nucleotide sequences"))
    bg_grp.add_argument('-B', '--bg_struct', default=None, dest="bg_struct",
                        help=("Load file of pre-computed background "
                          "probabilities for nucleotide sequences"))
    parser.add_argument('-x', '--debug', action="store_true", default=False,
                        dest="debug",
                        help=("Turn on debug mode  "
                              "(aka disable parallelization) [%(default)s]"))
    args = parser.parse_args()

    if not (args.pfm_seq or args.pfm_struct or args.bgonly):
        parser.error("Must specify PFMs with -p and/or -q")

    if args.uniform_background and (args.bg_seq or args.bg_struct):
        parser.error("You cannot set uniform and custom background options "
                     "at the same time\n")

    nfiles = len(args.fastafiles)

    if nfiles == 2:
        if not (args.pfm_seq or args.pfm_struct):
            parser.error("Missing PFMs")
        args.seq_type == "RNASS"
    else:   # nfiles == 1
        if args.pfm_seq and args.pfm_struct and not args.testseq:
            parser.error("Can't specify two PFMs with one input file")
        elif args.pfm_seq and args.pfm_struct and args.testseq:
            args.seq_type = "RNASS"
        elif args.pfm_seq:
            args.seq_type = "RNA"
        elif args.pfm_struct:
            args.seq_type = "SS"
        else:
            parser.error("Must specify PFMs with -p and/or -q")

    return args

def batch_iterator(iterator, batch_size):
    """Returns lists of length batch_size.

    This can be used on any iterator, for example to batch up
    SeqRecord objects from Bio.SeqIO.parse(...), or to batch
    Alignment objects from Bio.AlignIO.parse(...), or simply
    lines from a file handle.

    This is a generator function, and it returns lists of the
    entries from the supplied iterator.  Each list will have
    batch_size entries, although the final list may be shorter.

    Source: http://biopython.org/wiki/Split_large_file
    """
    entry = True  # Make sure we loop once
    while entry:
        batch = []
        while len(batch) < batch_size:
            try:
                entry = iterator.next()
            except StopIteration:
                entry = None
            if entry is None:
                # End of file
                break
            batch.append(entry)
        if batch:
            yield batch

def load_motif(pfm_file, *args):
    """ Load PFM
    """
    motifs_set = {}
    print >> sys.stderr, "Loading PFM %s" % pfm_file,
    tic = time.time()
    try:
        motif_id = os.path.splitext(os.path.basename(pfm_file))[0]
        motifs_set[motif_id] = pfm2pssm(pfm_file, *args)
    except ValueError:
        print >> sys.stderr, "\nFailed to load motif %s" % pfm_file
    except KeyError:
        print >> sys.stderr, "\nFailed to load motif %s" % pfm_file
        print >> sys.stderr, "Check that you are using the correct --type"
        raise
    except:
        print "Unexpected error:", sys.exc_info()[0]
        raise
    print >> sys.stderr, "\b.",
    sys.stderr.flush()
    toc = time.time()
    print >> sys.stderr, "done in %0.2f seconds!" % (float(toc - tic))
    print >> sys.stderr, "Found %d motifs" % len(motifs_set)
    return motifs_set

def pfm2pssm(pfm_file, pseudocount, alphabet, background=None):
    """
    Convert load PFM and convert it to PSSM (take the log_odds)
    """
    pfm = pd.read_table(pfm_file)
    pfm = pfm.drop(pfm.columns[0], 1).to_dict(orient='list')
    pfm = motifs.Motif(alphabet=alphabet, counts=pfm)
    pfm = pfm.counts.normalize(pseudocount)

    pssm = pfm.log_odds(background=background)

    pssm = matrix.ExtendedPositionSpecificScoringMatrix(pssm.alphabet, pssm)

    return pssm

def preprocess_seq(seqrec, alphabet):
    """Pre-process the SeqRecord by setting the alphabet and performing
    transcription if necessary.

    Return Seq object
    """
    if not isinstance(seqrec, SeqRecord):
        raise TypeError("SeqRecord object must be supplied")

    if isinstance(alphabet, IUPAC.IUPACAmbiguousRNA) and \
        not isinstance(seqrec.seq.alphabet, RNAAlphabet):
        # If RNA alphabet is specified and input sequences are in DNA, we need
        # to transcribe them to RNA
        try:
            seq = seqrec.seq.transcribe()
            seq.alphabet = alphabet
            seq = seq.upper()
        except:
            raise
    else:
        seq = seqrec.seq

    ## If strand is specified, reverse-complement the sequence
    #strand_match = re.search(r'strand=([+-])', seqrec.description)
    #if strand_match and strand_match.group(1) == "-":
        #seq = seq.reverse_complement()

    return seq

def collect(motif_hits):
    """ Finalize results into a DataFrame for output
    """
    columns=['Motif_ID', 'Start', 'End', 'Sequence', 'LogOdds']

    # Create DataFrame from motif hits
    hits = pd.DataFrame(motif_hits, columns=columns)

    # Merge metadata with hits
    return hits.sort_values(['Start', 'Motif_ID'])

def scan(pssm, seq, alphabet, minscore):
    results = []
    (motif_id, pm) = pssm.items()[0]
    for position, score in pm.search(seq, threshold=minscore, both=False):
        end_position = position + len(pm.consensus)

        fragment = seq[position:end_position]
        #if isinstance(seq.alphabet, IUPAC.IUPACUnambiguousRNA):
            #fragment = fragment.transcribe()

        values = [motif_id,
                  position + 1, end_position,
                  str(fragment),
                  round(score, 3)]
        results.append(values)
    return results

def scan_all(seqrecord, pssm, alphabet, *args):
    """ Scan seq for all motifs in pssms
    """

    seq = preprocess_seq(seqrecord, alphabet)
    results = scan(pssm, seq, alphabet, *args)

    columns=['Motif_ID', 'Start', 'End', 'Sequence', 'LogOdds']
    final = pd.DataFrame(results, columns=columns)
    return final.sort_values(['Start', 'Motif_ID'])

def _scan_all_star(a_b):
    return scan_all(*a_b)

def parse_sequences(fasta_file):
    """Load FASTA sequence and return SeqRecord iterator
    """
    fin = fileinput.input(fasta_file, openhook=fileinput.hook_compressed)
    return SeqIO.parse(fin, 'fasta')

def compute_background(fastas, alphabet, verbose=True):
    """ Compute background probabiilities from all input sequences
    """
    print >> sys.stderr, "Calculating background probabilities..."
    content = defaultdict(int)
    total = len(alphabet.letters)       # add psuedocount for each letter
    seqiter = parse_sequences(fastas)

    for seqrecord in seqiter:
        seqobj = preprocess_seq(seqrecord, alphabet)
        for letter in alphabet.letters:
            amount = seqobj.count(letter)
            content[letter] += amount
            total += amount
    pct_sum = 0

    for letter, count in content.iteritems():
        content[letter] = (float(count) + 1) / total    # add pseudocount
        if content[letter] <= 0.05:
            warnings.warn("Letter %s has low content: %0.2f" \
                % (letter, content[letter]), Warning)
        pct_sum += content[letter]

    if verbose: print >> sys.stderr, dict(content)
    assert abs(1.0 - pct_sum) < 0.0001, "Background sums to %f" % pct_sum
    return content

def load_background(bg_file, uniform, *args):
    """ Load background probabilities if available, otherwise compute from
    input files or use uniform
    """
    if bg_file:
        print >> sys.stderr, ("Reading custom background probabilities "
            "from %s" % file)
        # load custom background
        # http://stackoverflow.com/a/11027069
        with open(file, 'r') as fin:
            bg = fin.read()
            bg = ast.literal_eval(bg)
            print >> sys.stderr, dict(bg)
    elif not uniform:
        bg = compute_background(args)
    else:
        bg = None
    return bg

def scan_main(fasta_file, pssm, alphabet, bg, args):

    final = pd.DataFrame()
    count = 0

    if isinstance(fasta_file, SeqRecord):
        final = scan_all(fasta_file, pssm, alphabet, args.minscore)
        final['Sequence_ID'] = 'testseq'
        final['Description'] = ''
        count += 1
    else:
        print >> sys.stderr, "Scanning sequences "

        results = []
        seqiter = parse_sequences(fastas)

        if args.debug:
            for seqrecord in seq_iter:
                hits = scan_all(seqrecord, pssm, args)
                hits['Sequence_ID'] = seqrecord.id
                hits['Description'] = seqrecord.description
                results.append(hits)
                count += 1
        else:
            p = multiprocessing.Pool(args.cores)
            for i, batch in enumerate(batch_iterator(seq_iter, 500)):
                batch_results = p.map(_scan_all_star, izip(batch,
                                                           repeat(pssm),
                                                           repeat(args)
                                                          )
                                     )

                # Process each result
                for j, hits in enumerate(batch_results):
                    if hits is None:
                        continue
                    hits['Sequence_ID'] = batch[j].id
                    hits['Description'] = batch[j].description
                    count += 1
                    results.append(hits)
            p.close()

        if len(results) != 0:
            final = pd.concat(results)

    print >> sys.stderr, "Processed %d sequences" % count
    cols = final.columns.tolist()
    cols = cols[-2:] + cols[:-2]
    return final[cols]

def combine(seq_results, struct_results):
    """ If scoring sequence and structure together, add up the log odds at
    every position
    """

    # Keys to match by : Sequence_ID, Start, End
    # Motif_ID may not match
    result = pd.merge(seq_results, struct_results,
                      on=['Sequence_ID', 'Start', 'End'])
    result.rename(columns={'Description_x': 'Description.Seq',
                           'Description_y': 'Description.Struct',
                           'Sequence_x': 'Sequence.Seq',
                           'Sequence_y': 'Sequence.Struct',
                           'Motif_ID_x': 'Motif_ID.Seq',
                           'Motif_ID_y': 'Motif_ID.Struct',
                           'LogOdds_x': 'LogOdds.Seq',
                           'LogOdds_y': 'LogOdds.Struct'}, inplace=True)
    result['LogOdds.SeqStruct'] = result['LogOdds.Seq'] + \
                                    result['LogOdds.Struct']
    return result

def main():
    tic = time.time()
    args = getoptions()
    bg = None

    if args.testseq:
        args.testseq = args.testseq.split(',')

    ## Sequence
    if args.seq_type in ['RNA', 'RNASS']:
        if args.testseq:
            seq_file = SeqRecord(Seq(args.testseq[0]))
        else:
            seq_file = args.fastafiles[0]

        if args.bg_seq and not args.testseq:
            bg = load_background(args.bg_seq,
                                 IUPAC.IUPACUnambiguousRNA(),
                                 args.fastafiles[0],
                                 args.uniform_background)

        if not args.bgonly:
            pssm = load_motif(args.pfm_seq,
                              args.pseudocount,
                              IUPAC.IUPACUnambiguousRNA(),
                              bg)
            seq_results = scan_main(seq_file,
                                    pssm,
                                    IUPAC.IUPACUnambiguousRNA(),
                                    bg, args)

    ## Structure
    if args.seq_type in ['SS', 'RNASS']:
        if args.testseq:
            struct_file = SeqRecord(Seq(args.testseq[1]))
        elif args.seq_type == 'SS':
            struct_file = args.fastafiles[0]
        else:
            struct_file = args.fastafiles[1]

        if args.bg_struct and not args.testseq:
            bg = load_background(args.bg_struct,
                                 ContextualSecondaryStructure(),
                                 struct_file,
                                 args.uniform_background)

        if not args.bgonly:
            pssm = load_motif(args.pfm_struct,
                              args.pseudocount,
                              ContextualSecondaryStructure(),
                              bg)
            struct_results = scan_main(struct_file,
                                       pssm,
                                       ContextualSecondaryStructure(),
                                       bg, args)

    if args.seq_type == 'RNASS':
        combined_results = combine(seq_results, struct_results)
        combined_results.to_csv(sys.stdout, sep="\t", index=False)
    elif args.seq_type == 'RNA':
        seq_results.to_csv(sys.stdout, sep="\t", index=False)
    else:
        struct_results.to_csv(sys.stdout, sep="\t", index=False)

    toc = time.time()

    runtime = float(toc - tic)
    if runtime > 60:
        print >> sys.stderr, "Done in %0.4f minutes!" % (runtime / 60)
    else:
        print >> sys.stderr, "Done in %0.4f seconds!" % (runtime)

if __name__ == '__main__':
    main()
<|endoftext|>