<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
feedjack
Gustavo Picón
update_feeds.py
"""

import os
import time
import optparse
import datetime
import socket
import traceback
import sys

import feedparser

from feedjack import models, fjcache

VERSION = '0.9.8a1'
URL = 'http://www.feedjack.org/'
USER_AGENT = 'Feedjack %s - %s' % (VERSION, URL)

def encode(tstr):
    """ Encodes a unicode string in utf-8
    """
    if not tstr:
        return ''
    return tstr.encode('utf-8', "xmlcharrefreplace")

def mtime(ttime):
    """ datetime auxiliar function.
    """
    return datetime.datetime.fromtimestamp(time.mktime(ttime))

def get_tags(entry, tagdict):
    """ Returns a list of tag objects from an entry.
    """
    fcat = []
    if entry.has_key('categories'):
        for tcat in entry.categories:
            qcat = encode(tcat[1]).strip()
            if ',' in qcat or '/' in qcat:
                qcat = qcat.replace(',', '/').split('/')
            else:
                qcat = [qcat]
            for zcat in qcat:
                tagname = zcat.lower()
                while '  ' in tagname:
                    tagname.replace('  ', ' ')
                if not tagname or tagname == ' ':
                    continue
                if tagname not in tagdict:
                    cobj = models.Tag(name=tagname)
                    cobj.save()
                    tagdict[tagname] = cobj
                fcat.append(tagdict[tagname])
    return fcat

def get_entry_data(entry, feed, tagdict):
    """ Retrieves data from a post and returns it in a tuple.
    """
    try:
        link = encode(entry.link)
    except AttributeError:
        link = feed.link
    try:
        title = encode(entry.title)
    except AttributeError:
        title = link
    guid = encode(entry.get('id', title))

    if entry.has_key('author_detail'):
        author = encode(entry.author_detail.get('name', ''))
        author_email = encode(entry.author_detail.get('email', ''))
    else:
        author, author_email = '', ''

    if not author:
        author = encode(entry.get('author', entry.get('creator', '')))
    if not author_email:
        author_email = 'nospam@nospam.com'

    try:
        content = encode(entry.content[0].value)
    except:
        content = encode(entry.get('summary', entry.get('description', '')))

    if entry.has_key('modified_parsed'):
        date_modified = mtime(entry.modified_parsed)
    else:
        date_modified = None

    fcat = get_tags(entry, tagdict)
    comments = encode(entry.get('comments', ''))

    return (link, title, guid, author, author_email, content, date_modified, \
      fcat, comments)

def process_entry(entry, fpf, feed, postdict, tagdict, options):
    """ Process a post in a feed and saves it in the DB if necessary.
    """

    (link, title, guid, author, author_email, content, date_modified, fcat, \
      comments) = get_entry_data(entry, feed, tagdict)

    if options.verbose:
        print 'entry:'
        print '  title:', title
        print '  link:', link
        print '  guid:', guid
        print '  author:', author
        print '  author_email:', author_email
        print '  tags:', [tcat.name for tcat in fcat]

    if guid in postdict:
        tobj = postdict[guid]
        if options.verbose:
            print '  - Existing previous post object, updating..'
        postdict[guid] = tobj
        if tobj.content != content or \
          (date_modified and tobj.date_modified != date_modified):
            if options.verbose:
                print '  - Post has changed, updating...'
            if not date_modified:
                # damn non-standard feeds
                date_modified = tobj.date_modified
            tobj.title = title
            tobj.link = link
            tobj.content = content
            tobj.guid = guid
            tobj.date_modified = date_modified
            tobj.author = author
            tobj.author_email = author_email
            tobj.comments = comments
            tobj.tags.clear()
            [tobj.tags.add(tcat) for tcat in fcat]
            tobj.save()
        elif options.verbose:
            print '  - Post has not changed, ignoring.'
    else:
        if options.verbose:
            print '  - Creating post object...'
        if not date_modified:
            # if the feed has no date_modified info, we use the feed mtime or
            # the current time
            if fpf.feed.has_key('modified_parsed'):
                date_modified = mtime(fpf.feed.modified_parsed)
            elif fpf.has_key('modified'):
                date_modified = mtime(fpf.modified)
            else:
                date_modified = datetime.datetime.now()
        tobj = models.Post(feed=feed, title=title, link=link,
            content=content, guid=guid, date_modified=date_modified,
            author=author, author_email=author_email,
            comments=comments)
        tobj.save()
        [tobj.tags.add(tcat) for tcat in fcat]

def process_feed(feed, tagdict, options):
    """ Downloads and parses a feed.
    """
    if options.verbose:
        print '#\n# Processing feed:', feed.feed_url, '\n#'
    else:
        print '# Processing feed:', feed.feed_url

    # we check the etag and the modified time to save bandwith and avoid bans
    try:
        fpf = feedparser.parse(feed.feed_url, agent=USER_AGENT,
            etag=feed.etag)
    except:
        print '! ERROR: feed cannot be parsed'
        return 1

    if hasattr(fpf, 'status'):
        if options.verbose:
            print 'fpf.status:', fpf.status
        if fpf.status == 304:
            # this means the feed has not changed
            if options.verbose:
                print 'Feed has not changed since last check, ignoring.'
            return 2

        if fpf.status >= 400:
            # http error, ignore
            print '! HTTP ERROR'
            return 3

    if hasattr(fpf, 'bozo') and fpf.bozo and options.verbose:
        print '!BOZO'

    # the feed has changed (or it is the first time we parse it)
    # saving the etag and last_modified fields
    feed.etag = encode(fpf.get('etag', ''))
    try:
        feed.last_modified = mtime(fpf.modified)
    except:
        pass

    feed.title = encode(fpf.feed.get('title', ''))[0:254]
    feed.tagline = encode(fpf.feed.get('tagline', ''))
    feed.link = encode(fpf.feed.get('link', ''))
    feed.last_checked = datetime.datetime.now()

    if options.verbose:
        print 'feed.title', feed.title
        print 'feed.tagline', feed.tagline
        print 'feed.link', feed.link
        print 'feed.last_checked', feed.last_checked

    guids = []
    for entry in fpf.entries:
        if encode(entry.get('id', '')):
            guids.append(encode(entry.get('id', '')))
        elif entry.title:
            guids.append(encode(entry.title))
        elif entry.link:
            guids.append(encode(entry.link))
    feed.save()
    if guids:
        postdict = dict([(post.guid, post) \
          for post in models.Post.objects.filter(feed=feed.id).filter(guid__in=guids)])
    else:
        postdict = {}

    for entry in fpf.entries:
        try:
            process_entry(entry, fpf, feed, postdict, tagdict, options)
        except:
            (etype, eobj, etb) = sys.exc_info()
            print '! -------------------------'
            print traceback.format_exception(etype, eobj, etb)
            traceback.print_exception(etype, eobj, etb)
            print '! -------------------------'

    feed.save()

    return 0

def update_feeds(tagdict, options):
    """ Updates all active feeds.
    """

    #for feed in models.Feed.objects.filter(is_active=True).iterator():
    for feed in models.Feed.objects.filter(is_active=True):
        try:
            process_feed(feed, tagdict, options)
        except:
            (etype, eobj, etb) = sys.exc_info()
            print '! -------------------------'
            print traceback.format_exception(etype, eobj, etb)
            traceback.print_exception(etype, eobj, etb)
            print '! -------------------------'

def main():
    """ Main function. Nothing to see here. Move along.
    """
    parser = optparse.OptionParser(usage='%prog [options]', version=USER_AGENT)
    parser.add_option('--settings', \
      help='Python path to settings module. If this isn\'t provided, the DJANGO_SETTINGS_MODULE enviroment variable will be used.')
    parser.add_option('-f', '--feed', action='append', type='int', \
      help='A feed id to be updated. This option can be given multiple times to update several feeds at the same time (-f 1 -f 4 -f 7).')
    parser.add_option('-s', '--site', type='int', \
      help='A site id to update.')
    parser.add_option('-v', '--verbose', action='store_true', dest='verbose', \
      default=False, help='Verbose output.')
    parser.add_option('-t', '--timeout', type='int', default=10, \
      help='Wait timeout in seconds when connecting to feeds.')
    options = parser.parse_args()[0]
    tagdict = dict([(tag.name, tag) for tag in models.Tag.objects.all()])
    if options.settings:
        os.environ["DJANGO_SETTINGS_MODULE"] = options.settings

    # settting socket timeout (default= 10 seconds)
    socket.setdefaulttimeout(options.timeout)

    if options.feed:
        for feed in options.feed:
            try:
                process_feed(models.Feed.objects.get(pk=feed), tagdict, options)
            except  models.Feed.DoesNotExist:
                print '! Unknown feed id: ', feed
    elif options.site:
        feeds = [sub.feed \
          for sub in \
          models.Site.objects.get(pk=int(options.site)).subscriber_set.all()]
        for feed in feeds:
            try:
                process_feed(feed, tagdict, options)
            except  models.Feed.DoesNotExist:
                print '! Unknown site id: ', feed
    else:
        update_feeds(tagdict, options)

    # removing the cached data in all sites, this will only work with the
    # memcached, db and file backends
    [fjcache.cache_delsite(site.id) for site in models.Site.objects.all()]

if __name__ == '__main__':
    main()

#~
<commit_msg>Fixed #15<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
feedjack
Gustavo Picón
update_feeds.py
"""

import os
import time
import optparse
import datetime
import socket
import traceback
import sys

import feedparser

from feedjack import models, fjcache

VERSION = '0.9.8a1'
URL = 'http://www.feedjack.org/'
USER_AGENT = 'Feedjack %s - %s' % (VERSION, URL)

def encode(tstr):
    """ Encodes a unicode string in utf-8
    """
    if not tstr:
        return ''
    return tstr.encode('utf-8', "xmlcharrefreplace")

def mtime(ttime):
    """ datetime auxiliar function.
    """
    return datetime.datetime.fromtimestamp(time.mktime(ttime))

def get_tags(entry, tagdict):
    """ Returns a list of tag objects from an entry.
    """
    fcat = []
    if entry.has_key('categories'):
        for tcat in entry.categories:
            qcat = encode(tcat[1]).strip()
            if ',' in qcat or '/' in qcat:
                qcat = qcat.replace(',', '/').split('/')
            else:
                qcat = [qcat]
            for zcat in qcat:
                tagname = zcat.lower()
                while '  ' in tagname:
                    tagname = tagname.replace('  ', ' ')
                if not tagname or tagname == ' ':
                    continue
                if tagname not in tagdict:
                    cobj = models.Tag(name=tagname)
                    cobj.save()
                    tagdict[tagname] = cobj
                fcat.append(tagdict[tagname])
    return fcat

def get_entry_data(entry, feed, tagdict):
    """ Retrieves data from a post and returns it in a tuple.
    """
    try:
        link = encode(entry.link)
    except AttributeError:
        link = feed.link
    try:
        title = encode(entry.title)
    except AttributeError:
        title = link
    guid = encode(entry.get('id', title))

    if entry.has_key('author_detail'):
        author = encode(entry.author_detail.get('name', ''))
        author_email = encode(entry.author_detail.get('email', ''))
    else:
        author, author_email = '', ''

    if not author:
        author = encode(entry.get('author', entry.get('creator', '')))
    if not author_email:
        author_email = 'nospam@nospam.com'

    try:
        content = encode(entry.content[0].value)
    except:
        content = encode(entry.get('summary', entry.get('description', '')))

    if entry.has_key('modified_parsed'):
        date_modified = mtime(entry.modified_parsed)
    else:
        date_modified = None

    fcat = get_tags(entry, tagdict)
    comments = encode(entry.get('comments', ''))

    return (link, title, guid, author, author_email, content, date_modified, \
      fcat, comments)

def process_entry(entry, fpf, feed, postdict, tagdict, options):
    """ Process a post in a feed and saves it in the DB if necessary.
    """

    (link, title, guid, author, author_email, content, date_modified, fcat, \
      comments) = get_entry_data(entry, feed, tagdict)

    if options.verbose:
        print 'entry:'
        print '  title:', title
        print '  link:', link
        print '  guid:', guid
        print '  author:', author
        print '  author_email:', author_email
        print '  tags:', [tcat.name for tcat in fcat]

    if guid in postdict:
        tobj = postdict[guid]
        if options.verbose:
            print '  - Existing previous post object, updating..'
        postdict[guid] = tobj
        if tobj.content != content or \
          (date_modified and tobj.date_modified != date_modified):
            if options.verbose:
                print '  - Post has changed, updating...'
            if not date_modified:
                # damn non-standard feeds
                date_modified = tobj.date_modified
            tobj.title = title
            tobj.link = link
            tobj.content = content
            tobj.guid = guid
            tobj.date_modified = date_modified
            tobj.author = author
            tobj.author_email = author_email
            tobj.comments = comments
            tobj.tags.clear()
            [tobj.tags.add(tcat) for tcat in fcat]
            tobj.save()
        elif options.verbose:
            print '  - Post has not changed, ignoring.'
    else:
        if options.verbose:
            print '  - Creating post object...'
        if not date_modified:
            # if the feed has no date_modified info, we use the feed mtime or
            # the current time
            if fpf.feed.has_key('modified_parsed'):
                date_modified = mtime(fpf.feed.modified_parsed)
            elif fpf.has_key('modified'):
                date_modified = mtime(fpf.modified)
            else:
                date_modified = datetime.datetime.now()
        tobj = models.Post(feed=feed, title=title, link=link,
            content=content, guid=guid, date_modified=date_modified,
            author=author, author_email=author_email,
            comments=comments)
        tobj.save()
        [tobj.tags.add(tcat) for tcat in fcat]

def process_feed(feed, tagdict, options):
    """ Downloads and parses a feed.
    """
    if options.verbose:
        print '#\n# Processing feed:', feed.feed_url, '\n#'
    else:
        print '# Processing feed:', feed.feed_url

    # we check the etag and the modified time to save bandwith and avoid bans
    try:
        fpf = feedparser.parse(feed.feed_url, agent=USER_AGENT,
            etag=feed.etag)
    except:
        print '! ERROR: feed cannot be parsed'
        return 1

    if hasattr(fpf, 'status'):
        if options.verbose:
            print 'fpf.status:', fpf.status
        if fpf.status == 304:
            # this means the feed has not changed
            if options.verbose:
                print 'Feed has not changed since last check, ignoring.'
            return 2

        if fpf.status >= 400:
            # http error, ignore
            print '! HTTP ERROR'
            return 3

    if hasattr(fpf, 'bozo') and fpf.bozo and options.verbose:
        print '!BOZO'

    # the feed has changed (or it is the first time we parse it)
    # saving the etag and last_modified fields
    feed.etag = encode(fpf.get('etag', ''))
    try:
        feed.last_modified = mtime(fpf.modified)
    except:
        pass

    feed.title = encode(fpf.feed.get('title', ''))[0:254]
    feed.tagline = encode(fpf.feed.get('tagline', ''))
    feed.link = encode(fpf.feed.get('link', ''))
    feed.last_checked = datetime.datetime.now()

    if options.verbose:
        print 'feed.title', feed.title
        print 'feed.tagline', feed.tagline
        print 'feed.link', feed.link
        print 'feed.last_checked', feed.last_checked

    guids = []
    for entry in fpf.entries:
        if encode(entry.get('id', '')):
            guids.append(encode(entry.get('id', '')))
        elif entry.title:
            guids.append(encode(entry.title))
        elif entry.link:
            guids.append(encode(entry.link))
    feed.save()
    if guids:
        postdict = dict([(post.guid, post) \
          for post in models.Post.objects.filter(feed=feed.id).filter(guid__in=guids)])
    else:
        postdict = {}

    for entry in fpf.entries:
        try:
            process_entry(entry, fpf, feed, postdict, tagdict, options)
        except:
            (etype, eobj, etb) = sys.exc_info()
            print '! -------------------------'
            print traceback.format_exception(etype, eobj, etb)
            traceback.print_exception(etype, eobj, etb)
            print '! -------------------------'

    feed.save()

    return 0

def update_feeds(tagdict, options):
    """ Updates all active feeds.
    """

    #for feed in models.Feed.objects.filter(is_active=True).iterator():
    for feed in models.Feed.objects.filter(is_active=True):
        try:
            process_feed(feed, tagdict, options)
        except:
            (etype, eobj, etb) = sys.exc_info()
            print '! -------------------------'
            print traceback.format_exception(etype, eobj, etb)
            traceback.print_exception(etype, eobj, etb)
            print '! -------------------------'

def main():
    """ Main function. Nothing to see here. Move along.
    """
    parser = optparse.OptionParser(usage='%prog [options]', version=USER_AGENT)
    parser.add_option('--settings', \
      help='Python path to settings module. If this isn\'t provided, the DJANGO_SETTINGS_MODULE enviroment variable will be used.')
    parser.add_option('-f', '--feed', action='append', type='int', \
      help='A feed id to be updated. This option can be given multiple times to update several feeds at the same time (-f 1 -f 4 -f 7).')
    parser.add_option('-s', '--site', type='int', \
      help='A site id to update.')
    parser.add_option('-v', '--verbose', action='store_true', dest='verbose', \
      default=False, help='Verbose output.')
    parser.add_option('-t', '--timeout', type='int', default=10, \
      help='Wait timeout in seconds when connecting to feeds.')
    options = parser.parse_args()[0]
    tagdict = dict([(tag.name, tag) for tag in models.Tag.objects.all()])
    if options.settings:
        os.environ["DJANGO_SETTINGS_MODULE"] = options.settings

    # settting socket timeout (default= 10 seconds)
    socket.setdefaulttimeout(options.timeout)

    if options.feed:
        for feed in options.feed:
            try:
                process_feed(models.Feed.objects.get(pk=feed), tagdict, options)
            except  models.Feed.DoesNotExist:
                print '! Unknown feed id: ', feed
    elif options.site:
        feeds = [sub.feed \
          for sub in \
          models.Site.objects.get(pk=int(options.site)).subscriber_set.all()]
        for feed in feeds:
            try:
                process_feed(feed, tagdict, options)
            except  models.Feed.DoesNotExist:
                print '! Unknown site id: ', feed
    else:
        update_feeds(tagdict, options)

    # removing the cached data in all sites, this will only work with the
    # memcached, db and file backends
    [fjcache.cache_delsite(site.id) for site in models.Site.objects.all()]

if __name__ == '__main__':
    main()

#~
<|endoftext|>


<commit_before>import re
import logging
from indra.resources import load_resource_json

logger = logging.getLogger(__name__)

identifiers_url = 'https://identifiers.org'

# These are just special cases of name spaces where the mapping from INDRA to
# identifiers.org is not a question of simplecapitalization.
identifiers_mappings = {
    'UP': 'uniprot',
    'UPPRO': 'uniprot.chain',
    'UPISO': 'uniprot.isoform',
    'REFSEQ_PROT': 'refseq',
    'PF': 'pfam',
    'IP': 'interpro',
    'ECCODE': 'ec-code',
    'NONCODE': 'noncodev4.rna',
    'LNCRNADB': 'rnacentral',
    'MIRBASEM': 'mirbase.mature',
    'EGID': 'ncbigene',
    'NCBI': 'ncibgene',
    'HGNC_GROUP': 'hgnc.genefamily',
    'LINCS': 'lincs.smallmolecule',
    'PUBCHEM': 'pubchem.compound',
    'CHEMBL': 'chembl.compound',
    'CTD': 'ctd.chemical',
    'CVCL': 'cellosaurus',
}

# These are namespaces used by INDRA that don't have corresponding
# identifiers.org entries
non_registry = {
    'SDIS', 'SCHEM', 'SFAM', 'SCOMP', 'SIGNOR', 'HMS-LINCS', 'NXPFA',
    'OMIM', 'LSPCI', 'UPLOC', 'BFO', 'CCLE', 'CLO', 'GENBANK',
    'DRUGBANK.SALT', 'SMILES',
}

# These are namespaces that can appear in db_refs but are actually not
# representing grounding.
non_grounding = {
    'TEXT', 'TEXT_NORM'
}

# These are reverse mappings from identifiers.org namespaces to INDRA
# namespaces
identifiers_reverse = {
    v: k for k, v in identifiers_mappings.items()
}
# We have to patch this one because it is ambiguous
identifiers_reverse['ncbigene'] = 'EGID'

# These are only the URLs that are strictly prefixes and not more complicated
# patterns. This is because some downstream code uses these as prefixes
# rather than arbitrary patterns.
url_prefixes = {
    # Biology namespaces
    'NXPFA': 'https://www.nextprot.org/term/FA-',
    'SIGNOR': 'https://signor.uniroma2.it/relation_result.php?id=',
    'LSPCI': 'https://labsyspharm.github.io/lspci/',
    # WM namespaces
    'UN': 'https://github.com/clulab/eidos/wiki/JSON-LD#Grounding/',
    'WDI': 'https://github.com/clulab/eidos/wiki/JSON-LD#Grounding/',
    'FAO': 'https://github.com/clulab/eidos/wiki/JSON-LD#Grounding/',
    'HUME': ('https://github.com/BBN-E/Hume/blob/master/resource/ontologies'
             '/hume_ontology/'),
    'CWMS': 'http://trips.ihmc.us/',
    'SOFIA': 'http://cs.cmu.edu/sofia/',
}

def get_ns_from_identifiers(identifiers_ns):
    """"Return a namespace compatible with INDRA from an identifiers namespace.

    For example, this function can be used to map 'uniprot' to 'UP'.

    Parameters
    ----------
    identifiers_ns : str
        An identifiers.org standard namespace.

    Returns
    -------
    str or None
        The namespace compatible with INDRA's internal representation or
        None if the given namespace isn't an identifiers.org standard.
    """
    reg_entry = identifiers_registry.get(identifiers_ns.lower())
    if not reg_entry:
        return None
    mapping = identifiers_reverse.get(identifiers_ns.lower())
    if mapping:
        return mapping
    else:
        return identifiers_ns.upper()

def get_ns_id_from_identifiers(identifiers_ns, identifiers_id):
    """Return a namespace/ID pair compatible with INDRA from identifiers.

    Parameters
    ----------
    identifiers_ns : str
        An identifiers.org standard namespace.
    identifiers_id : str
        An identifiers.org standard ID in the given namespace.

    Returns
    -------
    (str, str)
        A namespace and ID that are valid in INDRA db_refs.
    """
    reg_entry = identifiers_registry.get(identifiers_ns.lower())
    db_ns = get_ns_from_identifiers(identifiers_ns)
    if db_ns is None:
        return None, None
    db_id = identifiers_id
    if reg_entry['namespace_embedded']:
        if not identifiers_id.startswith(identifiers_ns.upper()):
            db_id = '%s:%s' % (identifiers_ns.upper(), identifiers_id)
    return db_ns, db_id

def get_identifiers_ns(db_name):
    """Map an INDRA namespace to an identifiers.org namespace when possible.

    Example: this can be used to map 'UP' to 'uniprot'.

    Parameters
    ----------
    db_name : str
        An INDRA namespace to map to identifiers.org

    Returns
    -------
    str or None
        An identifiers.org namespace or None if not available.
    """
    mapped_db_name = identifiers_mappings.get(db_name, db_name.lower())
    if mapped_db_name not in identifiers_registry:
        return None
    return mapped_db_name

def get_url_prefix(db_name):
    """Return the URL prefix for a given namespace."""
    identifiers_ns = get_identifiers_ns(db_name)

    if identifiers_ns:
        identifiers_entry = identifiers_registry.get(identifiers_ns)
        if not identifiers_entry['namespace_embedded']:
            return '%s/%s:' % (identifiers_url, identifiers_ns.lower())
        else:
            return '%s/' % identifiers_url
    else:
        if db_name in url_prefixes:
            return url_prefixes[db_name]
    return None

def get_identifiers_url(db_name, db_id):
    """Return an identifiers.org URL for a given database name and ID.

    Parameters
    ----------
    db_name : str
        An internal database name: HGNC, UP, CHEBI, etc.
    db_id : str
        An identifier in the given database.

    Returns
    -------
    url : str
        An identifiers.org URL corresponding to the given database name and ID.
    """
    # This is the case where we have a prefix that we can simply attach the
    # db_id to to get the desired URL.
    if db_name == 'CHEMBL':
        db_id = ensure_chembl_prefix(db_id)
    elif db_name == 'CHEBI':
        db_id = ensure_chebi_prefix(db_id)

    prefix = get_url_prefix(db_name)
    if prefix:
        return '%s%s' % (prefix, db_id)

    # Otherwise, we have to handle some special cases
    bel_scai_url = 'https://arty.scai.fraunhofer.de/artifactory/bel/namespace/'
    if db_name == 'LINCS':
        if db_id.startswith('LSM-'):  # Lincs Small Molecule ID
            url = identifiers_url + '/lincs.smallmolecule:%s' % db_id
        elif db_id.startswith('LCL-'):  # Lincs Cell Line ID
            url = identifiers_url + '/lincs.cell:%s' % db_id
        else:  # Assume LINCS Protein
            url = identifiers_url + '/lincs.protein:%s' % db_id
    elif db_name == 'CHEMBL':
        if not db_id.startswith('CHEMBL'):
            db_id = 'CHEMBL%s' % db_id
        url = identifiers_url + '/chembl.compound:%s' % db_id
    elif db_name == 'HMS-LINCS':
        url = 'http://lincs.hms.harvard.edu/db/sm/%s-101' % db_id
    # Special cases with no identifiers entry
    elif db_name == 'SCHEM':
        url = bel_scai_url + 'selventa-legacy-chemicals/' + \
            'selventa-legacy-chemicals-20150601.belns'
    elif db_name == 'SCOMP':
        url = bel_scai_url + 'selventa-named-complexes/' + \
            'selventa-named-complexes-20150601.belns'
    elif db_name == 'SFAM':
        url = bel_scai_url + 'selventa-protein-families/' + \
            'selventa-protein-families-20150601.belns'
    elif db_name == 'TEXT' or db_name == 'TEXT_NORM':
        return None
    else:
        logger.warning('Unhandled name space %s' % db_name)
        url = None
    return url

def parse_identifiers_url(url):
    """Retrieve database name and ID given the URL.

    Parameters
    ----------
    url : str
        An identifiers.org URL to parse.

    Returns
    -------
    db_name : str
        An internal database name: HGNC, UP, CHEBI, etc. corresponding to the
        given URL.
    db_id : str
        An identifier in the database.
    """
    # Try matching by string pattern
    db_ns, db_id = None, None
    url_pattern = \
        r'(?:https?)://identifiers.org/([A-Za-z0-9.-]+)(/|:)([A-Za-z0-9:_.-]+)'
    match = re.match(url_pattern, url)
    if match is not None:
        g = match.groups()
        if len(g) == 3:
            pattern_ns, pattern_id = g[0], g[2]
            db_ns, db_id = get_ns_id_from_identifiers(pattern_ns, pattern_id)
            if db_ns == 'HGNC':
                if db_id.startswith('HGNC:'):
                    db_id = db_id[5:]
            # If we got UP and UPPRO, return UPPRO
            if db_ns == 'UP' and '#PRO_' in url:
                db_ns = 'UPPRO'
                db_id = url[url.find('PRO_'):]
            if db_ns and db_id:
                return db_ns, db_id
    for ns, prefix in url_prefixes.items():
        if url.startswith(prefix):
            return ns, url[len(prefix):]

    # Handle other special cases
    for part in ['/lincs.smallmolecule', '/lincs.cell', '/lincs.protein']:
        if part in url:
            return 'LINCS', url[(url.find(part) + len(part) + 1):]
    if '/chembl.compound' in url:
        return 'CHEMBL', url[
            (url.find('/chembl.compound') + len('/chembl.compound:')):]
    if 'lincs.hms.harvard.edu' in url:
        return 'HMS-LINCS', url[len('http://lincs.hms.harvard.edu/db/sm/'):-4]
    if 'selventa-legacy-chemicals/' in url:
        return 'SCHEM', None
    if 'selventa-named-complexes/' in url:
        return 'SCOMP', None
    if 'selventa-protein-families/' in url:
        return 'SFAM', None
    else:
        logger.warning('Could not parse URL %s' % url)
    return None, None

def namespace_embedded(db_ns: str) -> bool:
    """Return true if this namespace requires IDs to have namespace embedded.

    This function first maps the given namespace to an identifiers.org
    namespace and then checks the registry to see if namespaces need
    to be embedded in IDs. If yes, it returns True. If not, or the ID can't
    be mapped to identifiers.org, it returns False

    Parameters
    ----------
    db_ns :
        The namespace to check.

    Returns
    -------
    :
        True if the namespace is known to be embedded in IDs of this
        namespace. False if unknown or known not to be embedded.
    """
    identifiers_ns = get_identifiers_ns(db_ns)
    if identifiers_ns:
        identifiers_entry = identifiers_registry.get(identifiers_ns)
        if identifiers_entry['namespace_embedded']:
            return True
    return False

def ensure_prefix_if_needed(db_ns: str, db_id: str) -> str:
    """Return an ID ensuring a namespace prefix if known to be needed.

    Parameters
    ----------
    db_ns :
        The namespace associated with the identifier.
    db_id :
        The original identifier.

    Returns
    -------
    :
        The identifier with namespace embedded if needed.
    """
    if namespace_embedded(db_ns):
        return ensure_prefix(db_ns, db_id)
    return db_id

def ensure_prefix(db_ns, db_id, with_colon=True):
    """Return a valid ID that has the given namespace embedded.

    This is useful for namespaces such as CHEBI, GO or BTO that require
    the namespace to be part of the ID. Note that this function always
    ensures that the given db_ns is embedded in the ID and can handle the
    case whene it's already present.

    Parameters
    ----------
    db_ns : str
        A namespace.
    db_id : str
        An ID within that namespace which should have the namespace
        as a prefix in it.
    with_colon: Optional[bool]
        If True, the namespace prefix is followed by a colon in the ID (e.g.,
        CHEBI:12345). Otherwise, no colon is added (e.g., CHEMBL1234).
        Default: True
    """
    if db_id is None:
        return None
    colon = ':' if with_colon else ''
    if not db_id.startswith(f'{db_ns}{colon}'):
        return f'{db_ns}{colon}{db_id}'
    return db_id

def ensure_chebi_prefix(chebi_id):
    """Return a valid CHEBI ID that has the appropriate CHEBI: prefix."""
    return ensure_prefix('CHEBI', chebi_id)

def ensure_chembl_prefix(chembl_id):
    """Return a valid CHEMBL ID that has the appropriate CHEMBL prefix."""
    return ensure_prefix('CHEMBL', chembl_id, with_colon=False)

def _load_identifiers_registry():
    identifiers_registry = load_resource_json('identifiers_patterns.json')
    # Override pattern otherwise patterns like 1.1 can't be used
    identifiers_registry['ec-code']['pattern'] = '^\\d{1,2}(\\.\\d{0,3}){0,3}$'
    identifiers_registry['mondo'] = {
        "pattern": "^\\d+$",
        "namespace_embedded": False,
    }
    for value in identifiers_registry.values():
        value["pattern_compiled"] = re.compile(value["pattern"])
    return identifiers_registry

identifiers_registry = _load_identifiers_registry()
<commit_msg>Remove SIGNOR from non-registry list<commit_after>import re
import logging
from indra.resources import load_resource_json

logger = logging.getLogger(__name__)

identifiers_url = 'https://identifiers.org'

# These are just special cases of name spaces where the mapping from INDRA to
# identifiers.org is not a question of simplecapitalization.
identifiers_mappings = {
    'UP': 'uniprot',
    'UPPRO': 'uniprot.chain',
    'UPISO': 'uniprot.isoform',
    'REFSEQ_PROT': 'refseq',
    'PF': 'pfam',
    'IP': 'interpro',
    'ECCODE': 'ec-code',
    'NONCODE': 'noncodev4.rna',
    'LNCRNADB': 'rnacentral',
    'MIRBASEM': 'mirbase.mature',
    'EGID': 'ncbigene',
    'NCBI': 'ncibgene',
    'HGNC_GROUP': 'hgnc.genefamily',
    'LINCS': 'lincs.smallmolecule',
    'PUBCHEM': 'pubchem.compound',
    'CHEMBL': 'chembl.compound',
    'CTD': 'ctd.chemical',
    'CVCL': 'cellosaurus',
}

# These are namespaces used by INDRA that don't have corresponding
# identifiers.org entries
non_registry = {
    'SDIS', 'SCHEM', 'SFAM', 'SCOMP', 'HMS-LINCS', 'NXPFA',
    'OMIM', 'LSPCI', 'UPLOC', 'BFO', 'CCLE', 'CLO', 'GENBANK',
    'DRUGBANK.SALT', 'SMILES',
}

# These are namespaces that can appear in db_refs but are actually not
# representing grounding.
non_grounding = {
    'TEXT', 'TEXT_NORM'
}

# These are reverse mappings from identifiers.org namespaces to INDRA
# namespaces
identifiers_reverse = {
    v: k for k, v in identifiers_mappings.items()
}
# We have to patch this one because it is ambiguous
identifiers_reverse['ncbigene'] = 'EGID'

# These are only the URLs that are strictly prefixes and not more complicated
# patterns. This is because some downstream code uses these as prefixes
# rather than arbitrary patterns.
url_prefixes = {
    # Biology namespaces
    'NXPFA': 'https://www.nextprot.org/term/FA-',
    'SIGNOR': 'https://signor.uniroma2.it/relation_result.php?id=',
    'LSPCI': 'https://labsyspharm.github.io/lspci/',
    # WM namespaces
    'UN': 'https://github.com/clulab/eidos/wiki/JSON-LD#Grounding/',
    'WDI': 'https://github.com/clulab/eidos/wiki/JSON-LD#Grounding/',
    'FAO': 'https://github.com/clulab/eidos/wiki/JSON-LD#Grounding/',
    'HUME': ('https://github.com/BBN-E/Hume/blob/master/resource/ontologies'
             '/hume_ontology/'),
    'CWMS': 'http://trips.ihmc.us/',
    'SOFIA': 'http://cs.cmu.edu/sofia/',
}

def get_ns_from_identifiers(identifiers_ns):
    """"Return a namespace compatible with INDRA from an identifiers namespace.

    For example, this function can be used to map 'uniprot' to 'UP'.

    Parameters
    ----------
    identifiers_ns : str
        An identifiers.org standard namespace.

    Returns
    -------
    str or None
        The namespace compatible with INDRA's internal representation or
        None if the given namespace isn't an identifiers.org standard.
    """
    reg_entry = identifiers_registry.get(identifiers_ns.lower())
    if not reg_entry:
        return None
    mapping = identifiers_reverse.get(identifiers_ns.lower())
    if mapping:
        return mapping
    else:
        return identifiers_ns.upper()

def get_ns_id_from_identifiers(identifiers_ns, identifiers_id):
    """Return a namespace/ID pair compatible with INDRA from identifiers.

    Parameters
    ----------
    identifiers_ns : str
        An identifiers.org standard namespace.
    identifiers_id : str
        An identifiers.org standard ID in the given namespace.

    Returns
    -------
    (str, str)
        A namespace and ID that are valid in INDRA db_refs.
    """
    reg_entry = identifiers_registry.get(identifiers_ns.lower())
    db_ns = get_ns_from_identifiers(identifiers_ns)
    if db_ns is None:
        return None, None
    db_id = identifiers_id
    if reg_entry['namespace_embedded']:
        if not identifiers_id.startswith(identifiers_ns.upper()):
            db_id = '%s:%s' % (identifiers_ns.upper(), identifiers_id)
    return db_ns, db_id

def get_identifiers_ns(db_name):
    """Map an INDRA namespace to an identifiers.org namespace when possible.

    Example: this can be used to map 'UP' to 'uniprot'.

    Parameters
    ----------
    db_name : str
        An INDRA namespace to map to identifiers.org

    Returns
    -------
    str or None
        An identifiers.org namespace or None if not available.
    """
    mapped_db_name = identifiers_mappings.get(db_name, db_name.lower())
    if mapped_db_name not in identifiers_registry:
        return None
    return mapped_db_name

def get_url_prefix(db_name):
    """Return the URL prefix for a given namespace."""
    identifiers_ns = get_identifiers_ns(db_name)

    if identifiers_ns:
        identifiers_entry = identifiers_registry.get(identifiers_ns)
        if not identifiers_entry['namespace_embedded']:
            return '%s/%s:' % (identifiers_url, identifiers_ns.lower())
        else:
            return '%s/' % identifiers_url
    else:
        if db_name in url_prefixes:
            return url_prefixes[db_name]
    return None

def get_identifiers_url(db_name, db_id):
    """Return an identifiers.org URL for a given database name and ID.

    Parameters
    ----------
    db_name : str
        An internal database name: HGNC, UP, CHEBI, etc.
    db_id : str
        An identifier in the given database.

    Returns
    -------
    url : str
        An identifiers.org URL corresponding to the given database name and ID.
    """
    # This is the case where we have a prefix that we can simply attach the
    # db_id to to get the desired URL.
    if db_name == 'CHEMBL':
        db_id = ensure_chembl_prefix(db_id)
    elif db_name == 'CHEBI':
        db_id = ensure_chebi_prefix(db_id)

    prefix = get_url_prefix(db_name)
    if prefix:
        return '%s%s' % (prefix, db_id)

    # Otherwise, we have to handle some special cases
    bel_scai_url = 'https://arty.scai.fraunhofer.de/artifactory/bel/namespace/'
    if db_name == 'LINCS':
        if db_id.startswith('LSM-'):  # Lincs Small Molecule ID
            url = identifiers_url + '/lincs.smallmolecule:%s' % db_id
        elif db_id.startswith('LCL-'):  # Lincs Cell Line ID
            url = identifiers_url + '/lincs.cell:%s' % db_id
        else:  # Assume LINCS Protein
            url = identifiers_url + '/lincs.protein:%s' % db_id
    elif db_name == 'CHEMBL':
        if not db_id.startswith('CHEMBL'):
            db_id = 'CHEMBL%s' % db_id
        url = identifiers_url + '/chembl.compound:%s' % db_id
    elif db_name == 'HMS-LINCS':
        url = 'http://lincs.hms.harvard.edu/db/sm/%s-101' % db_id
    # Special cases with no identifiers entry
    elif db_name == 'SCHEM':
        url = bel_scai_url + 'selventa-legacy-chemicals/' + \
            'selventa-legacy-chemicals-20150601.belns'
    elif db_name == 'SCOMP':
        url = bel_scai_url + 'selventa-named-complexes/' + \
            'selventa-named-complexes-20150601.belns'
    elif db_name == 'SFAM':
        url = bel_scai_url + 'selventa-protein-families/' + \
            'selventa-protein-families-20150601.belns'
    elif db_name == 'TEXT' or db_name == 'TEXT_NORM':
        return None
    else:
        logger.warning('Unhandled name space %s' % db_name)
        url = None
    return url

def parse_identifiers_url(url):
    """Retrieve database name and ID given the URL.

    Parameters
    ----------
    url : str
        An identifiers.org URL to parse.

    Returns
    -------
    db_name : str
        An internal database name: HGNC, UP, CHEBI, etc. corresponding to the
        given URL.
    db_id : str
        An identifier in the database.
    """
    # Try matching by string pattern
    db_ns, db_id = None, None
    url_pattern = \
        r'(?:https?)://identifiers.org/([A-Za-z0-9.-]+)(/|:)([A-Za-z0-9:_.-]+)'
    match = re.match(url_pattern, url)
    if match is not None:
        g = match.groups()
        if len(g) == 3:
            pattern_ns, pattern_id = g[0], g[2]
            db_ns, db_id = get_ns_id_from_identifiers(pattern_ns, pattern_id)
            if db_ns == 'HGNC':
                if db_id.startswith('HGNC:'):
                    db_id = db_id[5:]
            # If we got UP and UPPRO, return UPPRO
            if db_ns == 'UP' and '#PRO_' in url:
                db_ns = 'UPPRO'
                db_id = url[url.find('PRO_'):]
            if db_ns and db_id:
                return db_ns, db_id
    for ns, prefix in url_prefixes.items():
        if url.startswith(prefix):
            return ns, url[len(prefix):]

    # Handle other special cases
    for part in ['/lincs.smallmolecule', '/lincs.cell', '/lincs.protein']:
        if part in url:
            return 'LINCS', url[(url.find(part) + len(part) + 1):]
    if '/chembl.compound' in url:
        return 'CHEMBL', url[
            (url.find('/chembl.compound') + len('/chembl.compound:')):]
    if 'lincs.hms.harvard.edu' in url:
        return 'HMS-LINCS', url[len('http://lincs.hms.harvard.edu/db/sm/'):-4]
    if 'selventa-legacy-chemicals/' in url:
        return 'SCHEM', None
    if 'selventa-named-complexes/' in url:
        return 'SCOMP', None
    if 'selventa-protein-families/' in url:
        return 'SFAM', None
    else:
        logger.warning('Could not parse URL %s' % url)
    return None, None

def namespace_embedded(db_ns: str) -> bool:
    """Return true if this namespace requires IDs to have namespace embedded.

    This function first maps the given namespace to an identifiers.org
    namespace and then checks the registry to see if namespaces need
    to be embedded in IDs. If yes, it returns True. If not, or the ID can't
    be mapped to identifiers.org, it returns False

    Parameters
    ----------
    db_ns :
        The namespace to check.

    Returns
    -------
    :
        True if the namespace is known to be embedded in IDs of this
        namespace. False if unknown or known not to be embedded.
    """
    identifiers_ns = get_identifiers_ns(db_ns)
    if identifiers_ns:
        identifiers_entry = identifiers_registry.get(identifiers_ns)
        if identifiers_entry['namespace_embedded']:
            return True
    return False

def ensure_prefix_if_needed(db_ns: str, db_id: str) -> str:
    """Return an ID ensuring a namespace prefix if known to be needed.

    Parameters
    ----------
    db_ns :
        The namespace associated with the identifier.
    db_id :
        The original identifier.

    Returns
    -------
    :
        The identifier with namespace embedded if needed.
    """
    if namespace_embedded(db_ns):
        return ensure_prefix(db_ns, db_id)
    return db_id

def ensure_prefix(db_ns, db_id, with_colon=True):
    """Return a valid ID that has the given namespace embedded.

    This is useful for namespaces such as CHEBI, GO or BTO that require
    the namespace to be part of the ID. Note that this function always
    ensures that the given db_ns is embedded in the ID and can handle the
    case whene it's already present.

    Parameters
    ----------
    db_ns : str
        A namespace.
    db_id : str
        An ID within that namespace which should have the namespace
        as a prefix in it.
    with_colon: Optional[bool]
        If True, the namespace prefix is followed by a colon in the ID (e.g.,
        CHEBI:12345). Otherwise, no colon is added (e.g., CHEMBL1234).
        Default: True
    """
    if db_id is None:
        return None
    colon = ':' if with_colon else ''
    if not db_id.startswith(f'{db_ns}{colon}'):
        return f'{db_ns}{colon}{db_id}'
    return db_id

def ensure_chebi_prefix(chebi_id):
    """Return a valid CHEBI ID that has the appropriate CHEBI: prefix."""
    return ensure_prefix('CHEBI', chebi_id)

def ensure_chembl_prefix(chembl_id):
    """Return a valid CHEMBL ID that has the appropriate CHEMBL prefix."""
    return ensure_prefix('CHEMBL', chembl_id, with_colon=False)

def _load_identifiers_registry():
    identifiers_registry = load_resource_json('identifiers_patterns.json')
    # Override pattern otherwise patterns like 1.1 can't be used
    identifiers_registry['ec-code']['pattern'] = '^\\d{1,2}(\\.\\d{0,3}){0,3}$'
    identifiers_registry['mondo'] = {
        "pattern": "^\\d+$",
        "namespace_embedded": False,
    }
    for value in identifiers_registry.values():
        value["pattern_compiled"] = re.compile(value["pattern"])
    return identifiers_registry

identifiers_registry = _load_identifiers_registry()
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# (C) 2005 British Broadcasting Corporation and Kamaelia Contributors(1)
#     All Rights Reserved.
#
# You may only modify and redistribute this under the terms of any of the
# following licenses(2): Mozilla Public License, V1.1, GNU General
# Public License, V2.0, GNU Lesser General Public License, V2.1
#
# (1) Kamaelia Contributors are listed in the AUTHORS file and at
#     http://kamaelia.sourceforge.net/AUTHORS - please extend this file,
#     not this notice.
# (2) Reproduced in the COPYING file, and at:
#     http://kamaelia.sourceforge.net/COPYING
# Under section 3.5 of the MPL, we are using this text since we deem the MPL
# notice inappropriate for this file. As per MPL/GPL/LGPL removal of this
# notice is prohibited.
#
# Please contact us via: kamaelia-list-owner@lists.sourceforge.net
# to discuss alternative licensing.
# -------------------------------------------------------------------------
#

import pygame
import Axon
from Axon.Ipc import producerFinished
from Kamaelia.UI.PygameDisplay import PygameDisplay

class KeyEvent(Axon.Component.component):
   """
   """

   Inboxes = { "inbox"    : "Receive events from PygameDisplay",
               "control"  : "",
               "callback" : "Receive callbacks from PygameDisplay"
             }
   Outboxes = { "outbox" : "button click events emitted here",
                "allkeys" : "Outbox that receives *every* keystroke if enabled",
                "signal" : "",
                "display_signal" : "Outbox used for communicating to the display surface" }

   def __init__(self, caption=None,
                position=None, margin=8,
                bgcolour = (224,224,224),
                fgcolour = (0,0,0),
                msg=None,
                allkeys = False,
                key_events = None,
                transparent = False,
                outboxes = {}):
      """Creates and activates a button widget
         caption  = text label for the button / None for default label
         position = (x,y) pair / None
         margin   = margin size (around the text) in pixels
         bgcolour = background colour
         fgcolour = text colour
         msg      = message to be sent when this button is clicked / None for default
         """
      self.Outboxes = self.__class__.Outboxes
      self.Outboxes.update(outboxes)
      super(KeyEvent,self).__init__()

      self.allkeys = allkeys
      self.backgroundColour = bgcolour
      self.foregroundColour = fgcolour
      self.margin = margin
      self.key_events = key_events
      if self.key_events is None: self.key_events = {}

      if caption is None:
         caption = "Button "+str(self.id)

      pygame.font.init()
#      self.buildCaption(caption)

      if msg is None:
         msg = ("CLICK", self.id)
      self.eventMsg = msg
      if transparent:
         transparency = bgcolour
      else:
         transparency = None
      self.disprequest = { "DISPLAYREQUEST" : True,
                           "callback" : (self,"callback"),
                           "events" : (self, "inbox"),
                           "size": (0,0),
                           "transparency" : transparency }

      if not position is None:
        self.disprequest["position"] = position

   def buildCaption(self, text):
      """Render the text to go on the button label.
      (This doesn't actually place the text onto the 'surface')
      """
      font = pygame.font.Font(None, 14)
      self.image = font.render(text,True, self.foregroundColour, )

      (w,h) = self.image.get_size()
      self.size = (w + 2*self.margin, h + 2*self.margin)
      self.imagePosition = (self.margin, self.margin)

   def waitBox(self,boxname):
      waiting = True
      while waiting:
        if self.dataReady(boxname): return
        else: yield 1

   def main(self):
      displayservice = PygameDisplay.getDisplayService()
      self.link((self,"display_signal"), displayservice)

      self.send( self.disprequest,
                  "display_signal")

      for _ in self.waitBox("callback"): yield 1
      self.display = self.recv("callback")
      self.blitToSurface()

      self.send({ "ADDLISTENEVENT" : pygame.MOUSEBUTTONDOWN,
                  "surface" : self.display},
                  "display_signal")
      if (self.key_events is not None) or self.allkeys:
         message = { "ADDLISTENEVENT" : pygame.KEYDOWN,
                     "surface" : self.display,
                     "TRACE" : "ME"}
         self.send(message, "display_signal")
      done = False
      while not done:
         if self.dataReady("control"):
            cmsg = self.recv("control")
            if isinstance(cmsg, producerFinished) or isinstance(cmsg, shutdownMicroprocess):
               done = True

         while self.dataReady("inbox"):
            for event in self.recv("inbox"):
                if event.type == pygame.MOUSEBUTTONDOWN:
                   bounds = self.display.get_rect()
                   if bounds.collidepoint(*event.pos):
                      self.send( self.eventMsg, "outbox" )
                if event.type == pygame.KEYDOWN:
                   if event.key in self.key_events:
                      self.send( self.key_events[event.key][0] , self.key_events[event.key][1] )
                   if self.allkeys:
                      self.send(event.key, "allkeys")
         yield 1

   def blitToSurface(self):
       try:
           self.display.fill( self.backgroundColour )
           self.display.blit( self.image, self.imagePosition )
       except:
           pass

if __name__ == "__main__":
   from Kamaelia.Util.ConsoleEcho import consoleEchoer

   button1 = Button().activate()
   button2 = Button(caption="Reverse colours",fgcolour=(255,255,255),bgcolour=(0,0,0)).activate()
   button3 = Button(caption="Mary...",msg="Mary had a little lamb").activate()

   ce = consoleEchoer().activate()
   button1.link( (button1,"outbox"), (ce,"inbox") )
   button2.link( (button2,"outbox"), (ce,"inbox") )
   button3.link( (button3,"outbox"), (ce,"inbox") )

   Axon.Scheduler.scheduler.run.runThreads()  <commit_msg>Better handling of all key presses. Allows for someone to deduce if a key should be captalised (amongst other things) or not.<commit_after>#!/usr/bin/python
#
# (C) 2005 British Broadcasting Corporation and Kamaelia Contributors(1)
#     All Rights Reserved.
#
# You may only modify and redistribute this under the terms of any of the
# following licenses(2): Mozilla Public License, V1.1, GNU General
# Public License, V2.0, GNU Lesser General Public License, V2.1
#
# (1) Kamaelia Contributors are listed in the AUTHORS file and at
#     http://kamaelia.sourceforge.net/AUTHORS - please extend this file,
#     not this notice.
# (2) Reproduced in the COPYING file, and at:
#     http://kamaelia.sourceforge.net/COPYING
# Under section 3.5 of the MPL, we are using this text since we deem the MPL
# notice inappropriate for this file. As per MPL/GPL/LGPL removal of this
# notice is prohibited.
#
# Please contact us via: kamaelia-list-owner@lists.sourceforge.net
# to discuss alternative licensing.
# -------------------------------------------------------------------------
#

import pygame
import Axon
from Axon.Ipc import producerFinished
from Kamaelia.UI.PygameDisplay import PygameDisplay

class KeyEvent(Axon.Component.component):
   """
   """

   Inboxes = { "inbox"    : "Receive events from PygameDisplay",
               "control"  : "",
               "callback" : "Receive callbacks from PygameDisplay"
             }
   Outboxes = { "outbox" : "button click events emitted here",
                "allkeys" : "Outbox that receives *every* keystroke if enabled",
                "signal" : "",
                "display_signal" : "Outbox used for communicating to the display surface" }

   def __init__(self, caption=None,
                position=None, margin=8,
                bgcolour = (224,224,224),
                fgcolour = (0,0,0),
                msg=None,
                allkeys = False,
                key_events = None,
                transparent = False,
                outboxes = {}):
      """Creates and activates a button widget
         caption  = text label for the button / None for default label
         position = (x,y) pair / None
         margin   = margin size (around the text) in pixels
         bgcolour = background colour
         fgcolour = text colour
         msg      = message to be sent when this button is clicked / None for default
         """
      self.Outboxes = self.__class__.Outboxes
      self.Outboxes.update(outboxes)
      super(KeyEvent,self).__init__()

      self.allkeys = allkeys
      self.backgroundColour = bgcolour
      self.foregroundColour = fgcolour
      self.margin = margin
      self.key_events = key_events
      if self.key_events is None: self.key_events = {}

      if caption is None:
         caption = "Button "+str(self.id)

      pygame.font.init()
#      self.buildCaption(caption)

      if msg is None:
         msg = ("CLICK", self.id)
      self.eventMsg = msg
      if transparent:
         transparency = bgcolour
      else:
         transparency = None
      self.disprequest = { "DISPLAYREQUEST" : True,
                           "callback" : (self,"callback"),
                           "events" : (self, "inbox"),
                           "size": (0,0),
                           "transparency" : transparency }

      if not position is None:
        self.disprequest["position"] = position

   def buildCaption(self, text):
      """Render the text to go on the button label.
      (This doesn't actually place the text onto the 'surface')
      """
      font = pygame.font.Font(None, 14)
      self.image = font.render(text,True, self.foregroundColour, )

      (w,h) = self.image.get_size()
      self.size = (w + 2*self.margin, h + 2*self.margin)
      self.imagePosition = (self.margin, self.margin)

   def waitBox(self,boxname):
      waiting = True
      while waiting:
        if self.dataReady(boxname): return
        else: yield 1

   def main(self):
      displayservice = PygameDisplay.getDisplayService()
      self.link((self,"display_signal"), displayservice)

      self.send( self.disprequest,
                  "display_signal")

      for _ in self.waitBox("callback"): yield 1
      self.display = self.recv("callback")
      self.blitToSurface()

      self.send({ "ADDLISTENEVENT" : pygame.MOUSEBUTTONDOWN,
                  "surface" : self.display},
                  "display_signal")
      if (self.key_events is not None) or self.allkeys:
         message = { "ADDLISTENEVENT" : pygame.KEYDOWN,
                     "surface" : self.display,
                     "TRACE" : "ME"}
         self.send(message, "display_signal")

      if self.allkeys:
         message = { "ADDLISTENEVENT" : pygame.KEYUP,
                     "surface" : self.display,
                     "TRACE" : "METO"}
         self.send(message, "display_signal")

      done = False
      while not done:
         if self.dataReady("control"):
            cmsg = self.recv("control")
            if isinstance(cmsg, producerFinished) or isinstance(cmsg, shutdownMicroprocess):
               done = True

         while self.dataReady("inbox"):
            for event in self.recv("inbox"):
                if event.type == pygame.MOUSEBUTTONDOWN:
                   bounds = self.display.get_rect()
                   if bounds.collidepoint(*event.pos):
                      self.send( self.eventMsg, "outbox" )
                if event.type == pygame.KEYDOWN:
                   if event.key in self.key_events:
                      self.send( self.key_events[event.key][0] , self.key_events[event.key][1] )
                   if self.allkeys:
                      self.send(("DOWN", event.key), "allkeys")
                if event.type == pygame.KEYDOWN:
                   if self.allkeys:
                      self.send(("UP", event.key), "allkeys")
         yield 1

   def blitToSurface(self):
       try:
           self.display.fill( self.backgroundColour )
           self.display.blit( self.image, self.imagePosition )
       except:
           pass

if __name__ == "__main__":
   from Kamaelia.Util.ConsoleEcho import consoleEchoer

   button1 = Button().activate()
   button2 = Button(caption="Reverse colours",fgcolour=(255,255,255),bgcolour=(0,0,0)).activate()
   button3 = Button(caption="Mary...",msg="Mary had a little lamb").activate()

   ce = consoleEchoer().activate()
   button1.link( (button1,"outbox"), (ce,"inbox") )
   button2.link( (button2,"outbox"), (ce,"inbox") )
   button3.link( (button3,"outbox"), (ce,"inbox") )

   Axon.Scheduler.scheduler.run.runThreads()  <|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# NiftyNet documentation build configuration file, created by
# sphinx-quickstart on Wed Aug 30 14:13:50 2017.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import subprocess
import os
import sys

working_dir = os.path.abspath(os.path.dirname(__file__))
root_dir = os.path.abspath(os.path.join(working_dir, '..', '..'))
module_path = root_dir
sys.path.insert(0, module_path)

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = [
    'tests',
    'run_*',
    'net_*',
    'setup.py',
]

def generate_apidocs(*args):
    """Generate API docs automatically by trawling the available modules"""
    global working_dir, module_path
    output_path = working_dir
    apidoc_command_path = 'sphinx-apidoc'
    if hasattr(sys, 'real_prefix'):  # called from a virtualenv
        apidoc_command_path = os.path.join(sys.prefix, 'bin', 'sphinx-apidoc')
        apidoc_command_path = os.path.abspath(apidoc_command_path)
    subprocess.check_call(
        [apidoc_command_path, '-o', output_path, module_path] +
        [os.path.join(root_dir, exclude_pattern) for exclude_pattern in exclude_patterns])

def setup(app):
    # Hook to allow for automatic generation of API docs
    # before doc deployment begins.
    app.connect('builder-inited', generate_apidocs)

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['sphinx.ext.autodoc',
    'sphinx.ext.imgmath']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'NiftyNet'
copyright = u'2017, NiftyNet Consortium'
author = u'NiftyNet Consortium'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = u''
# The full version, including alpha/beta/rc tags.
release = u''

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
logo_path = os.path.join(root_dir, 'niftynet-logo.png')
html_theme_options = {
    'logo': logo_path,
    'logo_name': 'true',
    'description': 'An open-source CNN platform for medical image analysis and image-guided therapy',
    'touch_icon': logo_path,
}

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# -- Options for HTMLHelp output ------------------------------------------

# Output file base name for HTML help builder.
htmlhelp_basename = 'NiftyNetdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #
    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #
    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #
    # 'preamble': '',

    # Latex figure (float) alignment
    #
    # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'NiftyNet.tex', u'NiftyNet Documentation',
     u'NiftyNet Consortium', 'manual'),
]

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'niftynet', u'NiftyNet Documentation',
     [author], 1)
]

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'NiftyNet', u'NiftyNet Documentation',
     author, 'NiftyNet', 'One line description of project.',
     'Miscellaneous'),
]

<commit_msg>Issue #112: renamed root_dir => root_dir_abs for clarity<commit_after># -*- coding: utf-8 -*-
#
# NiftyNet documentation build configuration file, created by
# sphinx-quickstart on Wed Aug 30 14:13:50 2017.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import subprocess
import os
import sys

working_dir = os.path.abspath(os.path.dirname(__file__))
root_dir_abs = os.path.abspath(os.path.join(working_dir, '..', '..'))
module_path = root_dir_abs
sys.path.insert(0, module_path)

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = [
    'tests',
    'run_*',
    'net_*',
    'setup.py',
]

def generate_apidocs(*args):
    """Generate API docs automatically by trawling the available modules"""
    global working_dir, module_path
    output_path = working_dir
    apidoc_command_path = 'sphinx-apidoc'
    if hasattr(sys, 'real_prefix'):  # called from a virtualenv
        apidoc_command_path = os.path.join(sys.prefix, 'bin', 'sphinx-apidoc')
        apidoc_command_path = os.path.abspath(apidoc_command_path)
    subprocess.check_call(
        [apidoc_command_path, '-o', output_path, module_path] +
        [os.path.join(root_dir_abs, exclude_pattern) for exclude_pattern in exclude_patterns])

def setup(app):
    # Hook to allow for automatic generation of API docs
    # before doc deployment begins.
    app.connect('builder-inited', generate_apidocs)

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['sphinx.ext.autodoc',
    'sphinx.ext.imgmath']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'NiftyNet'
copyright = u'2017, NiftyNet Consortium'
author = u'NiftyNet Consortium'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = u''
# The full version, including alpha/beta/rc tags.
release = u''

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
logo_path = os.path.join(root_dir_abs, 'niftynet-logo.png')
html_theme_options = {
    'logo': logo_path,
    'logo_name': 'true',
    'description': 'An open-source CNN platform for medical image analysis and image-guided therapy',
    'touch_icon': logo_path,
}

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# -- Options for HTMLHelp output ------------------------------------------

# Output file base name for HTML help builder.
htmlhelp_basename = 'NiftyNetdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #
    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #
    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #
    # 'preamble': '',

    # Latex figure (float) alignment
    #
    # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'NiftyNet.tex', u'NiftyNet Documentation',
     u'NiftyNet Consortium', 'manual'),
]

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'niftynet', u'NiftyNet Documentation',
     [author], 1)
]

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'NiftyNet', u'NiftyNet Documentation',
     author, 'NiftyNet', 'One line description of project.',
     'Miscellaneous'),
]

<|endoftext|>


<commit_before>"""Detect structural variation in genomes using high-throughput sequencing data.
"""
import collections
import copy
import operator
import os

import toolz as tz

from bcbio import utils
from bcbio.cwl import cwlutils
from bcbio.pipeline import datadict as dd
from bcbio.structural import (battenberg, cn_mops, cnvkit, delly, gridss,
                              lumpy, manta, metasv, prioritize, plot,
                              seq2c, titancna, validate, wham)
from bcbio.variation import vcfutils

# Stratify callers by stage -- see `run` documentation below for definitions
_CALLERS = {
  "precall": {"seq2c": seq2c.precall},
  "initial": {"cnvkit": cnvkit.run},
  "standard": {"cn.mops": cn_mops.run, "manta": manta.run, "cnvkit": cnvkit.run,
               "delly": delly.run, "lumpy": lumpy.run, "wham": wham.run,
               "battenberg": battenberg.run, "seq2c": seq2c.run, "gridss": gridss.run,
               "titancna": titancna.run},
  "ensemble": {"metasv": metasv.run,
               "prioritize": prioritize.run}}
_NEEDS_BACKGROUND = set(["cn.mops"])
_GLOBAL_BATCHING = set(["seq2c"])

def _get_callers(items, stage):
    """Retrieve available callers for the provided stage.

    Handles special cases like CNVkit that can be in initial or standard
    depending on if fed into Lumpy analysis.
    """
    callers = utils.deepish_copy(_CALLERS[stage])
    if "cnvkit" in callers:
        has_lumpy = any("lumpy" in get_svcallers(d) or "lumpy" in d["config"]["algorithm"].get("svcaller_orig", [])
                        for d in items)
        if has_lumpy and any("lumpy_usecnv" in dd.get_tools_on(d) for d in items):
            if stage != "initial":
                del callers["cnvkit"]
        else:
            if stage != "standard":
                del callers["cnvkit"]
    return callers

def get_svcallers(data):
    svs = data["config"]["algorithm"].get("svcaller")
    if svs is None:
        svs = []
    elif isinstance(svs, basestring):
        svs = [svs]
    return svs

def _handle_multiple_svcallers(data, stage):
    """Retrieve configured structural variation caller, handling multiple.
    """
    svs = get_svcallers(data)
    # special cases -- prioritization
    if stage == "ensemble" and dd.get_svprioritize(data):
        svs.append("prioritize")
    out = []
    for svcaller in svs:
        if svcaller in _get_callers([data], stage):
            base = copy.deepcopy(data)
            # clean SV callers present in multiple rounds and not this caller
            final_svs = []
            for sv in data.get("sv", []):
                if stage == "ensemble" or sv["variantcaller"] == svcaller or sv["variantcaller"] not in svs:
                    final_svs.append(sv)
            base["sv"] = final_svs
            base["config"]["algorithm"]["svcaller"] = svcaller
            base["config"]["algorithm"]["svcaller_orig"] = svs
            out.append(base)
    return out

def finalize_sv(samples, config):
    """Combine results from multiple sv callers into a single ordered 'sv' key.
    """
    by_bam = collections.OrderedDict()
    for x in samples:
        batch = dd.get_batch(x) or [dd.get_sample_name(x)]
        try:
            by_bam[x["align_bam"], tuple(batch)].append(x)
        except KeyError:
            by_bam[x["align_bam"], tuple(batch)] = [x]
    by_batch = collections.OrderedDict()
    lead_batches = {}
    for grouped_calls in by_bam.values():
        def orig_svcaller_order(x):
            orig_callers = tz.get_in(["config", "algorithm", "svcaller_orig"], x)
            cur_caller = tz.get_in(["config", "algorithm", "svcaller"], x)
            return orig_callers.index(cur_caller)
        sorted_svcalls = sorted([x for x in grouped_calls if "sv" in x],
                                key=orig_svcaller_order)
        final = grouped_calls[0]
        if len(sorted_svcalls) > 0:
            final["sv"] = reduce(operator.add, [x["sv"] for x in sorted_svcalls])
        final["config"]["algorithm"]["svcaller"] = final["config"]["algorithm"].pop("svcaller_orig")
        batch = dd.get_batch(final) or dd.get_sample_name(final)
        batches = batch if isinstance(batch, (list, tuple)) else [batch]
        if len(batches) > 1:
            lead_batches[(dd.get_sample_name(final), dd.get_phenotype(final) == "germline")] = batches[0]
        for batch in batches:
            try:
                by_batch[batch].append(final)
            except KeyError:
                by_batch[batch] = [final]
    out = []
    for batch, items in by_batch.items():
        if any("svplots" in dd.get_tools_on(d) for d in items):
            items = plot.by_regions(items)
        for data in items:
            if lead_batches.get((dd.get_sample_name(data), dd.get_phenotype(data) == "germline")) in [batch, None]:
                out.append([data])
    return out

def validate_sv(data):
    """Validate structural variant calls for a sample.
    """
    return [[validate.evaluate(data)]]

def batch_for_sv(samples):
    """Prepare a set of samples for parallel structural variant calling.

    CWL input target -- groups samples into batches and structural variant
    callers for parallel processing.
    """
    to_process, extras, background = _batch_split_by_sv(samples, "standard")
    out = [cwlutils.samples_to_records(xs) for xs in to_process.values()] + extras
    return out

def _batch_split_by_sv(samples, stage):
    to_process = collections.OrderedDict()
    extras = []
    background = []
    for data in (utils.to_single_data(x) for x in samples):
        ready_data = _handle_multiple_svcallers(data, stage)
        if len(ready_data) > 0:
            background.append(data)
            for x in ready_data:
                svcaller = tz.get_in(["config", "algorithm", "svcaller"], x)
                batch = dd.get_batch(x) or dd.get_sample_name(x)
                if stage in ["precall", "ensemble"]:  # no batching for precall or ensemble methods
                    if isinstance(batch, basestring) and batch != dd.get_sample_name(x):
                        batch += "_%s" % dd.get_sample_name(x)
                    else:
                        batch = dd.get_sample_name(x)
                    if dd.get_phenotype(x) == "germline":
                        batch += "_germline"
                elif svcaller in _GLOBAL_BATCHING:  # All samples batched together for analyses
                    batch = "all"
                batches = batch if isinstance(batch, (list, tuple)) else [batch]
                for b in batches:
                    try:
                        to_process[(svcaller, b)].append(x)
                    except KeyError:
                        to_process[(svcaller, b)] = [x]
        else:
            extras.append([data])
    return to_process, extras, background

def run(samples, run_parallel, stage):
    """Run structural variation detection.

    The stage indicates which level of structural variant calling to run.
      - precall, perform initial sample based assessment of samples
      - initial, callers that can be used in subsequent structural variation steps (cnvkit -> lumpy)
      - standard, regular batch calling
      - ensemble, post-calling, combine other callers or prioritize results
    """
    to_process, extras, background = _batch_split_by_sv(samples, stage)
    processed = run_parallel("detect_sv", ([xs, background, stage]
                                           for xs in to_process.values()))
    finalized = (run_parallel("finalize_sv", [([xs[0] for xs in processed], processed[0][0]["config"])])
                 if len(processed) > 0 else [])
    return extras + finalized

def detect_sv(items, all_items=None, stage="standard"):
    """Top level parallel target for examining structural variation.
    """
    items = [utils.to_single_data(x) for x in items]
    svcaller = items[0]["config"]["algorithm"].get("svcaller")
    caller_fn = _get_callers(items, stage).get(svcaller)
    out = []
    if svcaller and caller_fn:
        if (all_items and svcaller in _NEEDS_BACKGROUND and
                not vcfutils.is_paired_analysis([x.get("align_bam") for x in items], items)):
            names = set([dd.get_sample_name(x) for x in items])
            background = [x for x in all_items if dd.get_sample_name(x) not in names]
            for svdata in caller_fn(items, background):
                out.append([svdata])
        else:
            for svdata in caller_fn(items):
                out.append([svdata])
    else:
        for data in items:
            out.append([data])
    # Avoid nesting of callers for CWL runs for easier extraction
    if "cwl_keys" in items[0]:
        out_cwl = []
        for data in [utils.to_single_data(x) for x in out]:
            svs = data.get("sv")
            if svs:
                assert len(svs) == 1, svs
                data["sv"] = svs[0]
            out_cwl.append([data])
        return out_cwl
    return out

def summarize_sv(items):
    """CWL target: summarize structural variants for multiple samples.

    XXX Need to support non-VCF output as tabix indexed output
    """
    items = [utils.to_single_data(x) for x in utils.flatten(items)]
    out = {"sv": {"calls": []}}
    added = set([])
    for data in items:
        if data.get("sv"):
            names = dd.get_batches(data)
            if not names:
                names = [dd.get_sample_name(data)]
            batch_name = names[0]
            cur_name = "%s-%s" % (batch_name, data["sv"]["variantcaller"])
            if data["sv"].get("vrn_file"):
                ext = utils.splitext_plus(data["sv"]["vrn_file"])[-1]
                if cur_name not in added and ext.startswith(".vcf"):
                    added.add(cur_name)
                    out_file = os.path.join(utils.safe_makedir(os.path.join(dd.get_work_dir(data),
                                                                            "sv", "calls")),
                                            "%s%s" % (cur_name, ext))
                    utils.copy_plus(data["sv"]["vrn_file"], out_file)
                    out_file = vcfutils.bgzip_and_index(out_file, data["config"])
                    out["sv"]["calls"].append(out_file)
    return [out]

# ## configuration

def parallel_multiplier(items):
    """Use more resources (up to available limits) if we have multiple QC samples/svcallers.
    """
    machines = []
    for data in (xs[0] for xs in items):
        machines.append(max(1, len(get_svcallers(data)), len(dd.get_algorithm_qc(data))))
    return sum(machines)
<commit_msg>SV: do not skip CNVkit inclusion with lumpy_usecnv<commit_after>"""Detect structural variation in genomes using high-throughput sequencing data.
"""
import collections
import copy
import operator
import os

import toolz as tz

from bcbio import utils
from bcbio.cwl import cwlutils
from bcbio.pipeline import datadict as dd
from bcbio.structural import (battenberg, cn_mops, cnvkit, delly, gridss,
                              lumpy, manta, metasv, prioritize, plot,
                              seq2c, titancna, validate, wham)
from bcbio.variation import vcfutils

# Stratify callers by stage -- see `run` documentation below for definitions
_CALLERS = {
  "precall": {"seq2c": seq2c.precall},
  "initial": {"cnvkit": cnvkit.run},
  "standard": {"cn.mops": cn_mops.run, "manta": manta.run, "cnvkit": cnvkit.run,
               "delly": delly.run, "lumpy": lumpy.run, "wham": wham.run,
               "battenberg": battenberg.run, "seq2c": seq2c.run, "gridss": gridss.run,
               "titancna": titancna.run},
  "ensemble": {"metasv": metasv.run,
               "prioritize": prioritize.run}}
_NEEDS_BACKGROUND = set(["cn.mops"])
_GLOBAL_BATCHING = set(["seq2c"])

def _get_callers(items, stage, special_cases=False):
    """Retrieve available callers for the provided stage.

    Handles special cases like CNVkit that can be in initial or standard
    depending on if fed into Lumpy analysis.
    """
    callers = utils.deepish_copy(_CALLERS[stage])
    if special_cases and "cnvkit" in callers:
        has_lumpy = any("lumpy" in get_svcallers(d) or "lumpy" in d["config"]["algorithm"].get("svcaller_orig", [])
                        for d in items)
        if has_lumpy and any("lumpy_usecnv" in dd.get_tools_on(d) for d in items):
            if stage != "initial":
                del callers["cnvkit"]
        else:
            if stage != "standard":
                del callers["cnvkit"]
    return callers

def get_svcallers(data):
    svs = data["config"]["algorithm"].get("svcaller")
    if svs is None:
        svs = []
    elif isinstance(svs, basestring):
        svs = [svs]
    return svs

def _handle_multiple_svcallers(data, stage):
    """Retrieve configured structural variation caller, handling multiple.
    """
    svs = get_svcallers(data)
    # special cases -- prioritization
    if stage == "ensemble" and dd.get_svprioritize(data):
        svs.append("prioritize")
    out = []
    for svcaller in svs:
        if svcaller in _get_callers([data], stage):
            base = copy.deepcopy(data)
            # clean SV callers present in multiple rounds and not this caller
            final_svs = []
            for sv in data.get("sv", []):
                if stage == "ensemble" or sv["variantcaller"] == svcaller or sv["variantcaller"] not in svs:
                    final_svs.append(sv)
            base["sv"] = final_svs
            base["config"]["algorithm"]["svcaller"] = svcaller
            base["config"]["algorithm"]["svcaller_orig"] = svs
            out.append(base)
    return out

def finalize_sv(samples, config):
    """Combine results from multiple sv callers into a single ordered 'sv' key.
    """
    by_bam = collections.OrderedDict()
    for x in samples:
        batch = dd.get_batch(x) or [dd.get_sample_name(x)]
        try:
            by_bam[x["align_bam"], tuple(batch)].append(x)
        except KeyError:
            by_bam[x["align_bam"], tuple(batch)] = [x]
    by_batch = collections.OrderedDict()
    lead_batches = {}
    for grouped_calls in by_bam.values():
        def orig_svcaller_order(x):
            orig_callers = tz.get_in(["config", "algorithm", "svcaller_orig"], x)
            cur_caller = tz.get_in(["config", "algorithm", "svcaller"], x)
            return orig_callers.index(cur_caller)
        sorted_svcalls = sorted([x for x in grouped_calls if "sv" in x],
                                key=orig_svcaller_order)
        final = grouped_calls[0]
        if len(sorted_svcalls) > 0:
            final["sv"] = reduce(operator.add, [x["sv"] for x in sorted_svcalls])
        final["config"]["algorithm"]["svcaller"] = final["config"]["algorithm"].pop("svcaller_orig")
        batch = dd.get_batch(final) or dd.get_sample_name(final)
        batches = batch if isinstance(batch, (list, tuple)) else [batch]
        if len(batches) > 1:
            lead_batches[(dd.get_sample_name(final), dd.get_phenotype(final) == "germline")] = batches[0]
        for batch in batches:
            try:
                by_batch[batch].append(final)
            except KeyError:
                by_batch[batch] = [final]
    out = []
    for batch, items in by_batch.items():
        if any("svplots" in dd.get_tools_on(d) for d in items):
            items = plot.by_regions(items)
        for data in items:
            if lead_batches.get((dd.get_sample_name(data), dd.get_phenotype(data) == "germline")) in [batch, None]:
                out.append([data])
    return out

def validate_sv(data):
    """Validate structural variant calls for a sample.
    """
    return [[validate.evaluate(data)]]

def batch_for_sv(samples):
    """Prepare a set of samples for parallel structural variant calling.

    CWL input target -- groups samples into batches and structural variant
    callers for parallel processing.
    """
    to_process, extras, background = _batch_split_by_sv(samples, "standard")
    out = [cwlutils.samples_to_records(xs) for xs in to_process.values()] + extras
    return out

def _batch_split_by_sv(samples, stage):
    to_process = collections.OrderedDict()
    extras = []
    background = []
    for data in (utils.to_single_data(x) for x in samples):
        ready_data = _handle_multiple_svcallers(data, stage)
        if len(ready_data) > 0:
            background.append(data)
            for x in ready_data:
                svcaller = tz.get_in(["config", "algorithm", "svcaller"], x)
                batch = dd.get_batch(x) or dd.get_sample_name(x)
                if stage in ["precall", "ensemble"]:  # no batching for precall or ensemble methods
                    if isinstance(batch, basestring) and batch != dd.get_sample_name(x):
                        batch += "_%s" % dd.get_sample_name(x)
                    else:
                        batch = dd.get_sample_name(x)
                    if dd.get_phenotype(x) == "germline":
                        batch += "_germline"
                elif svcaller in _GLOBAL_BATCHING:  # All samples batched together for analyses
                    batch = "all"
                batches = batch if isinstance(batch, (list, tuple)) else [batch]
                for b in batches:
                    try:
                        to_process[(svcaller, b)].append(x)
                    except KeyError:
                        to_process[(svcaller, b)] = [x]
        else:
            extras.append([data])
    return to_process, extras, background

def run(samples, run_parallel, stage):
    """Run structural variation detection.

    The stage indicates which level of structural variant calling to run.
      - precall, perform initial sample based assessment of samples
      - initial, callers that can be used in subsequent structural variation steps (cnvkit -> lumpy)
      - standard, regular batch calling
      - ensemble, post-calling, combine other callers or prioritize results
    """
    to_process, extras, background = _batch_split_by_sv(samples, stage)
    processed = run_parallel("detect_sv", ([xs, background, stage]
                                           for xs in to_process.values()))
    finalized = (run_parallel("finalize_sv", [([xs[0] for xs in processed], processed[0][0]["config"])])
                 if len(processed) > 0 else [])
    return extras + finalized

def detect_sv(items, all_items=None, stage="standard"):
    """Top level parallel target for examining structural variation.
    """
    items = [utils.to_single_data(x) for x in items]
    svcaller = items[0]["config"]["algorithm"].get("svcaller")
    caller_fn = _get_callers(items, stage, special_cases=True).get(svcaller)
    out = []
    if svcaller and caller_fn:
        if (all_items and svcaller in _NEEDS_BACKGROUND and
                not vcfutils.is_paired_analysis([x.get("align_bam") for x in items], items)):
            names = set([dd.get_sample_name(x) for x in items])
            background = [x for x in all_items if dd.get_sample_name(x) not in names]
            for svdata in caller_fn(items, background):
                out.append([svdata])
        else:
            for svdata in caller_fn(items):
                out.append([svdata])
    else:
        for data in items:
            out.append([data])
    # Avoid nesting of callers for CWL runs for easier extraction
    if "cwl_keys" in items[0]:
        out_cwl = []
        for data in [utils.to_single_data(x) for x in out]:
            svs = data.get("sv")
            if svs:
                assert len(svs) == 1, svs
                data["sv"] = svs[0]
            out_cwl.append([data])
        return out_cwl
    return out

def summarize_sv(items):
    """CWL target: summarize structural variants for multiple samples.

    XXX Need to support non-VCF output as tabix indexed output
    """
    items = [utils.to_single_data(x) for x in utils.flatten(items)]
    out = {"sv": {"calls": []}}
    added = set([])
    for data in items:
        if data.get("sv"):
            names = dd.get_batches(data)
            if not names:
                names = [dd.get_sample_name(data)]
            batch_name = names[0]
            cur_name = "%s-%s" % (batch_name, data["sv"]["variantcaller"])
            if data["sv"].get("vrn_file"):
                ext = utils.splitext_plus(data["sv"]["vrn_file"])[-1]
                if cur_name not in added and ext.startswith(".vcf"):
                    added.add(cur_name)
                    out_file = os.path.join(utils.safe_makedir(os.path.join(dd.get_work_dir(data),
                                                                            "sv", "calls")),
                                            "%s%s" % (cur_name, ext))
                    utils.copy_plus(data["sv"]["vrn_file"], out_file)
                    out_file = vcfutils.bgzip_and_index(out_file, data["config"])
                    out["sv"]["calls"].append(out_file)
    return [out]

# ## configuration

def parallel_multiplier(items):
    """Use more resources (up to available limits) if we have multiple QC samples/svcallers.
    """
    machines = []
    for data in (xs[0] for xs in items):
        machines.append(max(1, len(get_svcallers(data)), len(dd.get_algorithm_qc(data))))
    return sum(machines)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Copyright 2010 Asidev s.r.l.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

from sqlalchemy import engine_from_config
import aybu.manager.models

def setup(env):
    settings = env['request'].registry.settings
    env['models'] = aybu.manager.models
    env['engine'] = engine_from_config(settings, 'sqlalchemy.')
    env['request'].set_db_engine = env['engine']
    aybu.manager.models.Base.metadata.bind = env['engine']
    aybu.manager.models.Environment.initialize(settings)
    env['session'] = env['request'].db_session

<commit_msg>Add activity log to session in paster pshell<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Copyright 2010 Asidev s.r.l.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

from sqlalchemy import engine_from_config
import aybu.manager.models
from aybu.manager.activity_log import ActivityLog

def setup(env):
    settings = env['request'].registry.settings
    env['models'] = aybu.manager.models
    env['engine'] = engine_from_config(settings, 'sqlalchemy.')
    env['request'].set_db_engine = env['engine']
    aybu.manager.models.Base.metadata.bind = env['engine']
    aybu.manager.models.Environment.initialize(settings)
    env['session'] = env['request'].db_session
    ActivityLog.attach_to(env['session'])

<|endoftext|>