<commit_before>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import re
import logging
import os.path
import requests
from lxml import etree
from lxml.etree import QName
import xml.etree.ElementTree as ET

from indra.literature import pubmed_client
from indra.util import UnicodeXMLTreeBuilder as UTB

# Python 2
try:
    basestring
# Python 3
except:
    basestring = str

logger = logging.getLogger(__name__)

pmc_url = 'https://www.ncbi.nlm.nih.gov/pmc/oai/oai.cgi'
pmid_convert_url = 'https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/'

# Paths to resource files
pmids_fulltext_path = os.path.join(os.path.dirname(__file__),
                                   'pmids_fulltext.txt')
pmids_oa_xml_path = os.path.join(os.path.dirname(__file__),
                                 'pmids_oa_xml.txt')
pmids_oa_txt_path = os.path.join(os.path.dirname(__file__),
                                 'pmids_oa_txt.txt')
pmids_auth_xml_path = os.path.join(os.path.dirname(__file__),
                                   'pmids_auth_xml.txt')
# Define global dict containing lists of PMIDs among mineable PMCs
# to be lazily initialized
pmids_fulltext_dict = {}

def id_lookup(paper_id, idtype=None):
    """This function takes a Pubmed ID, Pubmed Central ID, or DOI
    and use the Pubmed ID mapping
    service and looks up all other IDs from one
    of these. The IDs are returned in a dictionary."""
    if idtype is not None and idtype not in ('pmid', 'pmcid', 'doi'):
        raise ValueError("Invalid idtype %s; must be 'pmid', 'pmcid', "
                         "or 'doi'." % idtype)
    if paper_id.upper().startswith('PMC'):
        idtype = 'pmcid'
    # Strip off any prefix
    if paper_id.upper().startswith('PMID'):
        paper_id = paper_id[4:]
    elif paper_id.upper().startswith('DOI'):
        paper_id = paper_id[3:]
    data = {'ids': paper_id}
    if idtype is not None:
        data['idtype'] = idtype
    try:
        tree = pubmed_client.send_request(pmid_convert_url, data)
    except Exception as e:
        logger.error('Error looking up PMID in PMC: %s' % e)
        return {}
    if tree is None:
        return {}
    record = tree.find('record')
    if record is None:
        return {}
    doi = record.attrib.get('doi')
    pmid = record.attrib.get('pmid')
    pmcid = record.attrib.get('pmcid')
    ids = {'doi': doi,
           'pmid': pmid,
           'pmcid': pmcid}
    return ids

def get_ids(search_term, retmax=1000):
    return pubmed_client.get_ids(search_term, retmax=retmax, db='pmc')

def get_xml(pmc_id):
    """Returns XML for the article corresponding to a PMC ID."""
    if pmc_id.upper().startswith('PMC'):
        pmc_id = pmc_id[3:]
    # Request params
    params = {}
    params['verb'] = 'GetRecord'
    params['identifier'] = 'oai:pubmedcentral.nih.gov:%s' % pmc_id
    params['metadataPrefix'] = 'pmc'
    # Submit the request
    res = requests.get(pmc_url, params)
    if not res.status_code == 200:
        logger.warning("Couldn't download %s" % pmc_id)
        return None
    # Read the bytestream
    xml_bytes = res.content
    # Check for any XML errors; xml_str should still be bytes
    tree = ET.XML(xml_bytes, parser=UTB())
    xmlns = "http://www.openarchives.org/OAI/2.0/"
    err_tag = tree.find('{%s}error' % xmlns)
    if err_tag is not None:
        err_code = err_tag.attrib['code']
        err_text = err_tag.text
        logger.warning('PMC client returned with error %s: %s'
                       % (err_code, err_text))
        return None
    # If no error, return the XML as a unicode string
    else:
        return xml_bytes.decode('utf-8')

def extract_text(xml_string):
    """Get text from the body of the given NLM XML string.

    Parameters
    ----------
    xml_string : str
        String containing valid NLM XML.

    Returns
    -------
    str
        Extracted plaintext.
    """
    paragraphs = extract_paragraphs(xml_string)
    if paragraphs:
        return '\n'.join(paragraphs) + '\n'
    else:
        return None

def _select_main_article_front(xml_string):
    """Return front of NLM XML string

    Parameters
    ----------
    xml_str : str
        A valid NLM XML string

    list
        List containing lxml Element of front section of main article
    """
    output = []
    tree = etree.fromstring(xml_string.encode('utf-8'))
    front_xpath = _namespace_unaware_xpath('article', 'front')
    for element in tree.xpath(front_xpath):
        output.append(element)
    return output

def _select_main_article_body(xml_string):
    """Return body of NLM XML string

    Parameters
    ----------
    xml_str : str
        A valid NLM XML string

    list
        List containing lxml Element of main body.
    """
    output = []
    tree = etree.fromstring(xml_string.encode('utf-8'))
    body_xpath = _namespace_unaware_xpath('article', 'body')
    for element in tree.xpath(body_xpath):
        output.append(element)
    return output

def _select_subarticles(xml_string):
    """Return sub-articles of NLM XML string

    Parameters
    ----------
    xml_str : str
        A valid NLM XML string

    list
        List containing lxml Element of front section of main article
    """
    output = []
    tree = etree.fromstring(xml_string.encode('utf-8'))
    sub_article_xpath = _namespace_unaware_xpath('article', 'sub-article')
    for element in tree.xpath(sub_article_xpath):
        output.append(element)
    return output

def _extract_from_front(front_element):
    """Return list of titles and paragraphs from front of NLM XML

    Parameters
    ----------
    front_element : :py:class:`lxml.etree._Element`
        etree element for front of a valid NLM XML
    Returns
    -------
    list of str
        List of relevant plain text titles and paragraphs taken from front
        section of NLM XML. These include the article title, alt title,
        and paragraphs within abstracts. Unwanted paragraphs such as
        author statements are excluded.
    """
    output = []
    title_xpath = _namespace_unaware_xpath('article-meta', 'title-group',
                                           'article-title')
    alt_title_xpath = _namespace_unaware_xpath('article-meta', 'title-group',
                                               'alt-title')
    abstracts_xpath = _namespace_unaware_xpath('article-meta', 'abstract')
    for element in front_element.xpath(_xpath_union(title_xpath,
                                                    alt_title_xpath,
                                                    abstracts_xpath)):
        if element.tag == 'abstract':
            # Extract paragraphs from abstracts
            output.extend(_extract_paragraphs_from_tree(element))
        else:
            # No paragraphs in titles, Just strip tags
            output.append(' '.join(element.itertext()))
    return output

def _extract_from_body(body_element):
    """Return list of paragraphs from main article body of NLM XML
    """
    return _extract_paragraphs_from_tree(body_element)

def _extract_from_subarticle(subarticle_element):
    return ''

def _extract_paragraphs_from_tree(tree):
    paragraphs = []
    # In NLM xml, all plaintext is within <p> tags and <title> tags.
    # There can be formatting tags nested within these tags, but no
    # unwanted elements such as figures and tables appear nested
    # within <p> tags and <title> tags. xpath local-name()= syntax
    # is used to ignore namespaces in the NLM XML
    pars_xpath = _xpath_union(_namespace_unaware_xpath('p', direct_only=False),
                              _namespace_unaware_xpath('title',
                                                       direct_only=False))
    for element in tree.xpath(pars_xpath):
        paragraph = ' '.join(element.itertext())
        paragraphs.append(paragraph)
    return paragraphs

def extract_paragraphs(xml_string):
    """Returns list of paragraphs in an NLM XML.

    Parameters
    ----------
    xml_string : str
        String containing valid NLM XML.

    Returns
    -------
    list of str
        List of extracted paragraphs in an NLM XML
    """
    tree = etree.fromstring(xml_string.encode('utf-8'))

    return _extract_paragraphs_from_tree(tree)

def filter_pmids(pmid_list, source_type):
    """Filter a list of PMIDs for ones with full text from PMC.

    Parameters
    ----------
    pmid_list : list of str
        List of PMIDs to filter.
    source_type : string
        One of 'fulltext', 'oa_xml', 'oa_txt', or 'auth_xml'.

    Returns
    -------
    list of str
        PMIDs available in the specified source/format type.
    """
    global pmids_fulltext_dict
    # Check args
    if source_type not in ('fulltext', 'oa_xml', 'oa_txt', 'auth_xml'):
        raise ValueError("source_type must be one of: 'fulltext', 'oa_xml', "
                         "'oa_txt', or 'auth_xml'.")
    # Check if we've loaded this type, and lazily initialize
    if pmids_fulltext_dict.get(source_type) is None:
        fulltext_list_path = os.path.join(os.path.dirname(__file__),
                                          'pmids_%s.txt' % source_type)
        with open(fulltext_list_path, 'rb') as f:
            fulltext_list = set([line.strip().decode('utf-8')
                                 for line in f.readlines()])
            pmids_fulltext_dict[source_type] = fulltext_list
    return list(set(pmid_list).intersection(
                                pmids_fulltext_dict.get(source_type)))

def _namespace_unaware_xpath(*tag_list, direct_only=True):
    """Create a namespace unaware xpath from a list of tags

    Created xpaths always start with current node, node from
    root of tree

    Parameters
    ----------
    *tag_list
        Variable length argument list of str. Contains tags for
        constructing xpath
    direct_only : Optional[bool]
        If True, xpath only selects nodes directly beneath the
        current node. Otherwise any descendents of the current
        node can be selected. Default: True
    """
    if direct_only:
        out = '.'
    else:
        out = './'
    for tag in tag_list:
        out += "/*[local-name()='%s']" % tag
    return out

def _xpath_union(*xpath_list):
    return ' | '.join(xpath_list)
<commit_msg>Consolidate multiple functions to select elements<commit_after>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import re
import logging
import os.path
import requests
from lxml import etree
from lxml.etree import QName
import xml.etree.ElementTree as ET

from indra.literature import pubmed_client
from indra.util import UnicodeXMLTreeBuilder as UTB

# Python 2
try:
    basestring
# Python 3
except:
    basestring = str

logger = logging.getLogger(__name__)

pmc_url = 'https://www.ncbi.nlm.nih.gov/pmc/oai/oai.cgi'
pmid_convert_url = 'https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/'

# Paths to resource files
pmids_fulltext_path = os.path.join(os.path.dirname(__file__),
                                   'pmids_fulltext.txt')
pmids_oa_xml_path = os.path.join(os.path.dirname(__file__),
                                 'pmids_oa_xml.txt')
pmids_oa_txt_path = os.path.join(os.path.dirname(__file__),
                                 'pmids_oa_txt.txt')
pmids_auth_xml_path = os.path.join(os.path.dirname(__file__),
                                   'pmids_auth_xml.txt')
# Define global dict containing lists of PMIDs among mineable PMCs
# to be lazily initialized
pmids_fulltext_dict = {}

def id_lookup(paper_id, idtype=None):
    """This function takes a Pubmed ID, Pubmed Central ID, or DOI
    and use the Pubmed ID mapping
    service and looks up all other IDs from one
    of these. The IDs are returned in a dictionary."""
    if idtype is not None and idtype not in ('pmid', 'pmcid', 'doi'):
        raise ValueError("Invalid idtype %s; must be 'pmid', 'pmcid', "
                         "or 'doi'." % idtype)
    if paper_id.upper().startswith('PMC'):
        idtype = 'pmcid'
    # Strip off any prefix
    if paper_id.upper().startswith('PMID'):
        paper_id = paper_id[4:]
    elif paper_id.upper().startswith('DOI'):
        paper_id = paper_id[3:]
    data = {'ids': paper_id}
    if idtype is not None:
        data['idtype'] = idtype
    try:
        tree = pubmed_client.send_request(pmid_convert_url, data)
    except Exception as e:
        logger.error('Error looking up PMID in PMC: %s' % e)
        return {}
    if tree is None:
        return {}
    record = tree.find('record')
    if record is None:
        return {}
    doi = record.attrib.get('doi')
    pmid = record.attrib.get('pmid')
    pmcid = record.attrib.get('pmcid')
    ids = {'doi': doi,
           'pmid': pmid,
           'pmcid': pmcid}
    return ids

def get_ids(search_term, retmax=1000):
    return pubmed_client.get_ids(search_term, retmax=retmax, db='pmc')

def get_xml(pmc_id):
    """Returns XML for the article corresponding to a PMC ID."""
    if pmc_id.upper().startswith('PMC'):
        pmc_id = pmc_id[3:]
    # Request params
    params = {}
    params['verb'] = 'GetRecord'
    params['identifier'] = 'oai:pubmedcentral.nih.gov:%s' % pmc_id
    params['metadataPrefix'] = 'pmc'
    # Submit the request
    res = requests.get(pmc_url, params)
    if not res.status_code == 200:
        logger.warning("Couldn't download %s" % pmc_id)
        return None
    # Read the bytestream
    xml_bytes = res.content
    # Check for any XML errors; xml_str should still be bytes
    tree = ET.XML(xml_bytes, parser=UTB())
    xmlns = "http://www.openarchives.org/OAI/2.0/"
    err_tag = tree.find('{%s}error' % xmlns)
    if err_tag is not None:
        err_code = err_tag.attrib['code']
        err_text = err_tag.text
        logger.warning('PMC client returned with error %s: %s'
                       % (err_code, err_text))
        return None
    # If no error, return the XML as a unicode string
    else:
        return xml_bytes.decode('utf-8')

def extract_text(xml_string):
    """Get text from the body of the given NLM XML string.

    Parameters
    ----------
    xml_string : str
        String containing valid NLM XML.

    Returns
    -------
    str
        Extracted plaintext.
    """
    paragraphs = extract_paragraphs(xml_string)
    if paragraphs:
        return '\n'.join(paragraphs) + '\n'
    else:
        return None

def _select_from_top_level(xml_string, tag):
    """Return top level elements from an NLM XML article

    Parameters
    ----------
    xml_str : str
        A valid NLM XML string for an entire article

    list
        List containing lxml Element objects of selected top level elements
    """
    output = []
    tree = etree.fromstring(xml_string.encode('utf-8'))
    front_xpath = _namespace_unaware_xpath(tag)
    for element in tree.xpath(front_xpath):
        output.append(element)
    return output

def _extract_from_front(front_element):
    """Return list of titles and paragraphs from front of NLM XML

    Parameters
    ----------
    front_element : :py:class:`lxml.etree._Element`
        etree element for front of a valid NLM XML
    Returns
    -------
    list of str
        List of relevant plain text titles and paragraphs taken from front
        section of NLM XML. These include the article title, alt title,
        and paragraphs within abstracts. Unwanted paragraphs such as
        author statements are excluded.
    """
    output = []
    title_xpath = _namespace_unaware_xpath('article-meta', 'title-group',
                                           'article-title')
    alt_title_xpath = _namespace_unaware_xpath('article-meta', 'title-group',
                                               'alt-title')
    abstracts_xpath = _namespace_unaware_xpath('article-meta', 'abstract')
    for element in front_element.xpath(_xpath_union(title_xpath,
                                                    alt_title_xpath,
                                                    abstracts_xpath)):
        if element.tag == 'abstract':
            # Extract paragraphs from abstracts
            output.extend(_extract_paragraphs_from_tree(element))
        else:
            # No paragraphs in titles, Just strip tags
            output.append(' '.join(element.itertext()))
    return output

def _extract_from_body(body_element):
    """Return list of paragraphs from main article body of NLM XML
    """
    return _extract_paragraphs_from_tree(body_element)

def _extract_from_subarticle(subarticle_element):
    return ''

def _extract_paragraphs_from_tree(tree):
    paragraphs = []
    # In NLM xml, all plaintext is within <p> tags and <title> tags.
    # There can be formatting tags nested within these tags, but no
    # unwanted elements such as figures and tables appear nested
    # within <p> tags and <title> tags. xpath local-name()= syntax
    # is used to ignore namespaces in the NLM XML
    pars_xpath = _xpath_union(_namespace_unaware_xpath('p', direct_only=False),
                              _namespace_unaware_xpath('title',
                                                       direct_only=False))
    for element in tree.xpath(pars_xpath):
        paragraph = ' '.join(element.itertext())
        paragraphs.append(paragraph)
    return paragraphs

def extract_paragraphs(xml_string):
    """Returns list of paragraphs in an NLM XML.

    Parameters
    ----------
    xml_string : str
        String containing valid NLM XML.

    Returns
    -------
    list of str
        List of extracted paragraphs in an NLM XML
    """
    tree = etree.fromstring(xml_string.encode('utf-8'))

    return _extract_paragraphs_from_tree(tree)

def filter_pmids(pmid_list, source_type):
    """Filter a list of PMIDs for ones with full text from PMC.

    Parameters
    ----------
    pmid_list : list of str
        List of PMIDs to filter.
    source_type : string
        One of 'fulltext', 'oa_xml', 'oa_txt', or 'auth_xml'.

    Returns
    -------
    list of str
        PMIDs available in the specified source/format type.
    """
    global pmids_fulltext_dict
    # Check args
    if source_type not in ('fulltext', 'oa_xml', 'oa_txt', 'auth_xml'):
        raise ValueError("source_type must be one of: 'fulltext', 'oa_xml', "
                         "'oa_txt', or 'auth_xml'.")
    # Check if we've loaded this type, and lazily initialize
    if pmids_fulltext_dict.get(source_type) is None:
        fulltext_list_path = os.path.join(os.path.dirname(__file__),
                                          'pmids_%s.txt' % source_type)
        with open(fulltext_list_path, 'rb') as f:
            fulltext_list = set([line.strip().decode('utf-8')
                                 for line in f.readlines()])
            pmids_fulltext_dict[source_type] = fulltext_list
    return list(set(pmid_list).intersection(
                                pmids_fulltext_dict.get(source_type)))

def _namespace_unaware_xpath(*tag_list, direct_only=True):
    """Create a namespace unaware xpath from a list of tags

    Created xpaths always start with current node, node from
    root of tree

    Parameters
    ----------
    *tag_list
        Variable length argument list of str. Contains tags for
        constructing xpath
    direct_only : Optional[bool]
        If True, xpath only selects nodes directly beneath the
        current node. Otherwise any descendents of the current
        node can be selected. Default: True
    """
    if direct_only:
        out = '.'
    else:
        out = './'
    for tag in tag_list:
        out += "/*[local-name()='%s']" % tag
    return out

def _xpath_union(*xpath_list):
    return ' | '.join(xpath_list)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import re
from os.path import join

def needs_veclib_wrapper(info):
    """Returns true if needs special veclib wrapper."""
    import re
    r_accel = re.compile("Accelerate")
    r_vec = re.compile("vecLib")
    res = False
    try:
        tmpstr = info['extra_link_args']
        for i in tmpstr:
            if r_accel.search(i) or r_vec.search(i):
                res = True
    except KeyError:
        pass

    return res

def configuration(parent_package='',top_path=None):
    from numpy.distutils.system_info import get_info, NotFoundError
    from numpy.distutils.misc_util import Configuration

    config = Configuration('arpack',parent_package,top_path)

    lapack_opt = get_info('lapack_opt')

    if not lapack_opt:
        raise NotFoundError,'no lapack/blas resources found'

    config = Configuration('arpack', parent_package, top_path)

    arpack_sources=[join('ARPACK','SRC', '*.f')]
    arpack_sources.extend([join('ARPACK','UTIL', '*.f')])
    arpack_sources.extend([join('ARPACK','LAPACK', '*.f')])

    if needs_veclib_wrapper(lapack_opt):
        arpack_sources += [pjoin('ARPACK', 'FWRAPPERS', 'veclib_cabi_f.f'),
                           pjoin('ARPACK', 'FWRAPPERS', 'veclib_cabi_c.c')]
    else:
        arpack_sources += [pjoin('ARPACK', 'FWRAPPERS', 'dummy.f')]

    config.add_library('arpack', sources=arpack_sources,
                       include_dirs=[join('ARPACK', 'SRC')])

    config.add_extension('_arpack',
                         sources='arpack.pyf.src',
                         libraries=['arpack'],
                         extra_info = lapack_opt
                        )

    config.add_data_dir('tests')
    return config

if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(**configuration(top_path='').todict())
<commit_msg>Another typo.<commit_after>#!/usr/bin/env python
import re
from os.path import join

def needs_veclib_wrapper(info):
    """Returns true if needs special veclib wrapper."""
    import re
    r_accel = re.compile("Accelerate")
    r_vec = re.compile("vecLib")
    res = False
    try:
        tmpstr = info['extra_link_args']
        for i in tmpstr:
            if r_accel.search(i) or r_vec.search(i):
                res = True
    except KeyError:
        pass

    return res

def configuration(parent_package='',top_path=None):
    from numpy.distutils.system_info import get_info, NotFoundError
    from numpy.distutils.misc_util import Configuration

    config = Configuration('arpack',parent_package,top_path)

    lapack_opt = get_info('lapack_opt')

    if not lapack_opt:
        raise NotFoundError,'no lapack/blas resources found'

    config = Configuration('arpack', parent_package, top_path)

    arpack_sources=[join('ARPACK','SRC', '*.f')]
    arpack_sources.extend([join('ARPACK','UTIL', '*.f')])
    arpack_sources.extend([join('ARPACK','LAPACK', '*.f')])

    if needs_veclib_wrapper(lapack_opt):
        arpack_sources += [join('ARPACK', 'FWRAPPERS', 'veclib_cabi_f.f'),
                           join('ARPACK', 'FWRAPPERS', 'veclib_cabi_c.c')]
    else:
        arpack_sources += [join('ARPACK', 'FWRAPPERS', 'dummy.f')]

    config.add_library('arpack', sources=arpack_sources,
                       include_dirs=[join('ARPACK', 'SRC')])

    config.add_extension('_arpack',
                         sources='arpack.pyf.src',
                         libraries=['arpack'],
                         extra_info = lapack_opt
                        )

    config.add_data_dir('tests')
    return config

if __name__ == '__main__':
    from numpy.distutils.core import setup
    setup(**configuration(top_path='').todict())
<|endoftext|>


<commit_before>import pygraphviz as pgv
import itertools
from copy import copy
from indra.statements import *
from indra.databases import uniprot_client

class Preassembler(object):

    def __init__(self, entity_hierarchy, mod_hierarchy, stmts=None):
        self.entity_hierarchy = entity_hierarchy
        self.mod_hierarchy = mod_hierarchy
        if stmts:
            self.stmts = stmts
        else:
            self.stmts = []
        self.unique_stmts = []
        self.related_stmts = []

    def add_statements(self, stmts):
        """Add to the current list of statements."""
        self.stmts += stmts

    def combine_duplicates(self):
        """Combine evidence from duplicate Statements.

        Statements are deemed to be duplicates if they have the same key
        returned by the matches_key() method of the Statement class.

        This function keeps the first instance of each set of duplicate
        statements and merges the evidence lists from all of the other
        statements.

        Returns a list of unique Statements and stores it in
        self.unique_stmts.
        """

        self.unique_stmts = []
        # Group statements according to whether they are matches (differing
        # only in their evidence).
        # Sort the statements in place by matches_key()
        self.stmts.sort(key=lambda x: x.matches_key())
        for key, duplicates in itertools.groupby(self.stmts,
                                                 key=lambda x: x.matches_key()):
            # Get the first statement and add the evidence of all subsequent
            # Statements to it
            for stmt_ix, stmt in enumerate(duplicates):
                if stmt_ix == 0:
                    first_stmt = stmt
                else:
                    first_stmt.evidence += stmt.evidence
            # This should never be None or anything else
            assert isinstance(first_stmt, Statement)
            self.unique_stmts.append(first_stmt)
        return self.unique_stmts

    def combine_related(self):
        """Connect related statements based on their refinement relationships.

        This function takes as a starting point the unique statements (with
        duplicates removed) and returns a modified flat list of statements
        containing only those statements which do not represent a refinement of
        other existing statements. In other words, the more general versions of
        a given statement do not appear at the top level, but instead are
        listed in the supports field of the top-level statements.

        If self.unique_stmts has not been initialized with the de-duplicated
        statements, self.combine_duplicates is called.

        The procedure for combining statements in this way involves a series
        of steps:

        1. The statements are grouped by whether they are of the same type
           (e.g., Phosphorylation) and involve the same entities (e.g.,
           BRAF and MAP2K1).
        2. The groups of statements are then further compared to see if one
           group involves superfamily entities of another group. If so, the
           statements involving the superfamily are added into the group of
           statements involving the more concrete entities to create an
           "extended group."
        3. The statements within each extended group are then compared; if one
           statement represents a refinement of the other (as defined by the
           refinement_of() method implemented for the Statement), then the more
           refined statement is added to the `supports` field of the more
           general statement, and the more general statement is added to the
           `supported_by` field of the more refined statement.
        4. A new flat list of statements is created that contains only those
           statements that have no `supports` entries (statements containing
           such entries are not eliminated, because they will be retrievable
           from the `supported_by` fields of other statements. This list
           is returned to the caller.

        .. note:: Subfamily relationships must be consistent across arguments

            For now, we will require that merges can only occur if the isa
            relationships are all in the same direction for all the agents in a
            Statement. For example, the two statement groups: RAF_family ->
            MEK1 and BRAF -> MEK_family would not be merged, since BRAF isa
            RAF_family, but MEK_family is not a MEK1. In the future this
            restriction could be revisited.

        Returns
        -------
        list of Statements.
            The returned list contains Statements representing the more
            concrete/refined versions of the Statements involving particular
            entities.

        Example
        -------
        A more general statement with no information about a Phosphorylation
        site is identified as supporting a more specific statement::

            >>> from indra.preassembler.hierarchy_manager import \
                    entity_hierarchy as eh, modification_hierarchy as mh
            >>> braf = Agent('BRAF')
            >>> map2k1 = Agent('MAP2K1')
            >>> st1 = Phosphorylation(braf, map2k1, 'Phosphorylation', None)
            >>> st2 = Phosphorylation(braf, map2k1, 'Phosphorylation', '218')
            >>> pa = Preassembler(eh, mh, [st1, st2])
            >>> combined_stmts = pa.combine_related()
            >>> combined_stmts
            [Phosphorylation(BRAF, MAP2K1, Phosphorylation, 218)]
            >>> combined_stmts[0].supported_by
            [Phosphorylation(BRAF, MAP2K1, Phosphorylation, None)]
            >>> combined_stmts[0].supported_by[0].supports
            [Phosphorylation(BRAF, MAP2K1, Phosphorylation, 218)]
        """

        # If unique_stmts is not initialized, call combine_duplicates.
        if not self.unique_stmts:
            self.combine_duplicates()
        # Group statements according to whether they have matching entities,
        # and store the resulting lists in a dict, indexed by the key defined
        # by the statement type and its entities:
        # Sort the statements in place by entities_match_key():
        self.unique_stmts.sort(key=lambda x: x.entities_match_key())
        groups = {grouper[0]: list(grouper[1])
                  for grouper in itertools.groupby(self.unique_stmts,
                                          key=lambda x: x.entities_match_key())}
        # The ext_groups dict is where we store the extended groups, those
        # statements which involve either the same entities or entities with
        # family relationships.
        ext_groups = copy(groups)
        # We examine pairs of Statement groups, looking for "isa" relationships:
        for g1_key, g2_key in itertools.permutations(groups.keys(), 2):
            g1 = groups[g1_key]
            g2 = groups[g2_key]
            # If we have two groups G1 and G2, each containing Statements with
            # some number of Agent arguments, e.g.  G1_Stmt(Ag1, Ag2, Ag3) and
            # G2_Stmt(Ag4, Ag5, Ag6), we need to know whether each of the
            # corresponding Agent arguments are related for all of the
            # Statements in the two groups.  That is, each of the arguments
            # between G1 and G2 must either be an entity match (as determined
            # by the entities_match test of the respective agents) or have an
            # "isa" relationship, as determined by the HierarchyManager in use.
            # If the elements in both groups are related, then the
            # Statements in the group with the superfamily relationship
            # are added into the group with the more specific entities. However,
            # the superfamily group is not removed from the group list, as
            # it may be a superfamily supporting Statements from another group.

            # Get the first statement from each group (we've already determined
            # that all of the statements within each group have an entity
            # match, so we only need the first:
            g1_stmt = g1[0]
            g2_stmt = g2[0]
            # Check that the statements are of the same type; if not, no merge.
            if type(g1_stmt) is not type(g2_stmt):
                continue
            # Check that all of the agents match or have an isa relationship.
            # Because the statements are of the same type, they should have the
            # same number of agents as arguments.  First, let's keep track of
            # our checks that g1 is the "primary" group, i.e., it is the one
            # with the more refined/grounded entities. We build a list with one
            # boolean entry for each argument, where a True value indicates
            # that the arguments at that position imply that g1 is the primary
            # group.
            agent_pairs = zip(g1_stmt.agent_list(), g2_stmt.agent_list())
            g1_is_refinement = [ag1.entity_matches(ag2) or
                                self.entity_hierarchy.isa(ag1.name, ag2.name)
                                for ag1, ag2 in agent_pairs]
            # If g1_is_refinement is all True values, that means everything in
            # the group1 statements isa thing in the group2 statements.
            if all(g1_is_refinement):
                g1_ext_list = ext_groups[g1_key]
                ext_groups[g1_key] = g1_ext_list + g2
            continue
        # At this point we have, in ext_groups, a dict of lists of Statements
        # indexed by their entity_matches key, but now the groups contain not
        # only statements with matching entities, but also entities related by
        # isa relationships. The next step is to process each group, checking
        # each statement against each other statement to determine supports and
        # supported_by relationships. This is determined by calling the
        # refinement_of method on pairs of statements.
        # Iterate over each of the extended groups:
        for ext_group in ext_groups.values():
            # Iterate over pairs of statements in the group:
            for stmt1, stmt2 in itertools.permutations(ext_group, 2):
                if stmt1.refinement_of(stmt2, self.entity_hierarchy,
                                       self.mod_hierarchy):
                    stmt1.supported_by.append(stmt2)
                    stmt2.supports.append(stmt1)
        # Now that the groups have been processed, we need to find the
        # non-subsumed statements, those that have no supports relationships.
        self.related_stmts = [stmt for ext_group in ext_groups.values()
                               for stmt in ext_group
                               if not stmt.supports]
        return self.related_stmts

    def check_sequence(self):
        """Iterate over all Statements and check whether references to
        residues and sequence positions are consistent with sequence
        information in the UniProt database"""
        # TODO: extend to all Agent modifications
        for st in self.stmts:
            if isinstance(st, Phosphorylation):
                # Skip statements with no specified position
                if st.mod_pos is None:
                    continue
                # Is looking at the first element enough?
                sub_id = st.sub.db_refs['UP']
                if not isinstance(sub_id, basestring):
                    sub_id = sub_id[0]
                sub = uniprot_client.query_protein(sub_id)
                if st.mod == 'PhosphorylationSerine':
                    residue = 'S'
                elif st.mod == 'PhosphorylationThreonine':
                    residue = 'T'
                elif st.mod == 'PhosphorylationTyrosine':
                    residue = 'Y'
                else:
                    continue
                ver = uniprot_client.verify_location(sub, residue,
                                                     location=st.mod_pos)
                if ver is False:
                    print '%s:' % st
                    print '-> Sequence check failed; position %s on %s is not %s.' %\
                          (st.mod_pos, st.sub.name, residue)

def render_stmt_graph(statements, agent_style=None):
    """Renders the supports/supported_by relationships of a set of statements
    and returns a pygraphviz graph.

    Example
    -------
    Pattern for getting statements and rendering as a Graphviz graph::

        bp = biopax_api.process_pc_pathsfromto(['BRAF'], ['MAP2K1'])
        bp.get_phosphorylation()

        pa = Preassembler(eh, mh, bp.statements)
        pa.combine_related()

        graph = render_stmt_graph(pa.related_stmts)
        graph.write('braf.dot')
        graph.draw('braf.pdf', prog='dot')
    """

    # Set the default agent formatting properties
    if agent_style is None:
        agent_style = {'color': 'lightgray', 'style': 'filled',
                       'fontname': 'arial'}
    # Sets to store all of the nodes and edges as we recursively process all
    # of the statements
    nodes = set([])
    edges = set([])
    # Recursive function for processing all statements
    def process_stmt(stmt):
        nodes.add(stmt)
        for sby_ix, sby_stmt in enumerate(stmt.supported_by):
            edges.add((str(stmt.matches_key()), str(sby_stmt.matches_key())))
            process_stmt(sby_stmt)
    # Process all of the top-level statements, getting the supporting statements
    # recursively
    for stmt in statements:
        process_stmt(stmt)
    # Add the nodes and edges to the graph
    graph = pgv.AGraph(name='statements', directed=True)
    for node in nodes:
        graph.add_node(str(node.matches_key()), label=str(node), **agent_style)
    graph.add_edges_from(edges)
    return graph

<commit_msg>Change check statement loop structure in Preassembler<commit_after>import pygraphviz as pgv
import itertools
from copy import copy
from indra.statements import *
from indra.databases import uniprot_client

class Preassembler(object):

    def __init__(self, entity_hierarchy, mod_hierarchy, stmts=None):
        self.entity_hierarchy = entity_hierarchy
        self.mod_hierarchy = mod_hierarchy
        if stmts:
            self.stmts = stmts
        else:
            self.stmts = []
        self.unique_stmts = []
        self.related_stmts = []

    def add_statements(self, stmts):
        """Add to the current list of statements."""
        self.stmts += stmts

    def combine_duplicates(self):
        """Combine evidence from duplicate Statements.

        Statements are deemed to be duplicates if they have the same key
        returned by the matches_key() method of the Statement class.

        This function keeps the first instance of each set of duplicate
        statements and merges the evidence lists from all of the other
        statements.

        Returns a list of unique Statements and stores it in
        self.unique_stmts.
        """

        self.unique_stmts = []
        # Group statements according to whether they are matches (differing
        # only in their evidence).
        # Sort the statements in place by matches_key()
        self.stmts.sort(key=lambda x: x.matches_key())
        for key, duplicates in itertools.groupby(self.stmts,
                                                 key=lambda x: x.matches_key()):
            # Get the first statement and add the evidence of all subsequent
            # Statements to it
            for stmt_ix, stmt in enumerate(duplicates):
                if stmt_ix == 0:
                    first_stmt = stmt
                else:
                    first_stmt.evidence += stmt.evidence
            # This should never be None or anything else
            assert isinstance(first_stmt, Statement)
            self.unique_stmts.append(first_stmt)
        return self.unique_stmts

    def combine_related(self):
        """Connect related statements based on their refinement relationships.

        This function takes as a starting point the unique statements (with
        duplicates removed) and returns a modified flat list of statements
        containing only those statements which do not represent a refinement of
        other existing statements. In other words, the more general versions of
        a given statement do not appear at the top level, but instead are
        listed in the supports field of the top-level statements.

        If self.unique_stmts has not been initialized with the de-duplicated
        statements, self.combine_duplicates is called.

        The procedure for combining statements in this way involves a series
        of steps:

        1. The statements are grouped by whether they are of the same type
           (e.g., Phosphorylation) and involve the same entities (e.g.,
           BRAF and MAP2K1).
        2. The groups of statements are then further compared to see if one
           group involves superfamily entities of another group. If so, the
           statements involving the superfamily are added into the group of
           statements involving the more concrete entities to create an
           "extended group."
        3. The statements within each extended group are then compared; if one
           statement represents a refinement of the other (as defined by the
           refinement_of() method implemented for the Statement), then the more
           refined statement is added to the `supports` field of the more
           general statement, and the more general statement is added to the
           `supported_by` field of the more refined statement.
        4. A new flat list of statements is created that contains only those
           statements that have no `supports` entries (statements containing
           such entries are not eliminated, because they will be retrievable
           from the `supported_by` fields of other statements. This list
           is returned to the caller.

        .. note:: Subfamily relationships must be consistent across arguments

            For now, we will require that merges can only occur if the isa
            relationships are all in the same direction for all the agents in a
            Statement. For example, the two statement groups: RAF_family ->
            MEK1 and BRAF -> MEK_family would not be merged, since BRAF isa
            RAF_family, but MEK_family is not a MEK1. In the future this
            restriction could be revisited.

        Returns
        -------
        list of Statements.
            The returned list contains Statements representing the more
            concrete/refined versions of the Statements involving particular
            entities.

        Example
        -------
        A more general statement with no information about a Phosphorylation
        site is identified as supporting a more specific statement::

            >>> from indra.preassembler.hierarchy_manager import \
                    entity_hierarchy as eh, modification_hierarchy as mh
            >>> braf = Agent('BRAF')
            >>> map2k1 = Agent('MAP2K1')
            >>> st1 = Phosphorylation(braf, map2k1, 'Phosphorylation', None)
            >>> st2 = Phosphorylation(braf, map2k1, 'Phosphorylation', '218')
            >>> pa = Preassembler(eh, mh, [st1, st2])
            >>> combined_stmts = pa.combine_related()
            >>> combined_stmts
            [Phosphorylation(BRAF, MAP2K1, Phosphorylation, 218)]
            >>> combined_stmts[0].supported_by
            [Phosphorylation(BRAF, MAP2K1, Phosphorylation, None)]
            >>> combined_stmts[0].supported_by[0].supports
            [Phosphorylation(BRAF, MAP2K1, Phosphorylation, 218)]
        """

        # If unique_stmts is not initialized, call combine_duplicates.
        if not self.unique_stmts:
            self.combine_duplicates()
        # Group statements according to whether they have matching entities,
        # and store the resulting lists in a dict, indexed by the key defined
        # by the statement type and its entities:
        # Sort the statements in place by entities_match_key():
        self.unique_stmts.sort(key=lambda x: x.entities_match_key())
        groups = {grouper[0]: list(grouper[1])
                  for grouper in itertools.groupby(self.unique_stmts,
                                          key=lambda x: x.entities_match_key())}
        # The ext_groups dict is where we store the extended groups, those
        # statements which involve either the same entities or entities with
        # family relationships.
        ext_groups = copy(groups)
        # We examine pairs of Statement groups, looking for "isa" relationships:
        for g1_key, g2_key in itertools.permutations(groups.keys(), 2):
            g1 = groups[g1_key]
            g2 = groups[g2_key]
            # If we have two groups G1 and G2, each containing Statements with
            # some number of Agent arguments, e.g.  G1_Stmt(Ag1, Ag2, Ag3) and
            # G2_Stmt(Ag4, Ag5, Ag6), we need to know whether each of the
            # corresponding Agent arguments are related for all of the
            # Statements in the two groups.  That is, each of the arguments
            # between G1 and G2 must either be an entity match (as determined
            # by the entities_match test of the respective agents) or have an
            # "isa" relationship, as determined by the HierarchyManager in use.
            # If the elements in both groups are related, then the
            # Statements in the group with the superfamily relationship
            # are added into the group with the more specific entities. However,
            # the superfamily group is not removed from the group list, as
            # it may be a superfamily supporting Statements from another group.

            # Get the first statement from each group (we've already determined
            # that all of the statements within each group have an entity
            # match, so we only need the first:
            g1_stmt = g1[0]
            g2_stmt = g2[0]
            # Check that the statements are of the same type; if not, no merge.
            if type(g1_stmt) is not type(g2_stmt):
                continue
            # Check that all of the agents match or have an isa relationship.
            # Because the statements are of the same type, they should have the
            # same number of agents as arguments.  First, let's keep track of
            # our checks that g1 is the "primary" group, i.e., it is the one
            # with the more refined/grounded entities. We build a list with one
            # boolean entry for each argument, where a True value indicates
            # that the arguments at that position imply that g1 is the primary
            # group.
            agent_pairs = zip(g1_stmt.agent_list(), g2_stmt.agent_list())
            g1_is_refinement = [ag1.entity_matches(ag2) or
                                self.entity_hierarchy.isa(ag1.name, ag2.name)
                                for ag1, ag2 in agent_pairs]
            # If g1_is_refinement is all True values, that means everything in
            # the group1 statements isa thing in the group2 statements.
            if all(g1_is_refinement):
                g1_ext_list = ext_groups[g1_key]
                ext_groups[g1_key] = g1_ext_list + g2
            continue
        # At this point we have, in ext_groups, a dict of lists of Statements
        # indexed by their entity_matches key, but now the groups contain not
        # only statements with matching entities, but also entities related by
        # isa relationships. The next step is to process each group, checking
        # each statement against each other statement to determine supports and
        # supported_by relationships. This is determined by calling the
        # refinement_of method on pairs of statements.
        # Iterate over each of the extended groups:
        for ext_group in ext_groups.values():
            # Iterate over pairs of statements in the group:
            for stmt1, stmt2 in itertools.permutations(ext_group, 2):
                if stmt1.refinement_of(stmt2, self.entity_hierarchy,
                                       self.mod_hierarchy):
                    stmt1.supported_by.append(stmt2)
                    stmt2.supports.append(stmt1)
        # Now that the groups have been processed, we need to find the
        # non-subsumed statements, those that have no supports relationships.
        self.related_stmts = [stmt for ext_group in ext_groups.values()
                               for stmt in ext_group
                               if not stmt.supports]
        return self.related_stmts

    def check_statements(self):
        for st in self.stmts:
            print st
            self.check_sequence(st)

    @staticmethod
    def check_sequence(st):
        """Check whether references to
        residues and sequence positions are consistent with sequence
        information in the UniProt database"""
        # TODO: extend to all Agent modifications
        if isinstance(st, Phosphorylation):
            # Skip statements with no specified position
            if st.mod_pos is None:
                return
            # Is looking at the first element enough?
            sub_id = st.sub.db_refs['UP']
            if not isinstance(sub_id, basestring):
                sub_id = sub_id[0]
            sub = uniprot_client.query_protein(sub_id)
            if st.mod == 'PhosphorylationSerine':
                residue = 'S'
            elif st.mod == 'PhosphorylationThreonine':
                residue = 'T'
            elif st.mod == 'PhosphorylationTyrosine':
                residue = 'Y'
            else:
                return
            ver = uniprot_client.verify_location(sub, residue,
                                                 location=st.mod_pos)
            if ver is False:
                print '-> Sequence check failed; position %s on %s is not %s.' %\
                      (st.mod_pos, st.sub.name, residue)

def render_stmt_graph(statements, agent_style=None):
    """Renders the supports/supported_by relationships of a set of statements
    and returns a pygraphviz graph.

    Example
    -------
    Pattern for getting statements and rendering as a Graphviz graph::

        bp = biopax_api.process_pc_pathsfromto(['BRAF'], ['MAP2K1'])
        bp.get_phosphorylation()

        pa = Preassembler(eh, mh, bp.statements)
        pa.combine_related()

        graph = render_stmt_graph(pa.related_stmts)
        graph.write('braf.dot')
        graph.draw('braf.pdf', prog='dot')
    """

    # Set the default agent formatting properties
    if agent_style is None:
        agent_style = {'color': 'lightgray', 'style': 'filled',
                       'fontname': 'arial'}
    # Sets to store all of the nodes and edges as we recursively process all
    # of the statements
    nodes = set([])
    edges = set([])
    # Recursive function for processing all statements
    def process_stmt(stmt):
        nodes.add(stmt)
        for sby_ix, sby_stmt in enumerate(stmt.supported_by):
            edges.add((str(stmt.matches_key()), str(sby_stmt.matches_key())))
            process_stmt(sby_stmt)
    # Process all of the top-level statements, getting the supporting statements
    # recursively
    for stmt in statements:
        process_stmt(stmt)
    # Add the nodes and edges to the graph
    graph = pgv.AGraph(name='statements', directed=True)
    for node in nodes:
        graph.add_node(str(node.matches_key()), label=str(node), **agent_style)
    graph.add_edges_from(edges)
    return graph

<|endoftext|>


<commit_before>from __future__ import unicode_literals

from scrapi.base import OAIHarvester

valposcholar = OAIHarvester(
    name='valposcholar',
    base_url='http://scholar.valpo.edu/do/oai/',
    property_list=['type', 'source'],
    approved_sets=True
)

consume = valposcholar.harvest
normalize = valposcholar.normalize
<commit_msg>add list of approved sets<commit_after>from __future__ import unicode_literals

from scrapi.base import OAIHarvester

valposcholar = OAIHarvester(
    name='valposcholar',
    base_url='http://scholar.valpo.edu/do/oai/',
    property_list=['type', 'source'],
    approved_sets=[
        'cc_fac_pub',
        'it_pubs',
        'ccls_fac_pub',
        'journaloftolkienresearch',
        'art_fac_pub',
        'bio_fac_pub',
        'chem_fac_pub',
        'comm_fac_pubs',
        'econ_fac_pub',
        'ed_fac_pubs',
        'eng_fac_pub', 'vfr',
        'german_fac_pub',
        'spanish_fac_pub',
        'geomet_fac_pub',
        'history_fac_pub',
        'kin_fac_pubs',
        'mcs_fac_pubs',
        'phys_astro_fac_pub',
        'poli_sci_fac_pubs',
        'psych_fac_pub',
        'sociology_fac_pub',
        'theatre_fac_pubs',
        'theo_fac_pubs',
        'cba_fac_pub',
        'jvbl',
        'engineering_fac_pub',
        'ebpr',
        'msn_theses',
        'nursing_fac_pubs',
        'grad_student_pubs',
        'vulr',
        'law_fac_pubs',
        'ils_papers'
    ]
)

consume = valposcholar.harvest
normalize = valposcholar.normalize
''
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
from indra.preassembler.sitemapper import default_mapper as sm, MappedStatement
from indra.statements import *
from indra.util import unicode_strs

def test_check_agent_mod():
    mapk1_valid = Agent('MAPK1',
                        mods=[ModCondition('phosphorylation', 'T', '185'),
                              ModCondition('phosphorylation', 'Y', '187')],
                        db_refs={'UP': 'P28482'})
    res_valid = sm._map_agent_sites(mapk1_valid)
    assert len(res_valid) == 2
    assert res_valid[0] == []
    assert isinstance(res_valid[1], Agent)
    assert res_valid[1].matches(mapk1_valid)

    mapk1_invalid = Agent('MAPK1',
                          mods=[ModCondition('phosphorylation', 'T', '183'),
                                ModCondition('phosphorylation', 'Y', '185')],
                          db_refs={'UP': 'P28482'})
    res_invalid = sm._map_agent_sites(mapk1_invalid)
    assert len(res_invalid) == 2
    assert isinstance(res_invalid[0], list)
    assert isinstance(res_invalid[1], Agent)
    invalid_sites = res_invalid[0]
    assert len(invalid_sites) == 2
    map183 = invalid_sites[0]
    assert map183[0] == ('MAPK1', 'T', '183')
    assert len(map183[1]) == 3
    assert map183[1][0] == 'T'
    assert map183[1][1] == '185'
    map185 = invalid_sites[1]
    assert map185[0] == ('MAPK1', 'Y', '185')
    assert len(map185[1]) == 3
    assert map185[1][0] == 'Y'
    assert map185[1][1] == '187'
    new_agent = res_invalid[1]
    assert len(new_agent.mods) == 2
    assert new_agent.mods[0].matches(ModCondition('phosphorylation',
                                                  'T', '185'))
    assert new_agent.mods[1].matches(ModCondition('phosphorylation',
                                                  'Y', '187'))
    assert unicode_strs((mapk1_valid, res_valid, mapk1_invalid, res_invalid,
                         invalid_sites, map183, map185, new_agent))

def test_site_map_modification():
    mapk1_invalid = Agent('MAPK1',
                          mods=[ModCondition('phosphorylation', 'T', '183'),
                                ModCondition('phosphorylation', 'Y', '185')],
                          db_refs={'UP': 'P28482'})
    mapk3_invalid = Agent('MAPK3',
                          mods=[ModCondition('phosphorylation', 'T', '201')],
                          db_refs={'UP': 'P27361'})
    map2k1_invalid = Agent('MAP2K1',
                           mods=[ModCondition('phosphorylation', 'S', '217'),
                                 ModCondition('phosphorylation', 'S', '221')],
                           db_refs={'UP': 'Q02750'})

    st1 = Phosphorylation(mapk1_invalid, mapk3_invalid, 'Y', '203')
    st2 = Phosphorylation(map2k1_invalid, mapk1_invalid, 'Y', '218')
    res = sm.map_sites([st1, st2])

    assert len(res) == 2
    valid_stmts = res[0]
    mapped_stmts = res[1]
    assert isinstance(valid_stmts, list)
    assert isinstance(mapped_stmts, list)
    assert len(valid_stmts) == 0
    assert len(mapped_stmts) == 2
    # MAPK1 -> MAPK3
    mapped_stmt1 = mapped_stmts[0]
    assert isinstance(mapped_stmt1, MappedStatement)
    assert mapped_stmt1.original_stmt == st1
    assert isinstance(mapped_stmt1.mapped_mods, list)
    assert len(mapped_stmt1.mapped_mods) == 4, \
        "Got %d mapped mods." % mapped_stmt1.mapped_mods  # FIXME
    ms = mapped_stmt1.mapped_stmt
    assert isinstance(ms, Statement)
    agent1 = ms.enz
    agent2 = ms.sub
    assert agent1.name == 'MAPK1'
    assert len(agent1.mods) == 2
    assert agent1.mods[0].matches(ModCondition('phosphorylation', 'T', '185'))
    assert agent1.mods[1].matches(ModCondition('phosphorylation', 'Y', '187'))
    assert agent2.mods[0].matches(ModCondition('phosphorylation', 'T', '202'))
    assert ms.residue == 'Y'
    assert ms.position == '204'

    # MAP2K1 -> MAPK1
    mapped_stmt2 = mapped_stmts[1]
    assert isinstance(mapped_stmt2, MappedStatement)
    assert mapped_stmt2.original_stmt == st2
    assert isinstance(mapped_stmt2.mapped_mods, list)
    assert len(mapped_stmt2.mapped_mods) == 5, \
        "Got %d mapped mods." % mapped_stmt1.mapped_mods  # FIXME
    ms = mapped_stmt2.mapped_stmt
    assert isinstance(ms, Statement)
    agent1 = ms.enz
    agent2 = ms.sub
    assert agent1.name == 'MAP2K1'
    assert len(agent1.mods) == 2
    assert agent1.mods[0].matches(ModCondition('phosphorylation', 'S', '218'))
    assert agent1.mods[1].matches(ModCondition('phosphorylation', 'S', '222'))
    assert len(agent2.mods) == 2
    assert agent2.mods[0].matches(ModCondition('phosphorylation', 'T', '185'))
    assert agent2.mods[1].matches(ModCondition('phosphorylation', 'Y', '187'))
    # The incorrect phosphorylation residue is passed through to the new
    # statement unchanged
    assert ms.residue == 'Y'
    assert ms.position == '218'
    # Check for unicode
    assert unicode_strs((mapk1_invalid, mapk3_invalid, map2k1_invalid, st1,
                         st2, res, valid_stmts, mapped_stmts))

def test_site_map_activity_modification():
    mc = [ModCondition('phosphorylation', 'T', '183'),
          ModCondition('phosphorylation', 'Y', '185')]
    mapk1 = Agent('MAPK1', mods=mc, db_refs={'UP': 'P28482'})

    st1 = ActiveForm(mapk1, 'kinase', True)
    (valid, mapped) = sm.map_sites([st1])
    assert len(valid) == 0
    assert len(mapped) == 1
    ms = mapped[0]
    assert ms.mapped_mods[0][0] == ('MAPK1', 'T', '183')
    assert ms.mapped_mods[0][1][0] == 'T'
    assert ms.mapped_mods[0][1][1] == '185'
    assert ms.mapped_mods[1][0] == ('MAPK1', 'Y', '185')
    assert ms.mapped_mods[1][1][0] == 'Y'
    assert ms.mapped_mods[1][1][1] == '187'
    assert ms.original_stmt == st1
    assert ms.mapped_stmt.agent.mods[0].matches(ModCondition('phosphorylation',
                                                             'T', '185'))
    assert ms.mapped_stmt.agent.mods[1].matches(ModCondition('phosphorylation',
                                                             'Y', '187'))
    assert unicode_strs((mc, mapk1, st1, valid, mapped))

def test_site_map_selfmodification():
    mapk1_invalid = Agent('MAPK1',
                          mods=[ModCondition('phosphorylation', 'T', '183')],
                          db_refs={'UP': 'P28482'})
    st1 = SelfModification(mapk1_invalid, 'Y', '185')
    (valid, mapped) = sm.map_sites([st1])
    assert len(valid) == 0
    assert len(mapped) == 1
    mapped_stmt = mapped[0]
    assert mapped_stmt.mapped_mods[0][0] == ('MAPK1', 'T', '183')
    assert mapped_stmt.mapped_mods[0][1][0] == 'T'
    assert mapped_stmt.mapped_mods[0][1][1] == '185'
    assert mapped_stmt.mapped_mods[1][0] == ('MAPK1', 'Y', '185')
    assert mapped_stmt.mapped_mods[1][1][0] == 'Y'
    assert mapped_stmt.mapped_mods[1][1][1] == '187'
    assert mapped_stmt.original_stmt == st1
    ms = mapped_stmt.mapped_stmt
    agent1 = ms.enz
    assert agent1.mods[0].matches(ModCondition('phosphorylation', 'T', '185'))
    assert ms.residue == 'Y'
    assert ms.position == '187'
    assert unicode_strs((mapk1_invalid, st1, valid, mapped))

# The following Statements are all handled by the same block of code and hence
# can be tested in similar fashion

def test_site_map_complex():
    (mapk1_invalid, mapk3_invalid) = get_invalid_mapks()
    st1 = Complex([mapk1_invalid, mapk3_invalid])
    res = sm.map_sites([st1])
    check_validated_mapks(res, st1)

def test_site_map_gef():
    (mapk1_invalid, mapk3_invalid) = get_invalid_mapks()
    st1 = Gef(mapk1_invalid, mapk3_invalid)
    res = sm.map_sites([st1])
    check_validated_mapks(res, st1)

def test_site_map_gap():
    (mapk1_invalid, mapk3_invalid) = get_invalid_mapks()
    st1 = Gap(mapk1_invalid, mapk3_invalid)
    res = sm.map_sites([st1])
    check_validated_mapks(res, st1)

def test_site_map_activation():
    (mapk1_invalid, mapk3_invalid) = get_invalid_mapks()
    st1 = Activation(mapk1_invalid, mapk3_invalid, 'kinase')
    res = sm.map_sites([st1])
    check_validated_mapks(res, st1)

def test_site_map_hgnc():
    """Make sure site mapping is done even if only HGNC ID is given."""
    (mapk1_invalid, mapk3_invalid) = get_invalid_mapks()
    mapk1_invalid.db_refs = {'HGNC': '6871'}
    st1 = ActiveForm(mapk1_invalid, 'kinase', True)
    (valid, mapped) = sm.map_sites([st1])
    assert len(valid) == 0
    assert len(mapped) == 1

def test_site_map_within_bound_condition():
    # Here, we test to make sure that agents within a bound condition are
    # site-mapped
    (mapk1_invalid, mapk3_invalid) = get_invalid_mapks()

    # Add an agent to the bound condition for the object of the statement
    mapk3_invalid.bound_conditions = [BoundCondition(mapk1_invalid)]
    st1 = Activation(mapk1_invalid, mapk3_invalid, 'kinase')

    # Map sites
    res = sm.map_sites([st1])

    # Extract the mapped statement
    mapped_statements = res[1]
    assert(len(mapped_statements) == 1)
    mapped_s = mapped_statements[0].mapped_stmt

    # Verify that the agent in the object's bound condition got site-mapped
    validate_mapk1(mapped_s.obj.bound_conditions[0].agent)

def get_invalid_mapks():
    """A handy function for getting the invalid MAPK agents we want."""
    mapk1_invalid = Agent('MAPK1',
                          mods=[ModCondition('phosphorylation', 'T', '183'),
                                ModCondition('phosphorylation', 'Y', '185')],
                          db_refs={'UP': 'P28482'})
    mapk3_invalid = Agent('MAPK3',
                          mods=[ModCondition('phosphorylation', 'T', '201'),
                                ModCondition('phosphorylation', 'Y', '203')],
                          db_refs={'UP': 'P27361'})
    assert unicode_strs((mapk1_invalid, mapk3_invalid))
    return (mapk1_invalid, mapk3_invalid)

def check_validated_mapks(res, st1):
    """Validate that the invalid MAPKs have been fixed appropriately."""
    assert len(res) == 2
    valid_stmts = res[0]
    mapped_stmts = res[1]
    assert isinstance(valid_stmts, list)
    assert isinstance(mapped_stmts, list)
    assert len(valid_stmts) == 0
    assert len(mapped_stmts) == 1
    mapped_stmt = mapped_stmts[0]
    assert isinstance(mapped_stmt, MappedStatement)
    assert mapped_stmt.original_stmt == st1
    assert isinstance(mapped_stmt.mapped_mods, list)
    assert len(mapped_stmt.mapped_mods) == 4
    ms = mapped_stmt.mapped_stmt
    assert isinstance(ms, Statement)
    agents = ms.agent_list()
    assert len(agents) == 2
    agent1 = agents[0]
    agent2 = agents[1]
    validate_mapk1(agent1)
    assert agent2.mods[0].matches(ModCondition('phosphorylation', 'T', '202'))
    assert agent2.mods[1].matches(ModCondition('phosphorylation', 'Y', '204'))
    assert unicode_strs((res, st1))

def validate_mapk1(agent1):
    assert agent1.name == 'MAPK1'
    assert len(agent1.mods) == 2
    assert agent1.mods[0].matches(ModCondition('phosphorylation', 'T', '185'))
    assert agent1.mods[1].matches(ModCondition('phosphorylation', 'Y', '187'))
<commit_msg>Add failing test<commit_after>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
from indra.preassembler.sitemapper import default_mapper as sm, MappedStatement
from indra.statements import *
from indra.util import unicode_strs

def test_check_agent_mod():
    mapk1_valid = Agent('MAPK1',
                        mods=[ModCondition('phosphorylation', 'T', '185'),
                              ModCondition('phosphorylation', 'Y', '187')],
                        db_refs={'UP': 'P28482'})
    res_valid = sm._map_agent_sites(mapk1_valid)
    assert len(res_valid) == 2
    assert res_valid[0] == []
    assert isinstance(res_valid[1], Agent)
    assert res_valid[1].matches(mapk1_valid)

    mapk1_invalid = Agent('MAPK1',
                          mods=[ModCondition('phosphorylation', 'T', '183'),
                                ModCondition('phosphorylation', 'Y', '185')],
                          db_refs={'UP': 'P28482'})
    res_invalid = sm._map_agent_sites(mapk1_invalid)
    assert len(res_invalid) == 2
    assert isinstance(res_invalid[0], list)
    assert isinstance(res_invalid[1], Agent)
    invalid_sites = res_invalid[0]
    assert len(invalid_sites) == 2
    map183 = invalid_sites[0]
    assert map183[0] == ('MAPK1', 'T', '183')
    assert len(map183[1]) == 3
    assert map183[1][0] == 'T'
    assert map183[1][1] == '185'
    map185 = invalid_sites[1]
    assert map185[0] == ('MAPK1', 'Y', '185')
    assert len(map185[1]) == 3
    assert map185[1][0] == 'Y'
    assert map185[1][1] == '187'
    new_agent = res_invalid[1]
    assert len(new_agent.mods) == 2
    assert new_agent.mods[0].matches(ModCondition('phosphorylation',
                                                  'T', '185'))
    assert new_agent.mods[1].matches(ModCondition('phosphorylation',
                                                  'Y', '187'))
    assert unicode_strs((mapk1_valid, res_valid, mapk1_invalid, res_invalid,
                         invalid_sites, map183, map185, new_agent))

def test_site_map_modification():
    mapk1_invalid = Agent('MAPK1',
                          mods=[ModCondition('phosphorylation', 'T', '183'),
                                ModCondition('phosphorylation', 'Y', '185')],
                          db_refs={'UP': 'P28482'})
    mapk3_invalid = Agent('MAPK3',
                          mods=[ModCondition('phosphorylation', 'T', '201')],
                          db_refs={'UP': 'P27361'})
    map2k1_invalid = Agent('MAP2K1',
                           mods=[ModCondition('phosphorylation', 'S', '217'),
                                 ModCondition('phosphorylation', 'S', '221')],
                           db_refs={'UP': 'Q02750'})

    st1 = Phosphorylation(mapk1_invalid, mapk3_invalid, 'Y', '203')
    st2 = Phosphorylation(map2k1_invalid, mapk1_invalid, 'Y', '218')
    res = sm.map_sites([st1, st2])

    assert len(res) == 2
    valid_stmts = res[0]
    mapped_stmts = res[1]
    assert isinstance(valid_stmts, list)
    assert isinstance(mapped_stmts, list)
    assert len(valid_stmts) == 0
    assert len(mapped_stmts) == 2
    # MAPK1 -> MAPK3
    mapped_stmt1 = mapped_stmts[0]
    assert isinstance(mapped_stmt1, MappedStatement)
    assert mapped_stmt1.original_stmt == st1
    assert isinstance(mapped_stmt1.mapped_mods, list)
    assert len(mapped_stmt1.mapped_mods) == 4, \
        "Got %d mapped mods." % mapped_stmt1.mapped_mods  # FIXME
    ms = mapped_stmt1.mapped_stmt
    assert isinstance(ms, Statement)
    agent1 = ms.enz
    agent2 = ms.sub
    assert agent1.name == 'MAPK1'
    assert len(agent1.mods) == 2
    assert agent1.mods[0].matches(ModCondition('phosphorylation', 'T', '185'))
    assert agent1.mods[1].matches(ModCondition('phosphorylation', 'Y', '187'))
    assert agent2.mods[0].matches(ModCondition('phosphorylation', 'T', '202'))
    assert ms.residue == 'Y'
    assert ms.position == '204'

    # MAP2K1 -> MAPK1
    mapped_stmt2 = mapped_stmts[1]
    assert isinstance(mapped_stmt2, MappedStatement)
    assert mapped_stmt2.original_stmt == st2
    assert isinstance(mapped_stmt2.mapped_mods, list)
    assert len(mapped_stmt2.mapped_mods) == 5, \
        "Got %d mapped mods." % mapped_stmt1.mapped_mods  # FIXME
    ms = mapped_stmt2.mapped_stmt
    assert isinstance(ms, Statement)
    agent1 = ms.enz
    agent2 = ms.sub
    assert agent1.name == 'MAP2K1'
    assert len(agent1.mods) == 2
    assert agent1.mods[0].matches(ModCondition('phosphorylation', 'S', '218'))
    assert agent1.mods[1].matches(ModCondition('phosphorylation', 'S', '222'))
    assert len(agent2.mods) == 2
    assert agent2.mods[0].matches(ModCondition('phosphorylation', 'T', '185'))
    assert agent2.mods[1].matches(ModCondition('phosphorylation', 'Y', '187'))
    # The incorrect phosphorylation residue is passed through to the new
    # statement unchanged
    assert ms.residue == 'Y'
    assert ms.position == '218'
    # Check for unicode
    assert unicode_strs((mapk1_invalid, mapk3_invalid, map2k1_invalid, st1,
                         st2, res, valid_stmts, mapped_stmts))

def test_site_map_activity_modification():
    mc = [ModCondition('phosphorylation', 'T', '183'),
          ModCondition('phosphorylation', 'Y', '185')]
    mapk1 = Agent('MAPK1', mods=mc, db_refs={'UP': 'P28482'})

    st1 = ActiveForm(mapk1, 'kinase', True)
    (valid, mapped) = sm.map_sites([st1])
    assert len(valid) == 0
    assert len(mapped) == 1
    ms = mapped[0]
    assert ms.mapped_mods[0][0] == ('MAPK1', 'T', '183')
    assert ms.mapped_mods[0][1][0] == 'T'
    assert ms.mapped_mods[0][1][1] == '185'
    assert ms.mapped_mods[1][0] == ('MAPK1', 'Y', '185')
    assert ms.mapped_mods[1][1][0] == 'Y'
    assert ms.mapped_mods[1][1][1] == '187'
    assert ms.original_stmt == st1
    assert ms.mapped_stmt.agent.mods[0].matches(ModCondition('phosphorylation',
                                                             'T', '185'))
    assert ms.mapped_stmt.agent.mods[1].matches(ModCondition('phosphorylation',
                                                             'Y', '187'))
    assert unicode_strs((mc, mapk1, st1, valid, mapped))

def test_site_map_selfmodification():
    mapk1_invalid = Agent('MAPK1',
                          mods=[ModCondition('phosphorylation', 'T', '183')],
                          db_refs={'UP': 'P28482'})
    st1 = SelfModification(mapk1_invalid, 'Y', '185')
    (valid, mapped) = sm.map_sites([st1])
    assert len(valid) == 0
    assert len(mapped) == 1
    mapped_stmt = mapped[0]
    assert mapped_stmt.mapped_mods[0][0] == ('MAPK1', 'T', '183')
    assert mapped_stmt.mapped_mods[0][1][0] == 'T'
    assert mapped_stmt.mapped_mods[0][1][1] == '185'
    assert mapped_stmt.mapped_mods[1][0] == ('MAPK1', 'Y', '185')
    assert mapped_stmt.mapped_mods[1][1][0] == 'Y'
    assert mapped_stmt.mapped_mods[1][1][1] == '187'
    assert mapped_stmt.original_stmt == st1
    ms = mapped_stmt.mapped_stmt
    agent1 = ms.enz
    assert agent1.mods[0].matches(ModCondition('phosphorylation', 'T', '185'))
    assert ms.residue == 'Y'
    assert ms.position == '187'
    assert unicode_strs((mapk1_invalid, st1, valid, mapped))

# The following Statements are all handled by the same block of code and hence
# can be tested in similar fashion

def test_site_map_complex():
    (mapk1_invalid, mapk3_invalid) = get_invalid_mapks()
    st1 = Complex([mapk1_invalid, mapk3_invalid])
    res = sm.map_sites([st1])
    check_validated_mapks(res, st1)

def test_site_map_gef():
    (mapk1_invalid, mapk3_invalid) = get_invalid_mapks()
    st1 = Gef(mapk1_invalid, mapk3_invalid)
    res = sm.map_sites([st1])
    check_validated_mapks(res, st1)

def test_site_map_gap():
    (mapk1_invalid, mapk3_invalid) = get_invalid_mapks()
    st1 = Gap(mapk1_invalid, mapk3_invalid)
    res = sm.map_sites([st1])
    check_validated_mapks(res, st1)

def test_site_map_activation():
    (mapk1_invalid, mapk3_invalid) = get_invalid_mapks()
    st1 = Activation(mapk1_invalid, mapk3_invalid, 'kinase')
    res = sm.map_sites([st1])
    check_validated_mapks(res, st1)

def test_site_map_hgnc():
    """Make sure site mapping is done even if only HGNC ID is given."""
    (mapk1_invalid, mapk3_invalid) = get_invalid_mapks()
    mapk1_invalid.db_refs = {'HGNC': '6871'}
    st1 = ActiveForm(mapk1_invalid, 'kinase', True)
    (valid, mapped) = sm.map_sites([st1])
    assert len(valid) == 0
    assert len(mapped) == 1

def test_ignore_blank_entries():
    """Make sure sites curated as erroneous with no mappings don't appear
    to be valid mappings by setting the mapped_mods entry in the
    MappedStatement to None."""
    mapk1 = Agent('MAPK1', db_refs={'UP': 'P28482'})
    rps6 = Agent('RPS6', db_refs={'UP': 'P62753'})
    st1 = Phosphorylation(mapk1, rps6, 'T', '389')
    (valid, mapped) = sm.map_sites([st1])
    assert len(valid) == 0
    assert len(mapped) == 1
    ms = mapped[0]
    assert ms.mapped_mods[0][1] is None

def test_site_map_within_bound_condition():
    # Here, we test to make sure that agents within a bound condition are
    # site-mapped
    (mapk1_invalid, mapk3_invalid) = get_invalid_mapks()

    # Add an agent to the bound condition for the object of the statement
    mapk3_invalid.bound_conditions = [BoundCondition(mapk1_invalid)]
    st1 = Activation(mapk1_invalid, mapk3_invalid, 'kinase')

    # Map sites
    res = sm.map_sites([st1])

    # Extract the mapped statement
    mapped_statements = res[1]
    assert(len(mapped_statements) == 1)
    mapped_s = mapped_statements[0].mapped_stmt

    # Verify that the agent in the object's bound condition got site-mapped
    validate_mapk1(mapped_s.obj.bound_conditions[0].agent)

def get_invalid_mapks():
    """A handy function for getting the invalid MAPK agents we want."""
    mapk1_invalid = Agent('MAPK1',
                          mods=[ModCondition('phosphorylation', 'T', '183'),
                                ModCondition('phosphorylation', 'Y', '185')],
                          db_refs={'UP': 'P28482'})
    mapk3_invalid = Agent('MAPK3',
                          mods=[ModCondition('phosphorylation', 'T', '201'),
                                ModCondition('phosphorylation', 'Y', '203')],
                          db_refs={'UP': 'P27361'})
    assert unicode_strs((mapk1_invalid, mapk3_invalid))
    return (mapk1_invalid, mapk3_invalid)

def check_validated_mapks(res, st1):
    """Validate that the invalid MAPKs have been fixed appropriately."""
    assert len(res) == 2
    valid_stmts = res[0]
    mapped_stmts = res[1]
    assert isinstance(valid_stmts, list)
    assert isinstance(mapped_stmts, list)
    assert len(valid_stmts) == 0
    assert len(mapped_stmts) == 1
    mapped_stmt = mapped_stmts[0]
    assert isinstance(mapped_stmt, MappedStatement)
    assert mapped_stmt.original_stmt == st1
    assert isinstance(mapped_stmt.mapped_mods, list)
    assert len(mapped_stmt.mapped_mods) == 4
    ms = mapped_stmt.mapped_stmt
    assert isinstance(ms, Statement)
    agents = ms.agent_list()
    assert len(agents) == 2
    agent1 = agents[0]
    agent2 = agents[1]
    validate_mapk1(agent1)
    assert agent2.mods[0].matches(ModCondition('phosphorylation', 'T', '202'))
    assert agent2.mods[1].matches(ModCondition('phosphorylation', 'Y', '204'))
    assert unicode_strs((res, st1))

def validate_mapk1(agent1):
    assert agent1.name == 'MAPK1'
    assert len(agent1.mods) == 2
    assert agent1.mods[0].matches(ModCondition('phosphorylation', 'T', '185'))
    assert agent1.mods[1].matches(ModCondition('phosphorylation', 'Y', '187'))

if __name__ == '__main__':
    test_ignore_blank_entries()

<|endoftext|>


<commit_before>"""
Created on 06.07.2014

@author: JDC Chodera, JH Prinz
"""

import logging
import openpathsampling as paths
from openpathsampling.netcdfplus import ObjectStore
import openpathsampling.engines as peng

from openpathsampling.storage import Storage

from Queue import Empty
import re

logger = logging.getLogger(__name__)
init_log = logging.getLogger('openpathsampling.initialization')

class RemoteClientObject(ObjectStore):
    def _load(self, idx):
        # No loading, only stuff that is in cache and memory
        return None

    def _save(self, obj, idx):
        # No loading, only caching and what is left in memory
        pass

    def __len__(self):
        if len(self.index) == 0:
            return 0
        else:
            return max(self.index.values()) + 1

class RemoteMasterObject(ObjectStore):

    def __init__(self, content_class, no_store=False):
        super(RemoteMasterObject, self).__init__(content_class)
        self.no_store = no_store

    def _load(self, idx):
        # No loading, only stuff that is in cache and memory

        result = self.storage.ask(
            "_cache_.simplifier.simplify_object(_cache_.%s[%d])" % (self.prefix, idx))

        return self.storage.simplifier.from_json(result)

    def _save(self, obj, idx):
        # No loading, only caching and what is left in memory

        if hasattr(self.storage, 'client'):
            s = self.simplifier.simplify_object(obj)
            if self.no_store:
                # this will change the store on the client-side to
                # use the appropriate reconstruction
                # on client-side we want all stores to be RemoteClientObject
                # ones. if we would use the normal push then the
                # type on the master side would be recreated
                # this should only be used for the StoreStore itself
                s['_cls'] = 'RemoteClientObject'

            self.storage.tell(
                '_str = "%s"' % s)

            self.storage.tell(
                "_obj = _cache_.simplifier.from_json(_str)")

            res = self.storage.ask(
                "_cache_.%s.save(_obj)" % self.prefix)

            return int(res)
        else:
            return None

    def save(self, obj, idx=None):
        if obj in self.index:
            # has been saved so quit and do nothing
            return self.index[obj]

        if hasattr(obj, '_idx'):
            # is a proxy of a saved object so do nothing
            return obj._idx

        if not isinstance(obj, self.content_class):
            raise ValueError(
                'This store can only store object of base type "%s". Given obj is of type "%s". You'
                'might need to use another store.' % (self.content_class, obj.__class__.__name__)
            )

        n_idx = self._save(obj, None)

        if n_idx is not None:
            self.index[obj] = n_idx

            # store the name in the cache
            if hasattr(self, 'cache'):
                self.cache[n_idx] = obj

        return n_idx

    def __len__(self):
        return 10000000

# =============================================================================================
# REMOTE SPECIFIC STORAGE
# =============================================================================================

class RemoteClientStorage(Storage):
    """
    A netCDF4 wrapper to store trajectories based on snapshots of an OpenMM
    simulation. This allows effective storage of shooting trajectories
    """

    USE_FEATURE_SNAPSHOTS = True

    def __init__(self):
        """
        Create a netCDF+ storage for OPS Objects

        Parameters
        ----------
        filename : string
            filename of the netcdf file to be used or created
        mode : string, default: None
            the mode of file creation, one of ``w`` (write), ``a`` (append) or
            None, which will append any existing files.
        template : :class:`openpathsampling.Snapshot`
            a Snapshot instance that contains a reference to a Topology, the
            number of atoms and used units
        """

        self._setup_class()

        self.register_store('stores', RemoteClientObject(ObjectStore))
        self.stores.set_caching(True)

class RemoteMasterStorage(Storage):
    """
    A netCDF4 wrapper to store trajectories based on snapshots of an OpenMM
    simulation. This allows effective storage of shooting trajectories
    """

    USE_FEATURE_SNAPSHOTS = True

    ANSI_ESCAPE = re.compile(r'\x1b[^m]*m')

    @property
    def n_atoms(self):
        return self.topology.n_atoms

    @property
    def n_spatial(self):
        return self.topology.n_spatial

    def __init__(self, client):
        """
        Create a netCDF+ storage for OPS Objects

        Parameters
        ----------
        filename : string
            filename of the netcdf file to be used or created
        mode : string, default: None
            the mode of file creation, one of ``w`` (write), ``a`` (append) or
            None, which will append any existing files.
        template : :class:`openpathsampling.Snapshot`
            a Snapshot instance that contains a reference to a Topology, the
            number of atoms and used units
        """

        self.client = client

        self.iopub = client.iopub_channel
        self.shell = client.shell_channel

        self.client_name = '_cache_'

        self.tell(
"""
import openpathsampling as paths
paths.netcdfplus.base.StorableObject.set_observer(True)
_cache_ = paths.storage.remote.RemoteClientStorage()
"""
        )

        self._setup_class()

        self.register_store('stores', RemoteMasterObject(ObjectStore, ))
        self.stores.set_caching(True)

        self._create_storages()
        self._initialize()

        for store, idx in self.stores.index.iteritems():
            self.tell("_cache_.register_store('%s', _cache_.stores[%d])" % (store.name, idx))
            self.tell("_cache_.stores[%d].name = '%s'" % (idx, store.name))

        self.tell("_cache_.set_caching_mode('default')")
        self.tell("_cache_.simplifier.update_class_list()")

    def _tb(self, tb):
        return '\n'.join([self.ANSI_ESCAPE.sub('', l) for l in tb])

    def ask(self, cmd):
        uuid = self.tell(cmd)
        return self.listen(uuid)

    def listen(self, uuid):
        found = False
        try:
            while not found:
                msg = self.client.iopub_channel.get_msg(timeout=1.0)
                if 'msg_id' in msg['parent_header'] and msg['parent_header']['msg_id'] == uuid:
                    if msg['msg_type'] == 'execute_result':
                        found = True
                        result = msg['content']['data']['text/plain']
                    elif msg['msg_type'] =='error':
                        found = True
                        result = None
                        error = msg['content']
                        print error['evalue']
                        print self._tb(error['traceback'])

        except Empty:
            return None

        return result

    def clear(self):
        self.iopub.get_msgs()

    def tell(self, cmd):
        return self.client.execute(cmd)

    def __getitem__(self, item):
        ss = self.ask('_cache_.simplifier.simplify(%s)' % item)
        return self.simplifier.from_json(ss)

    def __setitem__(self, key, value):
        ss = self.simplifier.simplify(value)
        self.tell('%s = _cache_.simplifier.from_json("%s")' % (key, ss))

    def _create_storages(self):
        """
        Register all Stores used in the OpenPathSampling Storage

        """

        # objects with special storages

        stores = {
            'trajectories' : paths.Trajectory,
            'snapshots' : paths.BaseSnapshot,
            'samples' : paths.Sample,
            'samplesets' : paths.SampleSet,
            'pathmovechanges' : paths.PathMoveChange,
            'steps' : paths.MCStep,
            'cvs' : paths.CollectiveVariable,
            'details' : paths.Details,
            'topologies' : peng.Topology,
            'pathmovers' : paths.PathMover,
            'shootingpointselectors' : paths.ShootingPointSelector,
            'engines' : peng.DynamicsEngine,
            'pathsimulators' : paths.PathSimulator,
            'transitions' : paths.Transition,
            'schemes' : paths.MoveScheme,
            'volumes' : paths.Volume,
            'ensembles' : paths.Ensemble
        }

        for name, obj in stores.iteritems():
            self.create_store(name, RemoteMasterObject(obj))

    def _initialize(self):
        # Set global attributes.
        self.set_caching_mode('default')

    def close(self):
        self.kc.stop_channels()
        self.km.shutdown_kernel()
        del self.km<commit_msg>cleanup<commit_after>"""
Created on 06.07.2014

@author: JDC Chodera, JH Prinz
"""

import logging
import openpathsampling as paths
from openpathsampling.netcdfplus import ObjectStore
import openpathsampling.engines as peng

from openpathsampling.storage import Storage

from Queue import Empty
import re

logger = logging.getLogger(__name__)
init_log = logging.getLogger('openpathsampling.initialization')

class RemoteClientObject(ObjectStore):
    def _load(self, idx):
        # No loading, only stuff that is in cache and memory
        return None

    def _save(self, obj, idx):
        # No loading, only caching and what is left in memory
        pass

    def __len__(self):
        if len(self.index) == 0:
            return 0
        else:
            return max(self.index.values()) + 1

class RemoteMasterObject(ObjectStore):

    def __init__(self, content_class, no_store=False):
        super(RemoteMasterObject, self).__init__(content_class)
        self.no_store = no_store

    def _load(self, idx):
        # No loading, only stuff that is in cache and memory

        result = self.storage.ask(
            "_cache_.simplifier.simplify_object(_cache_.%s[%d])" % (self.prefix, idx))

        return self.storage.simplifier.from_json(result)

    def _save(self, obj, idx):
        # No loading, only caching and what is left in memory

        if hasattr(self.storage, 'client'):
            s = self.simplifier.simplify_object(obj)
            if self.no_store:
                # this will change the store on the client-side to
                # use the appropriate reconstruction
                # on client-side we want all stores to be RemoteClientObject
                # ones. if we would use the normal push then the
                # type on the master side would be recreated
                # this should only be used for the StoreStore itself
                s['_cls'] = 'RemoteClientObject'

            self.storage.tell(
                '_str = "%s"' % s)

            self.storage.tell(
                "_obj = _cache_.simplifier.from_json(_str)")

            res = self.storage.ask(
                "_cache_.%s.save(_obj)" % self.prefix)

            return int(res)
        else:
            return None

    def save(self, obj, idx=None):
        if obj in self.index:
            # has been saved so quit and do nothing
            return self.index[obj]

        if hasattr(obj, '_idx'):
            # is a proxy of a saved object so do nothing
            return obj._idx

        if not isinstance(obj, self.content_class):
            raise ValueError(
                'This store can only store object of base type "%s". Given obj is of type "%s". You'
                'might need to use another store.' % (self.content_class, obj.__class__.__name__)
            )

        n_idx = self._save(obj, None)

        if n_idx is not None:
            self.index[obj] = n_idx

            # store the name in the cache
            if hasattr(self, 'cache'):
                self.cache[n_idx] = obj

        return n_idx

    def __len__(self):
        return 10000000

# =============================================================================================
# REMOTE SPECIFIC STORAGE
# =============================================================================================

class RemoteClientStorage(Storage):
    """
    A netCDF4 wrapper to store trajectories based on snapshots of an OpenMM
    simulation. This allows effective storage of shooting trajectories
    """

    USE_FEATURE_SNAPSHOTS = True

    def __init__(self):
        """
        Create a netCDF+ storage for OPS Objects

        Parameters
        ----------
        filename : string
            filename of the netcdf file to be used or created
        mode : string, default: None
            the mode of file creation, one of ``w`` (write), ``a`` (append) or
            None, which will append any existing files.
        template : :class:`openpathsampling.Snapshot`
            a Snapshot instance that contains a reference to a Topology, the
            number of atoms and used units
        """

        self._setup_class()

        self.register_store('stores', RemoteClientObject(ObjectStore))
        self.stores.set_caching(True)

class RemoteMasterStorage(Storage):
    """
    A netCDF4 wrapper to store trajectories based on snapshots of an OpenMM
    simulation. This allows effective storage of shooting trajectories
    """

    USE_FEATURE_SNAPSHOTS = True

    ANSI_ESCAPE = re.compile(r'\x1b[^m]*m')

    @property
    def n_atoms(self):
        return self.topology.n_atoms

    @property
    def n_spatial(self):
        return self.topology.n_spatial

    def __init__(self, client):
        """
        Create a netCDF+ storage for OPS Objects

        Parameters
        ----------
        filename : string
            filename of the netcdf file to be used or created
        mode : string, default: None
            the mode of file creation, one of ``w`` (write), ``a`` (append) or
            None, which will append any existing files.
        template : :class:`openpathsampling.Snapshot`
            a Snapshot instance that contains a reference to a Topology, the
            number of atoms and used units
        """

        self.client = client

        self.iopub = client.iopub_channel
        self.shell = client.shell_channel

        self.client_name = '_cache_'

        self.tell(
"""
import openpathsampling as paths
paths.netcdfplus.base.StorableObject.set_observer(True)
_cache_ = paths.storage.remote.RemoteClientStorage()
"""
        )

        self._setup_class()

        self.register_store('stores', RemoteMasterObject(ObjectStore, True))
        self.stores.set_caching(True)

        self._create_storages()
        self._initialize()

        for store, idx in self.stores.index.iteritems():
            self.tell("_cache_.register_store('%s', _cache_.stores[%d])" % (store.name, idx))
            self.tell("_cache_.stores[%d].name = '%s'" % (idx, store.name))

        self.tell("_cache_.set_caching_mode('default')")
        self.tell("_cache_.simplifier.update_class_list()")

    def _tb(self, tb):
        return '\n'.join([self.ANSI_ESCAPE.sub('', l) for l in tb])

    def ask(self, cmd):
        uuid = self.tell(cmd)
        return self.listen(uuid)

    def listen(self, uuid):
        found = False
        try:
            while not found:
                msg = self.client.iopub_channel.get_msg(timeout=1.0)
                if 'msg_id' in msg['parent_header'] and msg['parent_header']['msg_id'] == uuid:
                    if msg['msg_type'] == 'execute_result':
                        found = True
                        result = msg['content']['data']['text/plain']
                    elif msg['msg_type'] =='error':
                        found = True
                        result = None
                        error = msg['content']
                        print error['evalue']
                        print self._tb(error['traceback'])

        except Empty:
            return None

        return result

    def clear(self):
        self.iopub.get_msgs()

    def tell(self, cmd):
        return self.client.execute(cmd)

    def __getitem__(self, item):
        ss = self.ask('_cache_.simplifier.simplify(%s)' % item)
        return self.simplifier.from_json(ss)

    def __setitem__(self, key, value):
        ss = self.simplifier.simplify(value)
        self.tell('%s = _cache_.simplifier.from_json("%s")' % (key, ss))

    def _create_storages(self):
        """
        Register all Stores used in the OpenPathSampling Storage

        """

        # objects with special storages

        stores = {
            'trajectories' : paths.Trajectory,
            'snapshots' : paths.BaseSnapshot,
            'samples' : paths.Sample,
            'samplesets' : paths.SampleSet,
            'pathmovechanges' : paths.PathMoveChange,
            'steps' : paths.MCStep,
            'cvs' : paths.CollectiveVariable,
            'details' : paths.Details,
            'topologies' : peng.Topology,
            'pathmovers' : paths.PathMover,
            'shootingpointselectors' : paths.ShootingPointSelector,
            'engines' : peng.DynamicsEngine,
            'pathsimulators' : paths.PathSimulator,
            'transitions' : paths.Transition,
            'schemes' : paths.MoveScheme,
            'volumes' : paths.Volume,
            'ensembles' : paths.Ensemble
        }

        for name, obj in stores.iteritems():
            self.create_store(name, RemoteMasterObject(obj))

    def _initialize(self):
        # Set global attributes.
        self.set_caching_mode('default')

    def close(self):
        self.kc.stop_channels()
        self.km.shutdown_kernel()
        del self.km<|endoftext|>