<commit_before>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import pickle
import logging
from indra.statements import Agent
from indra.assemblers import PysbAssembler
from indra.preassembler import Preassembler
from indra.preassembler.hierarchy_manager import hierarchies
from indra.preassembler import grounding_mapper as gm
from indra.belief import BeliefEngine
from indra.databases import uniprot_client

logger = logging.getLogger('incremental_model')

class IncrementalModel(object):
    """Assemble a model incrementally by iteratively adding new Statements.

    Parameters
    ----------
    model_fname : Optional[str]
        The name of the pickle file in which a set of INDRA Statements are
        stored in a dict keyed by PubMed IDs. This is the state of an
        IncrementalModel that is loaded upon instantiation.

    Attributes
    ----------
    stmts : dict[str, list[indra.statements.Statement]]
        A dictionary of INDRA Statements keyed by PMIDs that stores the current
        state of the IncrementalModel.
    unique_stmts : list[indra.statements.Statement]
        A list of INDRA Statements after de-duplication.
    toplevel_stmts : list[indra.statements.Statement]
        A list of the top level (most specific) INDRA Statements after
        preassembly.
    """
    def __init__(self, model_fname=None):
        if model_fname is None:
            self.stmts = {}
        else:
            try:
                with open(model_fname, 'rb') as f:
                    self.stmts = pickle.load(f)
            except:
                logger.warning('Could not load %s, starting new model.' %
                               model_fname)
                self.stmts = {}
        self.relevant_stmts = []
        self.unique_stmts = []
        self.toplevel_stmts = []

    def save(self, model_fname='model.pkl'):
        """Save the state of the IncrementalModel in a pickle file.

        model_fname : Optional[str]
            The name of the pickle file to save the state of the
            IncrementalModel in. Default: model.pkl
        """
        with open(model_fname, 'wb') as fh:
            pickle.dump(self.stmts, fh, protocol=2)

    def _relevance_filter(self, stmts, filters=None):
        stmts_to_add = list(range(len(stmts)))
        # Filter for grounding
        if 'grounding' in filters:
            for i, stmt in enumerate(stmts):
                # Check that all agents are grounded
                for ag in stmt.agent_list():
                    if ag is None:
                        continue
                    if not ag.db_refs or list(ag.db_refs.keys()) == ['TEXT']:
                        stmts_to_add.remove(i)
                        break

        if ('prior_all' in filters) or ('prior_one' in filters):
            prior_agents = self.get_prior_agents()
            if prior_agents:
                for i in stmts_to_add:
                    agents = [a for a in stmts[i].agent_list()
                              if a is not None]
                    if 'prior_all' in filters:
                        for st_agent in agents:
                            found = False
                            for pr_agent in prior_agents:
                                if self._agent_related(st_agent, pr_agent):
                                    found = True
                                    break
                            if not found:
                                stmts_to_add.remove(i)
                                break
                    if 'prior_one' in filters:
                        found = False
                        for st_agent in agents:
                            for pr_agent in prior_agents:
                                if self._agent_related(st_agent, pr_agent):
                                    found = True
                                    break
                            if found:
                                break
                        if not found:
                                stmts_to_add.remove(i)

        if ('model_all' in filters) or ('model_one' in filters):
            model_agents = self.get_model_agents()
            if model_agents:
                for i in stmts_to_add:
                    agents = [a for a in stmts[i].agent_list()
                              if a is not None]
                    if 'model_all' in filters:
                        for st_agent in agents:
                            found = False
                            for mo_agent in model_agents:
                                if self._agent_related(st_agent, mo_agent):
                                    found = True
                                    break
                            if not found:
                                stmts_to_add.remove(i)
                                break
                    if 'model_one' in filters:
                        found = False
                        for st_agent in agents:
                            for mo_agent in model_agents:
                                if self._agent_related(st_agent, mo_agent):
                                    found = True
                                    break
                            if found:
                                break
        if ('human_only' in filters):
            for i in stmts_to_add:
                agents = [a for a in stmts[i].agent_list()
                          if a is not None]
                for a in agents:
                    up_id = a.db_refs.get('UP')
                    hgnc_id = a.db_refs.get('HGNC')
                    if not hgnc_id:
                        if up_id:
                            mnemonic = uniprot_client.get_mnemonic(up_id, True)
                            if mnemonic is None:
                                stmts_to_add.remove(i)
                                break
                            if not mnemonic.endswith('HUMAN'):
                                stmts_to_add.remove(i)
                                break

        relevant_stmts = [stmts[i] for i in stmts_to_add]
        return relevant_stmts

    def add_statements(self, pmid, stmts, filters=None):
        """Add INDRA Statements to the incremental model indexed by PMID.

        Currently the following filter options are implemented:
        - grounding: require that all Agents in statements are grounded
        - model_one: require that at least one Agent is in the incremental
                      model
        - model_all: require that all Agents are in the incremental model
        - prior_one: require that at least one Agent is in the
                      prior model
        - prior_all: require that all Agents are in the prior model
        Note that model_one -> prior_all are increasingly more restrictive
        options.

        Parameters
        ----------
        pmid : str
            The PMID of the paper from which statements were extracted.
        stmts : list[indra.statements.Statement]
            A list of INDRA Statements to be added to the model.
        filter : Optional[list[str]]
            A list of filter options to apply when adding the statements.
            See description above for more details. Default: None
        """
        # If no filter is used, we add all statements to the model
        if not filters:
            self.stmts[pmid] = stmts
            return
        # If the statements are empty in the first place
        if not stmts:
            self.stmts[pmid] = []
            return

        relevant_stmts = self._relevance_filter(stmts, filters)
        self.stmts[pmid] = relevant_stmts

    def preassemble(self, filters=None):
        """Preassemble the Statements collected in the model.

        Use INDRA's GroundingMapper, Preassembler and BeliefEngine
        on the IncrementalModel and save the unique statements and
        the top level statements in class attributes.

        Currently the following filter options are implemented:
        - grounding: require that all Agents in statements are grounded
        - model_one: require that at least one Agent is in the incremental
                      model
        - model_all: require that all Agents are in the incremental model
        - prior_one: require that at least one Agent is in the
                      prior model
        - prior_all: require that all Agents are in the prior model
        Note that model_one -> prior_all are increasingly more restrictive
        options.

        Parameters
        ----------
        filter : Optional[list[str]]
            A list of filter options to apply when choosing the statements.
            See description above for more details. Default: None
        """
        stmts = self.get_statements_noprior()

        # Fix grounding
        logger.info('Running grounding map')
        twg = gm.agent_texts_with_grounding(stmts)
        prot_map = gm.protein_map_from_twg(twg)
        gm.default_grounding_map.update(prot_map)
        gmap = gm.GroundingMapper(gm.default_grounding_map)
        gmapped_stmts = gmap.map_agents(stmts, do_rename=True)

        # Merge the prior and the mapped non-prior
        stmts = gmapped_stmts + self.get_statements_prior()
        logger.info('%d unfiltered Statements' % len(stmts))

        if filters:
            if 'grounding' in filters:
                # Filter out ungrounded statements
                logger.info('Running grounding filter')
                stmts = self._relevance_filter(stmts, ['grounding'])
                logger.info('%s Statements after filter' % len(stmts))
            if 'human_only' in filters:
                # Filter out non-human proteins
                logger.info('Running non-human protein filter')
                stmts = self._relevance_filter(stmts, ['human_only'])
                logger.info('%s Statements after filter' % len(stmts))
            for rel_key in ('prior_one', 'model_one',
                             'prior_all', 'model_all'):
                if rel_key in filters:
                    logger.info('Running relevance filter')
                    stmts = self._relevance_filter(stmts, [rel_key])
                    logger.info('%s Statements after filter' % len(stmts))

        # Combine duplicates
        logger.info('Preassembling %d Statements' % len(stmts))
        pa = Preassembler(hierarchies, stmts)
        self.unique_stmts = pa.combine_duplicates()

        # Run BeliefEngine on unique statements
        be = BeliefEngine(self.unique_stmts)
        be.set_prior_probs(self.unique_stmts)

        # Build statement hierarchy
        self.toplevel_stmts = pa.combine_related()
        # Run BeliefEngine on hierarchy
        be.set_hierarchy_probs(self.toplevel_stmts)

    def load_prior(self, prior_fname):
        """Load a set of prior statements from a pickle file.

        The prior statements have a special key in the stmts dictionary
        called "prior".
        """
        with open(prior_fname, 'rb') as fh:
            self.stmts['prior'] = pickle.load(fh)

    def get_model_agents(self):
        """Return a list of all Agents from all Statements."""
        model_stmts = self.get_statements_noprior()
        # First, get all unique name/db_refs pairs
        agents = set()
        for stmt in model_stmts:
            for a in stmt.agent_list():
                if a is not None:
                    agents.add((a.name, a.get_grounding()))
        # Then construct sateless name/db_refs Agents
        agents = [Agent(n, db_refs={g[0]: g[1]}) for n, g in agents]
        return agents

    def get_prior_agents(self):
        """Return a list of all Agents from the prior Statements."""
        prior_stmts = self.get_statements_prior()
        # First, get all unique name/db_refs pairs
        agents = set()
        if prior_stmts is None:
            return agents
        for stmt in prior_stmts:
            for a in stmt.agent_list():
                if a is not None:
                    agents.add((a.name, a.get_grounding()))
        agents = [Agent(n, db_refs={g[0]: g[1]}) for n, g in agents]
        return agents

    @staticmethod
    def _agent_related(a1, a2):
        if a1.matches(a2) or \
            a1.isa(a2, hierarchies) or \
            a2.isa(a1, hierarchies):
            return True
        return False

    def get_statements(self):
        """Return a list of all Statements in a single list."""
        stmt_lists = [v for k, v in self.stmts.items()]
        stmts = []
        for s in stmt_lists:
            stmts += s
        return stmts

    def get_statements_noprior(self):
        """Return a list of all non-prior Statements in a single list."""
        stmt_lists = [v for k, v in self.stmts.items() if k != 'prior']
        stmts = []
        for s in stmt_lists:
            stmts += s
        return stmts

    def get_statements_prior(self):
        """Return a list of all prior Statements in a single list."""
        if self.stmts.get('prior') is not None:
            return self.stmts['prior']
        else:
            return []
<commit_msg>Site mapping and prior/no-prior merging in IncrementalModel<commit_after>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import pickle
import logging
from indra.statements import Agent
from indra.belief import BeliefEngine
from indra.assemblers import PysbAssembler
from indra.databases import uniprot_client
from indra.preassembler import Preassembler
from indra.preassembler import grounding_mapper as gm
from indra.preassembler.hierarchy_manager import hierarchies
from indra.preassembler.sitemapper import SiteMapper, default_site_map

logger = logging.getLogger('incremental_model')

class IncrementalModel(object):
    """Assemble a model incrementally by iteratively adding new Statements.

    Parameters
    ----------
    model_fname : Optional[str]
        The name of the pickle file in which a set of INDRA Statements are
        stored in a dict keyed by PubMed IDs. This is the state of an
        IncrementalModel that is loaded upon instantiation.

    Attributes
    ----------
    stmts : dict[str, list[indra.statements.Statement]]
        A dictionary of INDRA Statements keyed by PMIDs that stores the current
        state of the IncrementalModel.
    unique_stmts : list[indra.statements.Statement]
        A list of INDRA Statements after de-duplication.
    toplevel_stmts : list[indra.statements.Statement]
        A list of the top level (most specific) INDRA Statements after
        preassembly.
    """
    def __init__(self, model_fname=None):
        if model_fname is None:
            self.stmts = {}
        else:
            try:
                with open(model_fname, 'rb') as f:
                    self.stmts = pickle.load(f)
            except:
                logger.warning('Could not load %s, starting new model.' %
                               model_fname)
                self.stmts = {}
        self.relevant_stmts = []
        self.unique_stmts = []
        self.toplevel_stmts = []

    def save(self, model_fname='model.pkl'):
        """Save the state of the IncrementalModel in a pickle file.

        model_fname : Optional[str]
            The name of the pickle file to save the state of the
            IncrementalModel in. Default: model.pkl
        """
        with open(model_fname, 'wb') as fh:
            pickle.dump(self.stmts, fh, protocol=2)

    def _relevance_filter(self, stmts, filters=None):
        stmts_to_add = list(range(len(stmts)))
        # Filter for grounding
        if 'grounding' in filters:
            for i, stmt in enumerate(stmts):
                # Check that all agents are grounded
                for ag in stmt.agent_list():
                    if ag is None:
                        continue
                    if not ag.db_refs or list(ag.db_refs.keys()) == ['TEXT']:
                        stmts_to_add.remove(i)
                        break

        if ('prior_all' in filters) or ('prior_one' in filters):
            prior_agents = self.get_prior_agents()
            if prior_agents:
                for i in stmts_to_add:
                    agents = [a for a in stmts[i].agent_list()
                              if a is not None]
                    if 'prior_all' in filters:
                        for st_agent in agents:
                            found = False
                            for pr_agent in prior_agents:
                                if self._agent_related(st_agent, pr_agent):
                                    found = True
                                    break
                            if not found:
                                stmts_to_add.remove(i)
                                break
                    if 'prior_one' in filters:
                        found = False
                        for st_agent in agents:
                            for pr_agent in prior_agents:
                                if self._agent_related(st_agent, pr_agent):
                                    found = True
                                    break
                            if found:
                                break
                        if not found:
                                stmts_to_add.remove(i)

        if ('model_all' in filters) or ('model_one' in filters):
            model_agents = self.get_model_agents()
            if model_agents:
                for i in stmts_to_add:
                    agents = [a for a in stmts[i].agent_list()
                              if a is not None]
                    if 'model_all' in filters:
                        for st_agent in agents:
                            found = False
                            for mo_agent in model_agents:
                                if self._agent_related(st_agent, mo_agent):
                                    found = True
                                    break
                            if not found:
                                stmts_to_add.remove(i)
                                break
                    if 'model_one' in filters:
                        found = False
                        for st_agent in agents:
                            for mo_agent in model_agents:
                                if self._agent_related(st_agent, mo_agent):
                                    found = True
                                    break
                            if found:
                                break
        if ('human_only' in filters):
            for i in stmts_to_add:
                agents = [a for a in stmts[i].agent_list()
                          if a is not None]
                for a in agents:
                    up_id = a.db_refs.get('UP')
                    hgnc_id = a.db_refs.get('HGNC')
                    if not hgnc_id:
                        if up_id:
                            mnemonic = uniprot_client.get_mnemonic(up_id, True)
                            if mnemonic is None:
                                stmts_to_add.remove(i)
                                break
                            if not mnemonic.endswith('HUMAN'):
                                stmts_to_add.remove(i)
                                break

        relevant_stmts = [stmts[i] for i in stmts_to_add]
        return relevant_stmts

    def add_statements(self, pmid, stmts, filters=None):
        """Add INDRA Statements to the incremental model indexed by PMID.

        Currently the following filter options are implemented:
        - grounding: require that all Agents in statements are grounded
        - model_one: require that at least one Agent is in the incremental
                      model
        - model_all: require that all Agents are in the incremental model
        - prior_one: require that at least one Agent is in the
                      prior model
        - prior_all: require that all Agents are in the prior model
        Note that model_one -> prior_all are increasingly more restrictive
        options.

        Parameters
        ----------
        pmid : str
            The PMID of the paper from which statements were extracted.
        stmts : list[indra.statements.Statement]
            A list of INDRA Statements to be added to the model.
        filter : Optional[list[str]]
            A list of filter options to apply when adding the statements.
            See description above for more details. Default: None
        """
        # If no filter is used, we add all statements to the model
        if not filters:
            self.stmts[pmid] = stmts
            return
        # If the statements are empty in the first place
        if not stmts:
            self.stmts[pmid] = []
            return

        relevant_stmts = self._relevance_filter(stmts, filters)
        self.stmts[pmid] = relevant_stmts

    def preassemble(self, filters=None):
        """Preassemble the Statements collected in the model.

        Use INDRA's GroundingMapper, Preassembler and BeliefEngine
        on the IncrementalModel and save the unique statements and
        the top level statements in class attributes.

        Currently the following filter options are implemented:
        - grounding: require that all Agents in statements are grounded
        - model_one: require that at least one Agent is in the incremental
                      model
        - model_all: require that all Agents are in the incremental model
        - prior_one: require that at least one Agent is in the
                      prior model
        - prior_all: require that all Agents are in the prior model
        Note that model_one -> prior_all are increasingly more restrictive
        options.

        Parameters
        ----------
        filter : Optional[list[str]]
            A list of filter options to apply when choosing the statements.
            See description above for more details. Default: None
        """
        stmts = self.get_statements()
        logger.info('%d raw Statements in total' % len(stmts))

        # Fix grounding
        logger.info('Running grounding map')
        twg = gm.agent_texts_with_grounding(stmts)
        prot_map = gm.protein_map_from_twg(twg)
        gm.default_grounding_map.update(prot_map)
        gmap = gm.GroundingMapper(gm.default_grounding_map)
        stmts = gmap.map_agents(stmts, do_rename=True)

        logger.info('%d Statements after grounding map' % len(stmts))

        # Fix sites
        sm = SiteMapper(default_site_map)
        stmts, _ = sm.map_sites(stmts)

        logger.info('%d Statements with valid sequence' % len(stmts))

        if filters:
            if 'grounding' in filters:
                # Filter out ungrounded statements
                logger.info('Running grounding filter')
                stmts = self._relevance_filter(stmts, ['grounding'])
                logger.info('%s Statements after filter' % len(stmts))
            if 'human_only' in filters:
                # Filter out non-human proteins
                logger.info('Running non-human protein filter')
                stmts = self._relevance_filter(stmts, ['human_only'])
                logger.info('%s Statements after filter' % len(stmts))
            for rel_key in ('prior_one', 'model_one',
                             'prior_all', 'model_all'):
                if rel_key in filters:
                    logger.info('Running relevance filter')
                    stmts = self._relevance_filter(stmts, [rel_key])
                    logger.info('%s Statements after filter' % len(stmts))

        # Combine duplicates
        logger.info('Preassembling %d Statements' % len(stmts))
        pa = Preassembler(hierarchies, stmts)
        self.unique_stmts = pa.combine_duplicates()
        logger.info('%d unique Statements' % len(self.unique_stmts))

        # Run BeliefEngine on unique statements
        be = BeliefEngine(self.unique_stmts)
        be.set_prior_probs(self.unique_stmts)

        # Build statement hierarchy
        self.toplevel_stmts = pa.combine_related()
        logger.info('%d top-level Statements' % len(self.toplevel_stmts))
        # Run BeliefEngine on hierarchy
        be.set_hierarchy_probs(self.toplevel_stmts)

    def load_prior(self, prior_fname):
        """Load a set of prior statements from a pickle file.

        The prior statements have a special key in the stmts dictionary
        called "prior".
        """
        with open(prior_fname, 'rb') as fh:
            self.stmts['prior'] = pickle.load(fh)

    def get_model_agents(self):
        """Return a list of all Agents from all Statements."""
        model_stmts = self.get_statements_noprior()
        # First, get all unique name/db_refs pairs
        agents = set()
        for stmt in model_stmts:
            for a in stmt.agent_list():
                if a is not None:
                    agents.add((a.name, a.get_grounding()))
        # Then construct sateless name/db_refs Agents
        agents = [Agent(n, db_refs={g[0]: g[1]}) for n, g in agents]
        return agents

    def get_prior_agents(self):
        """Return a list of all Agents from the prior Statements."""
        prior_stmts = self.get_statements_prior()
        # First, get all unique name/db_refs pairs
        agents = set()
        if prior_stmts is None:
            return agents
        for stmt in prior_stmts:
            for a in stmt.agent_list():
                if a is not None:
                    agents.add((a.name, a.get_grounding()))
        agents = [Agent(n, db_refs={g[0]: g[1]}) for n, g in agents]
        return agents

    @staticmethod
    def _agent_related(a1, a2):
        if a1.matches(a2) or \
            a1.isa(a2, hierarchies) or \
            a2.isa(a1, hierarchies):
            return True
        return False

    def get_statements(self):
        """Return a list of all Statements in a single list."""
        stmt_lists = [v for k, v in self.stmts.items()]
        stmts = []
        for s in stmt_lists:
            stmts += s
        return stmts

    def get_statements_noprior(self):
        """Return a list of all non-prior Statements in a single list."""
        stmt_lists = [v for k, v in self.stmts.items() if k != 'prior']
        stmts = []
        for s in stmt_lists:
            stmts += s
        return stmts

    def get_statements_prior(self):
        """Return a list of all prior Statements in a single list."""
        if self.stmts.get('prior') is not None:
            return self.stmts['prior']
        else:
            return []
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Utilities for managing assets."""

import argparse
import json
import os
import shlex
import shutil
import subprocess
import sys

INFRA_BOTS_DIR = os.path.abspath(os.path.realpath(os.path.join(
    os.path.dirname(os.path.abspath(__file__)), os.pardir)))
sys.path.insert(0, INFRA_BOTS_DIR)
import utils
import zip_utils

ASSETS_DIR = os.path.join(INFRA_BOTS_DIR, 'assets')
SKIA_DIR = os.path.abspath(os.path.join(INFRA_BOTS_DIR, os.pardir, os.pardir))

CIPD_PACKAGE_NAME_TMPL = 'skia/bots/%s'
DEFAULT_CIPD_SERVICE_URL = 'https://chrome-infra-packages.appspot.com'

DEFAULT_GS_BUCKET = 'skia-assets'
GS_SUBDIR_TMPL = 'gs://%s/assets/%s'
GS_PATH_TMPL = '%s/%s.zip'

TAG_PROJECT_SKIA = 'project:skia'
TAG_VERSION_PREFIX = 'version:'
TAG_VERSION_TMPL = '%s%%s' % TAG_VERSION_PREFIX

VERSION_FILENAME = 'VERSION'
ZIP_BLACKLIST = ['.git', '.svn', '*.pyc', '.DS_STORE']

class CIPDStore(object):
  """Wrapper object for CIPD."""
  def __init__(self, cipd_url=DEFAULT_CIPD_SERVICE_URL,
               service_account_json=None):
    self._cipd = 'cipd'
    if sys.platform == 'win32':
      self._cipd = 'cipd.bat'
    self._cipd_url = cipd_url
    if service_account_json:
      self._service_account_json = os.path.abspath(service_account_json)
    self._check_setup()

  def _check_setup(self):
    """Verify that we have the CIPD binary and that we're authenticated."""
    try:
      self._run(['auth-info'], specify_service_url=False)
    except OSError:
      raise Exception('CIPD binary not found on your path (typically in '
                      'depot_tools). You may need to update depot_tools.')
    except subprocess.CalledProcessError:
      raise Exception('CIPD not authenticated. You may need to run:\n\n'
                      '$ %s auth-login' % self._cipd)

  def _run(self, cmd, specify_service_url=True):
    """Run the given command."""
    cipd_args = []
    if specify_service_url:
      cipd_args.extend(['--service-url', self._cipd_url])
    if self._service_account_json:
      cipd_args.extend(['-service-account-json', self._service_account_json])
    elif os.getenv('USE_CIPD_GCE_AUTH'):
      # Enable automatic GCE authentication. For context see
      # https://bugs.chromium.org/p/skia/issues/detail?id=6385#c3
      cipd_args.extend(['-service-account-json', ':gce'])
    return subprocess.check_output(
        [self._cipd] + cmd + cipd_args,
        stderr=subprocess.STDOUT)

  def _json_output(self, cmd):
    """Run the given command, return the JSON output."""
    with utils.tmp_dir():
      json_output = os.path.join(os.getcwd(), 'output.json')
      self._run(cmd + ['--json-output', json_output])
      with open(json_output) as f:
        parsed = json.load(f)
    return parsed.get('result', [])

  def _search(self, pkg_name):
    try:
      res = self._json_output(['search', pkg_name, '--tag', TAG_PROJECT_SKIA])
    except subprocess.CalledProcessError as e:
      if 'no such package' in e.output:
        return []
      raise
    return [r['instance_id'] for r in res or []]

  def _describe(self, pkg_name, instance_id):
    """Obtain details about the given package and instance ID."""
    return self._json_output(['describe', pkg_name, '--version', instance_id])

  def get_available_versions(self, name):
    """List available versions of the asset."""
    pkg_name = CIPD_PACKAGE_NAME_TMPL % name
    versions = []
    for instance_id in self._search(pkg_name):
      details = self._describe(pkg_name, instance_id)
      for tag in details.get('tags'):
        tag_name = tag.get('tag', '')
        if tag_name.startswith(TAG_VERSION_PREFIX):
          trimmed = tag_name[len(TAG_VERSION_PREFIX):]
          try:
            versions.append(int(trimmed))
          except ValueError:
            raise ValueError('Found package instance with invalid version '
                             'tag: %s' % tag_name)
    versions.sort()
    return versions

  def upload(self, name, version, target_dir, extra_tags=None):
    """Create a CIPD package."""
    cmd = [
        'create',
        '--name', CIPD_PACKAGE_NAME_TMPL % name,
        '--in', target_dir,
        '--tag', TAG_PROJECT_SKIA,
        '--tag', TAG_VERSION_TMPL % version,
        '--compression-level', '1',
        '-verification-timeout', '30m0s',
    ]
    if extra_tags:
      for tag in extra_tags:
        cmd.extend(['--tag', tag])
    self._run(cmd)

  def download(self, name, version, target_dir):
    """Download a CIPD package."""
    pkg_name = CIPD_PACKAGE_NAME_TMPL % name
    version_tag = TAG_VERSION_TMPL % version
    target_dir = os.path.abspath(target_dir)
    with utils.tmp_dir():
      infile = os.path.join(os.getcwd(), 'input')
      with open(infile, 'w') as f:
        f.write('%s %s' % (pkg_name, version_tag))
      self._run([
          'ensure',
          '--root', target_dir,
          '--list', infile,
      ])

  def delete_contents(self, name):
    """Delete data for the given asset."""
    self._run(['pkg-delete', CIPD_PACKAGE_NAME_TMPL % name])

class GSStore(object):
  """Wrapper object for interacting with Google Storage."""
  def __init__(self, gsutil=None, bucket=DEFAULT_GS_BUCKET,
               service_account_json=None):
    if gsutil:
      gsutil = os.path.abspath(gsutil)
    else:
      gsutils = subprocess.check_output([
          utils.WHICH, 'gsutil']).rstrip().splitlines()
      for g in gsutils:
        ok = True
        try:
          subprocess.check_call([g, 'version'])
        except OSError:
          ok = False
        if ok:
          gsutil = g
          break
    self._gsutil = [gsutil]
    if gsutil.endswith('.py'):
      self._gsutil = ['python', gsutil]
    if service_account_json:
      sa = os.path.abspath(service_account_json)
      self._gsutil += ['-o', 'Credentials:gs_service_key_file=' + sa]
    self._gs_bucket = bucket

  def copy(self, src, dst):
    """Copy src to dst."""
    subprocess.check_call(self._gsutil + ['cp', src, dst])

  def list(self, path):
    """List objects in the given path."""
    try:
      return subprocess.check_output(self._gsutil + ['ls', path]).splitlines()
    except subprocess.CalledProcessError:
      # If the prefix does not exist, we'll get an error, which is okay.
      return []

  def get_available_versions(self, name):
    """Return the existing version numbers for the asset."""
    files = self.list(GS_SUBDIR_TMPL % (self._gs_bucket, name))
    bnames = [os.path.basename(f) for f in files]
    suffix = '.zip'
    versions = [int(f[:-len(suffix)]) for f in bnames if f.endswith(suffix)]
    versions.sort()
    return versions

  # pylint: disable=unused-argument
  def upload(self, name, version, target_dir, extra_tags=None):
    """Upload to GS."""
    target_dir = os.path.abspath(target_dir)
    with utils.tmp_dir():
      zip_file = os.path.join(os.getcwd(), '%d.zip' % version)
      zip_utils.zip(target_dir, zip_file, blacklist=ZIP_BLACKLIST)
      gs_path = GS_PATH_TMPL % (GS_SUBDIR_TMPL % (self._gs_bucket, name),
                                str(version))
      self.copy(zip_file, gs_path)

  def download(self, name, version, target_dir):
    """Download from GS."""
    gs_path = GS_PATH_TMPL % (GS_SUBDIR_TMPL % (self._gs_bucket, name),
                              str(version))
    target_dir = os.path.abspath(target_dir)
    with utils.tmp_dir():
      zip_file = os.path.join(os.getcwd(), '%d.zip' % version)
      self.copy(gs_path, zip_file)
      zip_utils.unzip(zip_file, target_dir)

  def delete_contents(self, name):
    """Delete data for the given asset."""
    gs_path = GS_SUBDIR_TMPL % (self._gs_bucket, name)
    attempt_delete = True
    try:
      subprocess.check_call(self._gsutil + ['ls', gs_path])
    except subprocess.CalledProcessError:
      attempt_delete = False
    if attempt_delete:
      subprocess.check_call(self._gsutil + ['rm', '-rf', gs_path])

class MultiStore(object):
  """Wrapper object which uses CIPD as the primary store and GS for backup."""
  def __init__(self, cipd_url=DEFAULT_CIPD_SERVICE_URL,
               service_account_json=None,
               gsutil=None, gs_bucket=DEFAULT_GS_BUCKET):
    self._cipd = CIPDStore(cipd_url=cipd_url,
                           service_account_json=service_account_json)
    self._gs = GSStore(gsutil=gsutil, bucket=gs_bucket,
                       service_account_json=service_account_json)

  def get_available_versions(self, name):
    return self._cipd.get_available_versions(name)

  def upload(self, name, version, target_dir, extra_tags=None):
    self._cipd.upload(name, version, target_dir, extra_tags=extra_tags)
    self._gs.upload(name, version, target_dir, extra_tags=extra_tags)

  def download(self, name, version, target_dir):
    self._gs.download(name, version, target_dir)

  def delete_contents(self, name):
    self._cipd.delete_contents(name)
    self._gs.delete_contents(name)

def _prompt(prompt):
  """Prompt for input, return result."""
  return raw_input(prompt)

class Asset(object):
  def __init__(self, name, store):
    self._store = store
    self._name = name
    self._dir = os.path.join(ASSETS_DIR, self._name)

  @property
  def version_file(self):
    """Return the path to the version file for this asset."""
    return os.path.join(self._dir, VERSION_FILENAME)

  def get_current_version(self):
    """Obtain the current version of the asset."""
    if not os.path.isfile(self.version_file):
      return -1
    with open(self.version_file) as f:
      return int(f.read())

  def get_available_versions(self):
    """Return the existing version numbers for this asset."""
    return self._store.get_available_versions(self._name)

  def get_next_version(self):
    """Find the next available version number for the asset."""
    versions = self.get_available_versions()
    if len(versions) == 0:
      return 0
    return versions[-1] + 1

  def download_version(self, version, target_dir):
    """Download the specified version of the asset."""
    self._store.download(self._name, version, target_dir)

  def download_current_version(self, target_dir):
    """Download the version of the asset specified in its version file."""
    v = self.get_current_version()
    self.download_version(v, target_dir)

  def upload_new_version(self, target_dir, commit=False, extra_tags=None):
    """Upload a new version and update the version file for the asset."""
    version = self.get_next_version()
    self._store.upload(self._name, version, target_dir, extra_tags=extra_tags)

    def _write_version():
      with open(self.version_file, 'w') as f:
        f.write(str(version))
      subprocess.check_call([utils.GIT, 'add', self.version_file])

    with utils.chdir(SKIA_DIR):
      if commit:
        with utils.git_branch():
          _write_version()
          subprocess.check_call([
              utils.GIT, 'commit', '-m', 'Update %s version' % self._name])
          subprocess.check_call([utils.GIT, 'cl', 'upload', '--bypass-hooks'])
      else:
        _write_version()

  @classmethod
  def add(cls, name, store):
    """Add an asset."""
    asset = cls(name, store)
    if os.path.isdir(asset._dir):
      raise Exception('Asset %s already exists!' % asset._name)

    print 'Creating asset in %s' % asset._dir
    os.mkdir(asset._dir)
    def copy_script(script):
      src = os.path.join(ASSETS_DIR, 'scripts', script)
      dst = os.path.join(asset._dir, script)
      print 'Creating %s' % dst
      shutil.copy(src, dst)
      subprocess.check_call([utils.GIT, 'add', dst])

    for script in ('download.py', 'upload.py', 'common.py'):
      copy_script(script)
    resp = _prompt('Add script to automate creation of this asset? (y/n) ')
    if resp == 'y':
      copy_script('create.py')
      copy_script('create_and_upload.py')
      print 'You will need to add implementation to the creation script.'
    print 'Successfully created asset %s.' % asset._name
    return asset

  def remove(self, remove_in_store=False):
    """Remove this asset."""
    # Ensure that the asset exists.
    if not os.path.isdir(self._dir):
      raise Exception('Asset %s does not exist!' % self._name)

    # Cleanup the store.
    if remove_in_store:
      self._store.delete_contents(self._name)

    # Remove the asset.
    subprocess.check_call([utils.GIT, 'rm', '-rf', self._dir])
    if os.path.isdir(self._dir):
      shutil.rmtree(self._dir)
<commit_msg>Fix asset_utils when no service_account_json<commit_after>#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Utilities for managing assets."""

import argparse
import json
import os
import shlex
import shutil
import subprocess
import sys

INFRA_BOTS_DIR = os.path.abspath(os.path.realpath(os.path.join(
    os.path.dirname(os.path.abspath(__file__)), os.pardir)))
sys.path.insert(0, INFRA_BOTS_DIR)
import utils
import zip_utils

ASSETS_DIR = os.path.join(INFRA_BOTS_DIR, 'assets')
SKIA_DIR = os.path.abspath(os.path.join(INFRA_BOTS_DIR, os.pardir, os.pardir))

CIPD_PACKAGE_NAME_TMPL = 'skia/bots/%s'
DEFAULT_CIPD_SERVICE_URL = 'https://chrome-infra-packages.appspot.com'

DEFAULT_GS_BUCKET = 'skia-assets'
GS_SUBDIR_TMPL = 'gs://%s/assets/%s'
GS_PATH_TMPL = '%s/%s.zip'

TAG_PROJECT_SKIA = 'project:skia'
TAG_VERSION_PREFIX = 'version:'
TAG_VERSION_TMPL = '%s%%s' % TAG_VERSION_PREFIX

VERSION_FILENAME = 'VERSION'
ZIP_BLACKLIST = ['.git', '.svn', '*.pyc', '.DS_STORE']

class CIPDStore(object):
  """Wrapper object for CIPD."""
  def __init__(self, cipd_url=DEFAULT_CIPD_SERVICE_URL,
               service_account_json=None):
    self._cipd = 'cipd'
    if sys.platform == 'win32':
      self._cipd = 'cipd.bat'
    self._cipd_url = cipd_url
    if service_account_json:
      self._service_account_json = os.path.abspath(service_account_json)
    else:
      self._service_account_json = None
    self._check_setup()

  def _check_setup(self):
    """Verify that we have the CIPD binary and that we're authenticated."""
    try:
      self._run(['auth-info'], specify_service_url=False)
    except OSError:
      raise Exception('CIPD binary not found on your path (typically in '
                      'depot_tools). You may need to update depot_tools.')
    except subprocess.CalledProcessError:
      raise Exception('CIPD not authenticated. You may need to run:\n\n'
                      '$ %s auth-login' % self._cipd)

  def _run(self, cmd, specify_service_url=True):
    """Run the given command."""
    cipd_args = []
    if specify_service_url:
      cipd_args.extend(['--service-url', self._cipd_url])
    if self._service_account_json:
      cipd_args.extend(['-service-account-json', self._service_account_json])
    elif os.getenv('USE_CIPD_GCE_AUTH'):
      # Enable automatic GCE authentication. For context see
      # https://bugs.chromium.org/p/skia/issues/detail?id=6385#c3
      cipd_args.extend(['-service-account-json', ':gce'])
    return subprocess.check_output(
        [self._cipd] + cmd + cipd_args,
        stderr=subprocess.STDOUT)

  def _json_output(self, cmd):
    """Run the given command, return the JSON output."""
    with utils.tmp_dir():
      json_output = os.path.join(os.getcwd(), 'output.json')
      self._run(cmd + ['--json-output', json_output])
      with open(json_output) as f:
        parsed = json.load(f)
    return parsed.get('result', [])

  def _search(self, pkg_name):
    try:
      res = self._json_output(['search', pkg_name, '--tag', TAG_PROJECT_SKIA])
    except subprocess.CalledProcessError as e:
      if 'no such package' in e.output:
        return []
      raise
    return [r['instance_id'] for r in res or []]

  def _describe(self, pkg_name, instance_id):
    """Obtain details about the given package and instance ID."""
    return self._json_output(['describe', pkg_name, '--version', instance_id])

  def get_available_versions(self, name):
    """List available versions of the asset."""
    pkg_name = CIPD_PACKAGE_NAME_TMPL % name
    versions = []
    for instance_id in self._search(pkg_name):
      details = self._describe(pkg_name, instance_id)
      for tag in details.get('tags'):
        tag_name = tag.get('tag', '')
        if tag_name.startswith(TAG_VERSION_PREFIX):
          trimmed = tag_name[len(TAG_VERSION_PREFIX):]
          try:
            versions.append(int(trimmed))
          except ValueError:
            raise ValueError('Found package instance with invalid version '
                             'tag: %s' % tag_name)
    versions.sort()
    return versions

  def upload(self, name, version, target_dir, extra_tags=None):
    """Create a CIPD package."""
    cmd = [
        'create',
        '--name', CIPD_PACKAGE_NAME_TMPL % name,
        '--in', target_dir,
        '--tag', TAG_PROJECT_SKIA,
        '--tag', TAG_VERSION_TMPL % version,
        '--compression-level', '1',
        '-verification-timeout', '30m0s',
    ]
    if extra_tags:
      for tag in extra_tags:
        cmd.extend(['--tag', tag])
    self._run(cmd)

  def download(self, name, version, target_dir):
    """Download a CIPD package."""
    pkg_name = CIPD_PACKAGE_NAME_TMPL % name
    version_tag = TAG_VERSION_TMPL % version
    target_dir = os.path.abspath(target_dir)
    with utils.tmp_dir():
      infile = os.path.join(os.getcwd(), 'input')
      with open(infile, 'w') as f:
        f.write('%s %s' % (pkg_name, version_tag))
      self._run([
          'ensure',
          '--root', target_dir,
          '--list', infile,
      ])

  def delete_contents(self, name):
    """Delete data for the given asset."""
    self._run(['pkg-delete', CIPD_PACKAGE_NAME_TMPL % name])

class GSStore(object):
  """Wrapper object for interacting with Google Storage."""
  def __init__(self, gsutil=None, bucket=DEFAULT_GS_BUCKET,
               service_account_json=None):
    if gsutil:
      gsutil = os.path.abspath(gsutil)
    else:
      gsutils = subprocess.check_output([
          utils.WHICH, 'gsutil']).rstrip().splitlines()
      for g in gsutils:
        ok = True
        try:
          subprocess.check_call([g, 'version'])
        except OSError:
          ok = False
        if ok:
          gsutil = g
          break
    self._gsutil = [gsutil]
    if gsutil.endswith('.py'):
      self._gsutil = ['python', gsutil]
    if service_account_json:
      sa = os.path.abspath(service_account_json)
      self._gsutil += ['-o', 'Credentials:gs_service_key_file=' + sa]
    self._gs_bucket = bucket

  def copy(self, src, dst):
    """Copy src to dst."""
    subprocess.check_call(self._gsutil + ['cp', src, dst])

  def list(self, path):
    """List objects in the given path."""
    try:
      return subprocess.check_output(self._gsutil + ['ls', path]).splitlines()
    except subprocess.CalledProcessError:
      # If the prefix does not exist, we'll get an error, which is okay.
      return []

  def get_available_versions(self, name):
    """Return the existing version numbers for the asset."""
    files = self.list(GS_SUBDIR_TMPL % (self._gs_bucket, name))
    bnames = [os.path.basename(f) for f in files]
    suffix = '.zip'
    versions = [int(f[:-len(suffix)]) for f in bnames if f.endswith(suffix)]
    versions.sort()
    return versions

  # pylint: disable=unused-argument
  def upload(self, name, version, target_dir, extra_tags=None):
    """Upload to GS."""
    target_dir = os.path.abspath(target_dir)
    with utils.tmp_dir():
      zip_file = os.path.join(os.getcwd(), '%d.zip' % version)
      zip_utils.zip(target_dir, zip_file, blacklist=ZIP_BLACKLIST)
      gs_path = GS_PATH_TMPL % (GS_SUBDIR_TMPL % (self._gs_bucket, name),
                                str(version))
      self.copy(zip_file, gs_path)

  def download(self, name, version, target_dir):
    """Download from GS."""
    gs_path = GS_PATH_TMPL % (GS_SUBDIR_TMPL % (self._gs_bucket, name),
                              str(version))
    target_dir = os.path.abspath(target_dir)
    with utils.tmp_dir():
      zip_file = os.path.join(os.getcwd(), '%d.zip' % version)
      self.copy(gs_path, zip_file)
      zip_utils.unzip(zip_file, target_dir)

  def delete_contents(self, name):
    """Delete data for the given asset."""
    gs_path = GS_SUBDIR_TMPL % (self._gs_bucket, name)
    attempt_delete = True
    try:
      subprocess.check_call(self._gsutil + ['ls', gs_path])
    except subprocess.CalledProcessError:
      attempt_delete = False
    if attempt_delete:
      subprocess.check_call(self._gsutil + ['rm', '-rf', gs_path])

class MultiStore(object):
  """Wrapper object which uses CIPD as the primary store and GS for backup."""
  def __init__(self, cipd_url=DEFAULT_CIPD_SERVICE_URL,
               service_account_json=None,
               gsutil=None, gs_bucket=DEFAULT_GS_BUCKET):
    self._cipd = CIPDStore(cipd_url=cipd_url,
                           service_account_json=service_account_json)
    self._gs = GSStore(gsutil=gsutil, bucket=gs_bucket,
                       service_account_json=service_account_json)

  def get_available_versions(self, name):
    return self._cipd.get_available_versions(name)

  def upload(self, name, version, target_dir, extra_tags=None):
    self._cipd.upload(name, version, target_dir, extra_tags=extra_tags)
    self._gs.upload(name, version, target_dir, extra_tags=extra_tags)

  def download(self, name, version, target_dir):
    self._gs.download(name, version, target_dir)

  def delete_contents(self, name):
    self._cipd.delete_contents(name)
    self._gs.delete_contents(name)

def _prompt(prompt):
  """Prompt for input, return result."""
  return raw_input(prompt)

class Asset(object):
  def __init__(self, name, store):
    self._store = store
    self._name = name
    self._dir = os.path.join(ASSETS_DIR, self._name)

  @property
  def version_file(self):
    """Return the path to the version file for this asset."""
    return os.path.join(self._dir, VERSION_FILENAME)

  def get_current_version(self):
    """Obtain the current version of the asset."""
    if not os.path.isfile(self.version_file):
      return -1
    with open(self.version_file) as f:
      return int(f.read())

  def get_available_versions(self):
    """Return the existing version numbers for this asset."""
    return self._store.get_available_versions(self._name)

  def get_next_version(self):
    """Find the next available version number for the asset."""
    versions = self.get_available_versions()
    if len(versions) == 0:
      return 0
    return versions[-1] + 1

  def download_version(self, version, target_dir):
    """Download the specified version of the asset."""
    self._store.download(self._name, version, target_dir)

  def download_current_version(self, target_dir):
    """Download the version of the asset specified in its version file."""
    v = self.get_current_version()
    self.download_version(v, target_dir)

  def upload_new_version(self, target_dir, commit=False, extra_tags=None):
    """Upload a new version and update the version file for the asset."""
    version = self.get_next_version()
    self._store.upload(self._name, version, target_dir, extra_tags=extra_tags)

    def _write_version():
      with open(self.version_file, 'w') as f:
        f.write(str(version))
      subprocess.check_call([utils.GIT, 'add', self.version_file])

    with utils.chdir(SKIA_DIR):
      if commit:
        with utils.git_branch():
          _write_version()
          subprocess.check_call([
              utils.GIT, 'commit', '-m', 'Update %s version' % self._name])
          subprocess.check_call([utils.GIT, 'cl', 'upload', '--bypass-hooks'])
      else:
        _write_version()

  @classmethod
  def add(cls, name, store):
    """Add an asset."""
    asset = cls(name, store)
    if os.path.isdir(asset._dir):
      raise Exception('Asset %s already exists!' % asset._name)

    print 'Creating asset in %s' % asset._dir
    os.mkdir(asset._dir)
    def copy_script(script):
      src = os.path.join(ASSETS_DIR, 'scripts', script)
      dst = os.path.join(asset._dir, script)
      print 'Creating %s' % dst
      shutil.copy(src, dst)
      subprocess.check_call([utils.GIT, 'add', dst])

    for script in ('download.py', 'upload.py', 'common.py'):
      copy_script(script)
    resp = _prompt('Add script to automate creation of this asset? (y/n) ')
    if resp == 'y':
      copy_script('create.py')
      copy_script('create_and_upload.py')
      print 'You will need to add implementation to the creation script.'
    print 'Successfully created asset %s.' % asset._name
    return asset

  def remove(self, remove_in_store=False):
    """Remove this asset."""
    # Ensure that the asset exists.
    if not os.path.isdir(self._dir):
      raise Exception('Asset %s does not exist!' % self._name)

    # Cleanup the store.
    if remove_in_store:
      self._store.delete_contents(self._name)

    # Remove the asset.
    subprocess.check_call([utils.GIT, 'rm', '-rf', self._dir])
    if os.path.isdir(self._dir):
      shutil.rmtree(self._dir)
<|endoftext|>


<commit_before>import sys
sys.path.insert(0,".")
import unittest
import neuroml
import neuroml.writers as writers
import PyOpenWorm
from PyOpenWorm import *
import test_data as TD
import networkx
import rdflib
import rdflib as R
import pint as Q
import os
import subprocess as SP
import subprocess
import tempfile
import doctest

from glob import glob

USE_BINARY_DB = False
BINARY_DB = "OpenWormData/worm.db"
TEST_CONFIG = "tests/default_test.conf"
try:
    import bsddb
    has_bsddb = True
except ImportError:
    has_bsddb = False

try:
    import numpy
    has_numpy = True
except ImportError:
    has_numpy = False

namespaces = { "rdf" : "http://www.w3.org/1999/02/22-rdf-syntax-ns#" }

def clear_graph(graph):
    graph.update("CLEAR ALL")

def make_graph(size=100):
    """ Make an rdflib graph """
    g = R.Graph()
    for i in range(size):
        s = rdflib.URIRef("http://somehost.com/s"+str(i))
        p = rdflib.URIRef("http://somehost.com/p"+str(i))
        o = rdflib.URIRef("http://somehost.com/o"+str(i))
        g.add((s,p,o))
    return g

def delete_zodb_data_store(path):
    os.unlink(path)
    os.unlink(path + '.index')
    os.unlink(path + '.tmp')
    os.unlink(path + '.lock')

class DataIntegrityTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        import csv

        cls.neurons = [] #array that holds the names of the 302 neurons at class-level scope

        if not USE_BINARY_DB:
            PyOpenWorm.connect(conf=Data()) # Connect for integrity tests that use PyOpenWorm functions
            cls.g = PyOpenWorm.config('rdf.graph') # declare class-level scope for the database
            cls.g.parse("OpenWormData/WormData.n3", format="n3") # load in the database
        else:
            conf = Configure(**{ "rdf.source" : "ZODB", "rdf.store_conf" : BINARY_DB })
            PyOpenWorm.connect(conf=conf)
            cls.g = PyOpenWorm.config('rdf.graph')

        #grab the list of the names of the 302 neurons

        csvfile = open('OpenWormData/aux_data/neurons.csv', 'r')
        reader = csv.reader(csvfile, delimiter=';', quotechar='|')

        for row in reader:
            if len(row[0]) > 0: # Only saves valid neuron names
                cls.neurons.append(row[0])
    @classmethod
    def tearDownClass(cls):
        PyOpenWorm.disconnect()

    def test_correct_neuron_number(self):
        """
        This test verifies that the worm model has exactly 302 neurons.
        """
        net = PyOpenWorm.Worm().get_neuron_network()
        self.assertEqual(302, len(set(net.neurons())))

    def test_TH_neuropeptide_neuron_list(self):
        """
        This test verifies that the set of neurons which contain the
        neuropeptide TH is correct (the list is given below).
        """
        neuronlist = PyOpenWorm.Neuron()
        neuronlist.neuropeptide("TH")
        thlist = set(x.name() for x in neuronlist.load())
        self.assertEqual(set(['CEPDR', 'PDER', 'CEPDL', 'PDEL', 'CEPVR', 'CEPVL']), thlist)

    def testUniqueNeuronNode(self):
        """
        There should one and only one unique RDF node for every neuron.  If more than one is present for a given cell name,
        then our data is inconsistent.  If there is not at least one present, then we are missing neurons.
        """

        results = {}
        for n in self.neurons:
            #Create a SPARQL query per neuron that looks for all RDF nodes that have text matching the name of the neuron
            qres = self.g.query('SELECT distinct ?n WHERE {?n ?t ?s . ?s ?p \"' + n + '\" } LIMIT 5')
            results[n] = (len(qres.result), [x[0] for x in qres.result])

        # If there is not only one result back, then there is more than one RDF node.
        more_than_one = [(x, results[x]) for x in results if results[x][0] > 1]
        less_than_one = [(x, results[x]) for x in results if results[x][0] < 1]
        self.assertEqual(0, len(more_than_one), "Some neurons have more than 1 node: " + "\n".join(str(x) for x in more_than_one))
        self.assertEqual(0, len(less_than_one), "Some neurons have no node: " + "\n".join(str(x) for x in less_than_one))

    def testNeuronsHaveTypes(self):
        """
        Every Neuron should have a non-blank type
        """
        results = set()
        for n in self.neurons:
            qres = self.g.query('SELECT ?v WHERE { ?s <http://openworm.org/entities/SimpleProperty/value> \"' + n + '\". ' #per node ?s that has the name of a neuron associated
                                + '?k <http://openworm.org/entities/Cell/name> ?s .'
                                + '?k <http://openworm.org/entities/Neuron/type> ?o .' #look up its listed type ?o
                                + '?o <http://openworm.org/entities/SimpleProperty/value> ?v } ' #for that type ?o, get its property ?tp and its value ?v
                                )
            for x in qres:
                v = x[0]
                if isinstance(v,R.Literal):
                    results.add(n)

        # NOTE: Neurons ALNL, CANL, CANR, ALNR have unknown function and type
        self.assertEqual(len(results), len(self.neurons) - 4, "Some neurons are missing a type: {}".format(set(self.neurons) - results))

    def test_neuron_GJ_degree(self):
        """ Get the number of gap junctions from a networkx representation """
        self.assertEqual(PyOpenWorm.Neuron(name='AVAL').GJ_degree(), 40)

    def test_neuron_Syn_degree(self):
        """ Get the number of chemical synapses from a networkx representation """
        self.assertEqual(PyOpenWorm.Neuron(name='AVAL').Syn_degree(), 90)

    @unittest.skip("have not yet defined asserts")
    def testWhatNodesGetTypeInfo(self):
        qres = self.g.query('SELECT ?o ?p ?s WHERE {'
                                + '?o <http://openworm.org/entities/SimpleProperty/value> "motor". '
                                  '?o ?p ?s} ' #for that type ?o, get its value ?v
                                + 'LIMIT 10')
        for row in qres.result:
            print row

    def test_compare_to_xls(self):
        """ Compare the PyOpenWorm connections to the data in the spreadsheet """
        SAMPLE_CELL = 'AVAL'
        xls_conns = []
        pow_conns = []

        #QUERY TO GET ALL CONNECTIONS WHERE SAMPLE_CELL IS ON THE PRE SIDE
        qres = self.g.query("""SELECT ?post_name ?type (STR(?num) AS ?numval) WHERE {
                               #############################################################
                               # Find connections that have the ?pre_name as our passed in value
                               #############################################################
                               ?pre_namenode <http://openworm.org/entities/SimpleProperty/value> \'"""
                               + SAMPLE_CELL +
                               """\'.
                               ?pre_cell <http://openworm.org/entities/Cell/name> ?pre_namenode.
                               ?pre <http://openworm.org/entities/SimpleProperty/value> ?pre_cell.
                               ?conn <http://openworm.org/entities/Connection/pre_cell> ?pre.

                               #############################################################
                               # Find all the cells that are on the post side of those
                               #  connections and bind their names to ?post_name
                               #############################################################
                               ?conn <http://openworm.org/entities/Connection/post_cell> ?post.
                               ?post <http://openworm.org/entities/SimpleProperty/value> ?post_cell.
                               ?post_cell <http://openworm.org/entities/Cell/name> ?post_namenode.
                               ?post_namenode <http://openworm.org/entities/SimpleProperty/value> ?post_name.

                               ############################################################
                               # Go find the type of the connection and bind to ?type
                               #############################################################
                               ?conn <http://openworm.org/entities/Connection/syntype> ?syntype_node.
                               ?syntype_node <http://openworm.org/entities/SimpleProperty/value> ?type.

                               ############################################################
                               # Go find the number of the connection and bind to ?num
                               ############################################################
                               ?conn <http://openworm.org/entities/Connection/number> ?number_node.
                               ?number_node <http://openworm.org/entities/SimpleProperty/value> ?num.

                               ############################################################
                               # Filter out any ?pre_names or ?post_names that aren't literals
                               ############################################################
                               FILTER(isLiteral(?post_name))}""")
        def ff(x):
            return str(x.value)
        for line in qres.result:
            t = list(map(ff, line))
            t.insert(0,SAMPLE_CELL) #Insert sample cell name into the result set after the fact
            pow_conns.append(t)

        #QUERY TO GET ALL CONNECTIONS WHERE SAMPLE_CELL IS ON THE *POST* SIDE
        qres = self.g.query("""SELECT ?pre_name ?type (STR(?num) AS ?numval) WHERE {
                               #############################################################
                               # Find connections that have the ?post_name as our passed in value
                               #############################################################
                               ?post_namenode <http://openworm.org/entities/SimpleProperty/value> \'"""
                               + SAMPLE_CELL +
                               """\'.
                               ?post_cell <http://openworm.org/entities/Cell/name> ?post_namenode.
                               ?post <http://openworm.org/entities/SimpleProperty/value> ?post_cell.
                               ?conn <http://openworm.org/entities/Connection/post_cell> ?post.

                               #############################################################
                               # Find all the cells that are on the pre side of those
                               #  connections and bind their names to ?pre_name
                               #############################################################
                               ?conn <http://openworm.org/entities/Connection/pre_cell> ?pre.
                               ?pre <http://openworm.org/entities/SimpleProperty/value> ?pre_cell.
                               ?pre_cell <http://openworm.org/entities/Cell/name> ?pre_namenode.
                               ?pre_namenode <http://openworm.org/entities/SimpleProperty/value> ?pre_name.

                               ############################################################
                               # Go find the type of the connection and bind to ?type
                               #############################################################
                               ?conn <http://openworm.org/entities/Connection/syntype> ?syntype_node.
                               ?syntype_node <http://openworm.org/entities/SimpleProperty/value> ?type.

                               ############################################################
                               # Go find the number of the connection and bind to ?num
                               ############################################################
                               ?conn <http://openworm.org/entities/Connection/number> ?number_node.
                               ?number_node <http://openworm.org/entities/SimpleProperty/value> ?num.

                               ############################################################
                               # Filter out any ?pre_names or ?post_names that aren't literals
                               ############################################################
                               FILTER(isLiteral(?pre_name))}""")
        for line in qres.result:
            t = list(map(ff, line))
            t.insert(1,SAMPLE_CELL) #Insert sample cell name into the result set after the fact
            pow_conns.append(t)

        #Get connections from the sheet
        import xlrd
        combining_dict = {}
        # 's' is the workbook sheet
        s = xlrd.open_workbook('OpenWormData/aux_data/NeuronConnect.xls').sheets()[0]
        for row in range(1, s.nrows):
            if s.cell(row, 2).value in ('S', 'Sp', 'EJ') and 'AVAL' in [s.cell(row, 0).value, s.cell(row, 1).value]:
                #We're not going to include 'receives' ('R', 'Rp') since they're just the inverse of 'sends'
                #Also omitting 'NMJ' for the time being (no model in db)
                pre = s.cell(row, 0).value
                post = s.cell(row, 1).value
                num = int(s.cell(row, 3).value)
                if s.cell(row, 2).value == 'EJ':
                    syntype = 'gapJunction'
                elif s.cell(row, 2).value in ('S', 'Sp'):
                    syntype = 'send'

                # Add them to a dict to make sure Sends ('S') and Send-polys ('Sp') are summed.
                # keying by connection pairs as a string (e.g. 'SDQL,AVAL,send').
                # values are lists if the form [pre, post, number, syntype].
                string_key = '{},{},{}'.format(pre, post, syntype)
                if string_key in combining_dict.keys():
                    # if key already there, add to number
                    num += int(combining_dict[string_key][3])

                combining_dict[string_key] = [str(pre), str(post), str(syntype), str(int(num))]

        xls_conns = combining_dict.values()

        #assert that these two sorted lists are the same
        #using sorted lists because Set() removes multiples

        self.maxDiff = None
        self.assertEqual(sorted(pow_conns), sorted(xls_conns))
<commit_msg>Normalize names in synapse test<commit_after>import sys
sys.path.insert(0,".")
import unittest
import neuroml
import neuroml.writers as writers
import PyOpenWorm
from PyOpenWorm import *
import test_data as TD
import networkx
import rdflib
import rdflib as R
import pint as Q
import os
import subprocess as SP
import subprocess
import tempfile
import doctest

from glob import glob

USE_BINARY_DB = False
BINARY_DB = "OpenWormData/worm.db"
TEST_CONFIG = "tests/default_test.conf"
try:
    import bsddb
    has_bsddb = True
except ImportError:
    has_bsddb = False

try:
    import numpy
    has_numpy = True
except ImportError:
    has_numpy = False

namespaces = { "rdf" : "http://www.w3.org/1999/02/22-rdf-syntax-ns#" }

def clear_graph(graph):
    graph.update("CLEAR ALL")

def make_graph(size=100):
    """ Make an rdflib graph """
    g = R.Graph()
    for i in range(size):
        s = rdflib.URIRef("http://somehost.com/s"+str(i))
        p = rdflib.URIRef("http://somehost.com/p"+str(i))
        o = rdflib.URIRef("http://somehost.com/o"+str(i))
        g.add((s,p,o))
    return g

def delete_zodb_data_store(path):
    os.unlink(path)
    os.unlink(path + '.index')
    os.unlink(path + '.tmp')
    os.unlink(path + '.lock')

class DataIntegrityTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        import csv

        cls.neurons = [] #array that holds the names of the 302 neurons at class-level scope

        if not USE_BINARY_DB:
            PyOpenWorm.connect(conf=Data()) # Connect for integrity tests that use PyOpenWorm functions
            cls.g = PyOpenWorm.config('rdf.graph') # declare class-level scope for the database
            cls.g.parse("OpenWormData/WormData.n3", format="n3") # load in the database
        else:
            conf = Configure(**{ "rdf.source" : "ZODB", "rdf.store_conf" : BINARY_DB })
            PyOpenWorm.connect(conf=conf)
            cls.g = PyOpenWorm.config('rdf.graph')

        #grab the list of the names of the 302 neurons

        csvfile = open('OpenWormData/aux_data/neurons.csv', 'r')
        reader = csv.reader(csvfile, delimiter=';', quotechar='|')

        for row in reader:
            if len(row[0]) > 0: # Only saves valid neuron names
                cls.neurons.append(row[0])
    @classmethod
    def tearDownClass(cls):
        PyOpenWorm.disconnect()

    def test_correct_neuron_number(self):
        """
        This test verifies that the worm model has exactly 302 neurons.
        """
        net = PyOpenWorm.Worm().get_neuron_network()
        self.assertEqual(302, len(set(net.neurons())))

    def test_TH_neuropeptide_neuron_list(self):
        """
        This test verifies that the set of neurons which contain the
        neuropeptide TH is correct (the list is given below).
        """
        neuronlist = PyOpenWorm.Neuron()
        neuronlist.neuropeptide("TH")
        thlist = set(x.name() for x in neuronlist.load())
        self.assertEqual(set(['CEPDR', 'PDER', 'CEPDL', 'PDEL', 'CEPVR', 'CEPVL']), thlist)

    def testUniqueNeuronNode(self):
        """
        There should one and only one unique RDF node for every neuron.  If more than one is present for a given cell name,
        then our data is inconsistent.  If there is not at least one present, then we are missing neurons.
        """

        results = {}
        for n in self.neurons:
            #Create a SPARQL query per neuron that looks for all RDF nodes that have text matching the name of the neuron
            qres = self.g.query('SELECT distinct ?n WHERE {?n ?t ?s . ?s ?p \"' + n + '\" } LIMIT 5')
            results[n] = (len(qres.result), [x[0] for x in qres.result])

        # If there is not only one result back, then there is more than one RDF node.
        more_than_one = [(x, results[x]) for x in results if results[x][0] > 1]
        less_than_one = [(x, results[x]) for x in results if results[x][0] < 1]
        self.assertEqual(0, len(more_than_one), "Some neurons have more than 1 node: " + "\n".join(str(x) for x in more_than_one))
        self.assertEqual(0, len(less_than_one), "Some neurons have no node: " + "\n".join(str(x) for x in less_than_one))

    def testNeuronsHaveTypes(self):
        """
        Every Neuron should have a non-blank type
        """
        results = set()
        for n in self.neurons:
            qres = self.g.query('SELECT ?v WHERE { ?s <http://openworm.org/entities/SimpleProperty/value> \"' + n + '\". ' #per node ?s that has the name of a neuron associated
                                + '?k <http://openworm.org/entities/Cell/name> ?s .'
                                + '?k <http://openworm.org/entities/Neuron/type> ?o .' #look up its listed type ?o
                                + '?o <http://openworm.org/entities/SimpleProperty/value> ?v } ' #for that type ?o, get its property ?tp and its value ?v
                                )
            for x in qres:
                v = x[0]
                if isinstance(v,R.Literal):
                    results.add(n)

        # NOTE: Neurons ALNL, CANL, CANR, ALNR have unknown function and type
        self.assertEqual(len(results), len(self.neurons) - 4, "Some neurons are missing a type: {}".format(set(self.neurons) - results))

    def test_neuron_GJ_degree(self):
        """ Get the number of gap junctions from a networkx representation """
        self.assertEqual(PyOpenWorm.Neuron(name='AVAL').GJ_degree(), 40)

    def test_neuron_Syn_degree(self):
        """ Get the number of chemical synapses from a networkx representation """
        self.assertEqual(PyOpenWorm.Neuron(name='AVAL').Syn_degree(), 90)

    @unittest.skip("have not yet defined asserts")
    def testWhatNodesGetTypeInfo(self):
        qres = self.g.query('SELECT ?o ?p ?s WHERE {'
                                + '?o <http://openworm.org/entities/SimpleProperty/value> "motor". '
                                  '?o ?p ?s} ' #for that type ?o, get its value ?v
                                + 'LIMIT 10')
        for row in qres.result:
            print row

    def test_compare_to_xls(self):
        """ Compare the PyOpenWorm connections to the data in the spreadsheet """
        SAMPLE_CELL = 'AVAL'
        xls_conns = []
        pow_conns = []

        #QUERY TO GET ALL CONNECTIONS WHERE SAMPLE_CELL IS ON THE PRE SIDE
        qres = self.g.query("""SELECT ?post_name ?type (STR(?num) AS ?numval) WHERE {
                               #############################################################
                               # Find connections that have the ?pre_name as our passed in value
                               #############################################################
                               ?pre_namenode <http://openworm.org/entities/SimpleProperty/value> \'"""
                               + SAMPLE_CELL +
                               """\'.
                               ?pre_cell <http://openworm.org/entities/Cell/name> ?pre_namenode.
                               ?pre <http://openworm.org/entities/SimpleProperty/value> ?pre_cell.
                               ?conn <http://openworm.org/entities/Connection/pre_cell> ?pre.

                               #############################################################
                               # Find all the cells that are on the post side of those
                               #  connections and bind their names to ?post_name
                               #############################################################
                               ?conn <http://openworm.org/entities/Connection/post_cell> ?post.
                               ?post <http://openworm.org/entities/SimpleProperty/value> ?post_cell.
                               ?post_cell <http://openworm.org/entities/Cell/name> ?post_namenode.
                               ?post_namenode <http://openworm.org/entities/SimpleProperty/value> ?post_name.

                               ############################################################
                               # Go find the type of the connection and bind to ?type
                               #############################################################
                               ?conn <http://openworm.org/entities/Connection/syntype> ?syntype_node.
                               ?syntype_node <http://openworm.org/entities/SimpleProperty/value> ?type.

                               ############################################################
                               # Go find the number of the connection and bind to ?num
                               ############################################################
                               ?conn <http://openworm.org/entities/Connection/number> ?number_node.
                               ?number_node <http://openworm.org/entities/SimpleProperty/value> ?num.

                               ############################################################
                               # Filter out any ?pre_names or ?post_names that aren't literals
                               ############################################################
                               FILTER(isLiteral(?post_name))}""")
        def ff(x):
            return str(x.value)
        for line in qres.result:
            t = list(map(ff, line))
            t.insert(0,SAMPLE_CELL) #Insert sample cell name into the result set after the fact
            pow_conns.append(t)

        #QUERY TO GET ALL CONNECTIONS WHERE SAMPLE_CELL IS ON THE *POST* SIDE
        qres = self.g.query("""SELECT ?pre_name ?type (STR(?num) AS ?numval) WHERE {
                               #############################################################
                               # Find connections that have the ?post_name as our passed in value
                               #############################################################
                               ?post_namenode <http://openworm.org/entities/SimpleProperty/value> \'"""
                               + SAMPLE_CELL +
                               """\'.
                               ?post_cell <http://openworm.org/entities/Cell/name> ?post_namenode.
                               ?post <http://openworm.org/entities/SimpleProperty/value> ?post_cell.
                               ?conn <http://openworm.org/entities/Connection/post_cell> ?post.

                               #############################################################
                               # Find all the cells that are on the pre side of those
                               #  connections and bind their names to ?pre_name
                               #############################################################
                               ?conn <http://openworm.org/entities/Connection/pre_cell> ?pre.
                               ?pre <http://openworm.org/entities/SimpleProperty/value> ?pre_cell.
                               ?pre_cell <http://openworm.org/entities/Cell/name> ?pre_namenode.
                               ?pre_namenode <http://openworm.org/entities/SimpleProperty/value> ?pre_name.

                               ############################################################
                               # Go find the type of the connection and bind to ?type
                               #############################################################
                               ?conn <http://openworm.org/entities/Connection/syntype> ?syntype_node.
                               ?syntype_node <http://openworm.org/entities/SimpleProperty/value> ?type.

                               ############################################################
                               # Go find the number of the connection and bind to ?num
                               ############################################################
                               ?conn <http://openworm.org/entities/Connection/number> ?number_node.
                               ?number_node <http://openworm.org/entities/SimpleProperty/value> ?num.

                               ############################################################
                               # Filter out any ?pre_names or ?post_names that aren't literals
                               ############################################################
                               FILTER(isLiteral(?pre_name))}""")
        for line in qres.result:
            t = list(map(ff, line))
            t.insert(1,SAMPLE_CELL) #Insert sample cell name into the result set after the fact
            pow_conns.append(t)

        #get connections from the sheet
        import re
        search_string = re.compile(r'\w+[0]+[1-9]+')
        replace_string = re.compile(r'[0]+')

        def normalize(name):
            # normalize neuron names to match those used at other points
            # see #137 for elaboration
            # if there are zeroes in the middle of a name, remove them
            if re.match(search_string, name):
                name = replace_string.sub('', name)
            return name

        import xlrd
        combining_dict = {}
        # 's' is the workbook sheet
        s = xlrd.open_workbook('openwormdata/aux_data/neuronconnect.xls').sheets()[0]
        for row in range(1, s.nrows):
            if s.cell(row, 2).value in ('S', 'Sp', 'EJ') and SAMPLE_CELL in [s.cell(row, 0).value, s.cell(row, 1).value]:
                #we're not going to include 'receives' ('r', 'rp') since they're just the inverse of 'sends'
                #also omitting 'nmj' for the time being (no model in db)
                pre = normalize(s.cell(row, 0).value)
                post = normalize(s.cell(row, 1).value)
                num = int(s.cell(row, 3).value)
                if s.cell(row, 2).value == 'EJ':
                    syntype = 'gapJunction'
                elif s.cell(row, 2).value in ('S', 'Sp'):
                    syntype = 'send'

                # add them to a dict to make sure sends ('s') and send-polys ('sp') are summed.
                # keying by connection pairs as a string (e.g. 'sdql,aval,send').
                # values are lists if the form [pre, post, number, syntype].
                string_key = '{},{},{}'.format(pre, post, syntype)
                if string_key in combining_dict.keys():
                    # if key already there, add to number
                    num += int(combining_dict[string_key][3])

                combining_dict[string_key] = [str(pre), str(post), str(syntype), str(int(num))]

        xls_conns = combining_dict.values()

        #assert that these two sorted lists are the same
        #using sorted lists because Set() removes multiples

        self.maxDiff = None
        self.assertEqual(sorted(pow_conns), sorted(xls_conns))
<|endoftext|>