<commit_before>"""
Format a set of INDRA Statements into an HTML-formatted report which also
supports curation.
"""
from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str

import re
import uuid
import logging
import itertools

from os.path import abspath, dirname, join, exists, getmtime, sep
from jinja2 import Environment, BaseLoader, TemplateNotFound

from indra.statements import *
from indra.assemblers.english import EnglishAssembler
from indra.databases import get_identifiers_url
from indra.util.statement_presentation import group_and_sort_statements,\
    make_string_from_sort_key

logger = logging.getLogger(__name__)
HERE = dirname(abspath(__file__))

class IndraHTMLLoader(BaseLoader):
    """A home-grown template loader to load the INDRA templates.

    Based on the example found here:
    http://jinja.pocoo.org/docs/2.10/api/#loaders

    Parameters
    ----------
    root_paths : dict
        A dict of strings indicating possible roots for template directories,
        keyed by basename. To be more specific, an entry
        {'indra': '/path/to/module'} would mean that you expect to have
        'indra/template.html' mapped to '/path/to/module/templates/template.html'.
        The default is {None: HERE}.
    """
    native_path = HERE

    def __init__(self, root_paths=None):
        self.root_paths = root_paths
        if root_paths is None:
            self.root_paths = {None: HERE}

    def get_source(self, environment, template):
        path_parts = template.split(sep)
        if len(path_parts) == 1 or not path_parts[0]:
            root = self.root_paths[None]
        else:
            path_parts = path_parts[1:]
            root = self.root_paths[path_parts[0]]

        path = join(root, 'templates', *path_parts)
        if not exists(path):
            raise TemplateNotFound(template)
        mtime = getmtime(path)
        with open(path, 'r') as f:
            source = f.read()
        return source, path, lambda: mtime == getmtime(path)

env = Environment(loader=IndraHTMLLoader())

template = env.get_template('statements_view.html')

class HtmlAssembler(object):
    """Generates an HTML-formatted report from INDRA Statements.

    The HTML report format includes statements formatted in English
    (by the EnglishAssembler), text and metadata for the Evidence
    object associated with each Statement, and a Javascript-based curation
    interface linked to the INDRA database (access permitting). The interface
    allows for curation of statements at the evidence level by letting the
    user specify type of error and (optionally) provide a short description of
    of the error.

    Parameters
    ----------
    statements : Optional[list[indra.statements.Statement]]
        A list of INDRA Statements to be added to the assembler. Statements
        can also be added using the add_statements method after the assembler
        has been instantiated.
    summary_metadata : Optional[dict]
        Dictionary of statement corpus metadata such as that provided by the
        INDRA REST API. Default is None. Each value should be a concise
        summary of O(1), not of order the length of the list, such as the
        evidence totals. The keys should be informative human-readable strings.
    ev_totals : Optional[dict]
        A dictionary of the total evidence available for each
        statement indexed by hash. Default: None
    db_rest_url : Optional[str]
        The URL to a DB REST API to use for links out to further evidence.
        If given, this URL will be prepended to links that load additional
        evidence for a given Statement. One way to obtain this value is from
        the configuration entry indra.config.get_config('INDRA_DB_REST_URL').
        If None, the URLs are constructed as relative links.
        Default: None
    other_scripts : Optional[list]
        A list of links to other scripts to be added to the html document. Used
        in advanced configurations.
    ev_element : Optional[str]
        A extra element that may be placed at the beginning ahead of each line
        of evidence.

    Attributes
    ----------
    statements : list[indra.statements.Statement]
        A list of INDRA Statements to assemble.
    model : str
        The HTML report formatted as a single string.
    metadata : dict
        Dictionary of statement list metadata such as that provided by the
        INDRA REST API.
    ev_totals : dict
        A dictionary of the total evidence available for each
        statement indexed by hash.
    db_rest_url : str
        The URL to a DB REST API.
    """
    def __init__(self, statements=None, summary_metadata=None, ev_totals=None,
                 title='INDRA Results', db_rest_url=None, other_scripts=None,
                 ev_element=None):
        self.title = title
        self.statements = [] if statements is None else statements
        self.metadata = {} if summary_metadata is None \
            else summary_metadata
        self.ev_totals = {} if ev_totals is None else ev_totals
        self.db_rest_url = db_rest_url
        self.other_scripts = [] if other_scripts is None else other_scripts
        self.ev_element = ev_element
        self.model = None

    def add_statements(self, statements):
        """Add a list of Statements to the assembler.

        Parameters
        ----------
        statements : list[indra.statements.Statement]
            A list of INDRA Statements to be added to the assembler.
        """
        self.statements += statements

    def make_model(self):
        """Return the assembled HTML content as a string.

        Returns
        -------
        str
            The assembled HTML as a string.
        """
        stmts_formatted = []
        stmt_rows = group_and_sort_statements(self.statements,
                                              self.ev_totals if self.ev_totals else None)
        for key, verb, stmts in stmt_rows:
            # This will now be ordered by prevalence and entity pairs.
            stmt_info_list = []
            for stmt in stmts:
                stmt_hash = stmt.get_hash(shallow=True)
                ev_list = self._format_evidence_text(stmt)
                english = self._format_stmt_text(stmt)
                if self.ev_totals:
                    total_evidence = self.ev_totals.get(int(stmt_hash), '?')
                    if total_evidence == '?':
                        logger.warning('The hash %s was not found in the '
                                       'evidence totals dict.' % stmt_hash)
                    evidence_count_str = '%s / %s' % (len(ev_list), total_evidence)
                else:
                    evidence_count_str = str(len(ev_list))
                stmt_info_list.append({
                    'hash': stmt_hash,
                    'english': english,
                    'evidence': ev_list,
                    'evidence_count': evidence_count_str})
            short_name = make_string_from_sort_key(key, verb)
            short_name_key = str(uuid.uuid4())
            stmts_formatted.append((short_name, short_name_key, stmt_info_list))
        metadata = {k.replace('_', ' ').title(): v
                    for k, v in self.metadata.items()}
        if self.db_rest_url and not self.db_rest_url.endswith('statements'):
            db_rest_url = self.db_rest_url + '/statements'
        else:
            db_rest_url = '.'
        self.model = template.render(stmt_data=stmts_formatted,
                                     metadata=metadata, title=self.title,
                                     db_rest_url=db_rest_url,
                                     other_scripts=self.other_scripts,
                                     ev_element=self.ev_element)
        return self.model

    def append_warning(self, msg):
        """Append a warning message to the model to expose issues."""
        assert self.model is not None, "You must already have run make_model!"
        addendum = ('\t(CAUTION: %s occurred when '
                    'creating this page.)' % msg)
        self.model = self.model.replace(self.title, self.title + addendum)
        return self.model

    def save_model(self, fname):
        """Save the assembled HTML into a file.

        Parameters
        ----------
        fname : str
            The path to the file to save the HTML into.
        """
        if self.model is None:
            self.make_model()

        with open(fname, 'wb') as fh:
            fh.write(self.model.encode('utf-8'))

    @staticmethod
    def _format_evidence_text(stmt):
        """Returns evidence metadata with highlighted evidence text.

        Parameters
        ----------
        stmt : indra.Statement
            The Statement with Evidence to be formatted.

        Returns
        -------
        list of dicts
            List of dictionaries corresponding to each Evidence object in the
            Statement's evidence list. Each dictionary has keys 'source_api',
            'pmid' and 'text', drawn from the corresponding fields in the
            Evidence objects. The text entry of the dict includes
            `<span>` tags identifying the agents referenced by the Statement.
        """
        def get_role(ag_ix):
            if isinstance(stmt, Complex) or \
               isinstance(stmt, SelfModification) or \
               isinstance(stmt, ActiveForm) or isinstance(stmt, Conversion) or\
               isinstance(stmt, Translocation):
                return 'other'
            else:
                assert len(stmt.agent_list()) == 2, (len(stmt.agent_list()),
                                                     type(stmt))
                return 'subject' if ag_ix == 0 else 'object'

        ev_list = []
        for ix, ev in enumerate(stmt.evidence):
            # Expand the source api to include the sub-database
            if ev.source_api == 'biopax' and \
               'source_sub_id' in ev.annotations and \
               ev.annotations['source_sub_id']:
               source_api = '%s:%s' % (ev.source_api,
                                       ev.annotations['source_sub_id'])
            else:
                source_api = ev.source_api
            # Prepare the evidence text
            if ev.text is None:
                format_text = None
            else:
                indices = []
                for ix, ag in enumerate(stmt.agent_list()):
                    if ag is None:
                        continue
                    # If the statement has been preassembled, it will have
                    # this entry in annotations
                    try:
                        ag_text = ev.annotations['agents']['raw_text'][ix]
                        if ag_text is None:
                            raise KeyError
                    # Otherwise we try to get the agent text from db_refs
                    except KeyError:
                        ag_text = ag.db_refs.get('TEXT')
                    if ag_text is None:
                        continue
                    role = get_role(ix)
                    # Get the tag with the correct badge
                    tag_start = '' % role
                    tag_close = ''
                    # Build up a set of indices
                    indices += [(m.start(), m.start() + len(ag_text),
                                 ag_text, tag_start, tag_close)
                                 for m in re.finditer(re.escape(ag_text),
                                                      ev.text)]
                format_text = tag_text(ev.text, indices)

            ev_list.append({'source_api': source_api,
                            'pmid': ev.pmid,
                            'text_refs': ev.text_refs,
                            'text': format_text,
                            'source_hash': ev.source_hash })

        return ev_list

    @staticmethod
    def _format_stmt_text(stmt):
        # Get the English assembled statement
        ea = EnglishAssembler([stmt])
        english = ea.make_model()
        if not english:
            english = str(stmt)
        indices = []
        for ag in stmt.agent_list():
            if ag is None or not ag.name:
                continue
            url = id_url(ag)
            if url is None:
                continue
            # Build up a set of indices
            tag_start = "" % url
            tag_close = ""
            # FIXME: the EnglishAssembler capitalizes the first letter of
            # each sentence. In some cases this causes no match here
            # and not produce agent links.
            indices += [(m.start(), m.start() + len(ag.name), ag.name,
                         tag_start, tag_close)
                         for m in re.finditer(re.escape(ag.name), english)]
        return tag_text(english, indices)

def id_url(ag):
    # Return identifier URLs in a prioritized order
    for db_name in ('HGNC', 'FPLX', 'UP', 'IP', 'PF', 'NXPFA',
                    'MIRBASEM', 'MIRBASE',
                    'MESH', 'GO',
                    'HMDB', 'PUBCHEM', 'CHEBI',
                    'NCIT',
                    'UN', 'HUME', 'CWMS', 'SOFIA'):
        if db_name in ag.db_refs:
            # Handle a special case where a list of IDs is given
            if isinstance(ag.db_refs[db_name], list):
                db_id = ag.db_refs[db_name][0]
                if db_name == 'CHEBI':
                    if not db_id.startswith('CHEBI'):
                        db_id = 'CHEBI:%s' % db_id
                elif db_name in ('UN', 'HUME'):
                    db_id = db_id[0]
            else:
                db_id = ag.db_refs[db_name]
            return get_identifiers_url(db_name, db_id)

def tag_text(text, tag_info_list):
    """Apply start/end tags to spans of the given text.

    Parameters
    ----------
    text : str
        Text to be tagged
    tag_info_list : list of tuples
        Each tuple refers to a span of the given text. Fields are `(start_ix,
        end_ix, substring, start_tag, close_tag)`, where substring, start_tag,
        and close_tag are strings. If any of the given spans of text overlap,
        the longest span is used.

    Returns
    -------
    str
        String where the specified substrings have been surrounded by the
        given start and close tags.
    """
    # Check to tags for overlap and if there is any, return the subsumed
    # range. Return None if no overlap.
    def overlap(t1, t2):
        if range(max(t1[0], t2[0]), min(t1[1]-1, t2[1]-1)+1):
            if t1[1] - t1[0] >= t2[1] - t2[0]:
                return t2
            else:
                return t1
        else:
            return None
    # Remove subsumed tags
    for t1, t2 in list(itertools.combinations(tag_info_list, 2)):
        subsumed_tag = overlap(t1, t2)
        if subsumed_tag is not None:
            # Delete the subsumed tag from the list
            try:
                tag_ix = tag_info_list.index(subsumed_tag)
                del tag_info_list[tag_ix]
            # Ignore case where tag has already been deleted
            except ValueError:
                pass
    # Sort the indices by their start position
    tag_info_list.sort(key=lambda x: x[0])
    # Now, add the marker text for each occurrence of the strings
    format_text = ''
    start_pos = 0
    for i, j, ag_text, tag_start, tag_close in tag_info_list:
        # Add the text before this agent, if any
        format_text += text[start_pos:i]
        # Add wrapper for this entity
        format_text += tag_start + ag_text + tag_close
        # Now set the next start position
        start_pos = j
    # Add the last section of text
    format_text += text[start_pos:]
    return format_text
<commit_msg>Fix bug in loader.<commit_after>"""
Format a set of INDRA Statements into an HTML-formatted report which also
supports curation.
"""
from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str

import re
import uuid
import logging
import itertools

from os.path import abspath, dirname, join, exists, getmtime, sep
from jinja2 import Environment, BaseLoader, TemplateNotFound

from indra.statements import *
from indra.assemblers.english import EnglishAssembler
from indra.databases import get_identifiers_url
from indra.util.statement_presentation import group_and_sort_statements,\
    make_string_from_sort_key

logger = logging.getLogger(__name__)
HERE = dirname(abspath(__file__))

class IndraHTMLLoader(BaseLoader):
    """A home-grown template loader to load the INDRA templates.

    Based on the example found here:
    http://jinja.pocoo.org/docs/2.10/api/#loaders

    Parameters
    ----------
    root_paths : dict
        A dict of strings indicating possible roots for template directories,
        keyed by basename. To be more specific, an entry
        {'indra': '/path/to/module'} would mean that you expect to have
        'indra/template.html' mapped to '/path/to/module/templates/template.html'.
        The default is {None: HERE}.
    """
    native_path = HERE

    def __init__(self, root_paths=None):
        self.root_paths = root_paths
        if root_paths is None:
            self.root_paths = {None: HERE}

    def get_source(self, environment, template):
        path_parts = template.split(sep)
        if len(path_parts) == 1 or not path_parts[0]:
            root = self.root_paths[None]
        else:
            root = self.root_paths[path_parts[0]]
            path_parts = path_parts[1:]

        path = join(root, 'templates', *path_parts)
        if not exists(path):
            raise TemplateNotFound(template)
        mtime = getmtime(path)
        with open(path, 'r') as f:
            source = f.read()
        return source, path, lambda: mtime == getmtime(path)

env = Environment(loader=IndraHTMLLoader())

template = env.get_template('statements_view.html')

class HtmlAssembler(object):
    """Generates an HTML-formatted report from INDRA Statements.

    The HTML report format includes statements formatted in English
    (by the EnglishAssembler), text and metadata for the Evidence
    object associated with each Statement, and a Javascript-based curation
    interface linked to the INDRA database (access permitting). The interface
    allows for curation of statements at the evidence level by letting the
    user specify type of error and (optionally) provide a short description of
    of the error.

    Parameters
    ----------
    statements : Optional[list[indra.statements.Statement]]
        A list of INDRA Statements to be added to the assembler. Statements
        can also be added using the add_statements method after the assembler
        has been instantiated.
    summary_metadata : Optional[dict]
        Dictionary of statement corpus metadata such as that provided by the
        INDRA REST API. Default is None. Each value should be a concise
        summary of O(1), not of order the length of the list, such as the
        evidence totals. The keys should be informative human-readable strings.
    ev_totals : Optional[dict]
        A dictionary of the total evidence available for each
        statement indexed by hash. Default: None
    db_rest_url : Optional[str]
        The URL to a DB REST API to use for links out to further evidence.
        If given, this URL will be prepended to links that load additional
        evidence for a given Statement. One way to obtain this value is from
        the configuration entry indra.config.get_config('INDRA_DB_REST_URL').
        If None, the URLs are constructed as relative links.
        Default: None
    other_scripts : Optional[list]
        A list of links to other scripts to be added to the html document. Used
        in advanced configurations.
    ev_element : Optional[str]
        A extra element that may be placed at the beginning ahead of each line
        of evidence.

    Attributes
    ----------
    statements : list[indra.statements.Statement]
        A list of INDRA Statements to assemble.
    model : str
        The HTML report formatted as a single string.
    metadata : dict
        Dictionary of statement list metadata such as that provided by the
        INDRA REST API.
    ev_totals : dict
        A dictionary of the total evidence available for each
        statement indexed by hash.
    db_rest_url : str
        The URL to a DB REST API.
    """
    def __init__(self, statements=None, summary_metadata=None, ev_totals=None,
                 title='INDRA Results', db_rest_url=None, other_scripts=None,
                 ev_element=None):
        self.title = title
        self.statements = [] if statements is None else statements
        self.metadata = {} if summary_metadata is None \
            else summary_metadata
        self.ev_totals = {} if ev_totals is None else ev_totals
        self.db_rest_url = db_rest_url
        self.other_scripts = [] if other_scripts is None else other_scripts
        self.ev_element = ev_element
        self.model = None

    def add_statements(self, statements):
        """Add a list of Statements to the assembler.

        Parameters
        ----------
        statements : list[indra.statements.Statement]
            A list of INDRA Statements to be added to the assembler.
        """
        self.statements += statements

    def make_model(self):
        """Return the assembled HTML content as a string.

        Returns
        -------
        str
            The assembled HTML as a string.
        """
        stmts_formatted = []
        stmt_rows = group_and_sort_statements(self.statements,
                                              self.ev_totals if self.ev_totals else None)
        for key, verb, stmts in stmt_rows:
            # This will now be ordered by prevalence and entity pairs.
            stmt_info_list = []
            for stmt in stmts:
                stmt_hash = stmt.get_hash(shallow=True)
                ev_list = self._format_evidence_text(stmt)
                english = self._format_stmt_text(stmt)
                if self.ev_totals:
                    total_evidence = self.ev_totals.get(int(stmt_hash), '?')
                    if total_evidence == '?':
                        logger.warning('The hash %s was not found in the '
                                       'evidence totals dict.' % stmt_hash)
                    evidence_count_str = '%s / %s' % (len(ev_list), total_evidence)
                else:
                    evidence_count_str = str(len(ev_list))
                stmt_info_list.append({
                    'hash': stmt_hash,
                    'english': english,
                    'evidence': ev_list,
                    'evidence_count': evidence_count_str})
            short_name = make_string_from_sort_key(key, verb)
            short_name_key = str(uuid.uuid4())
            stmts_formatted.append((short_name, short_name_key, stmt_info_list))
        metadata = {k.replace('_', ' ').title(): v
                    for k, v in self.metadata.items()}
        if self.db_rest_url and not self.db_rest_url.endswith('statements'):
            db_rest_url = self.db_rest_url + '/statements'
        else:
            db_rest_url = '.'
        self.model = template.render(stmt_data=stmts_formatted,
                                     metadata=metadata, title=self.title,
                                     db_rest_url=db_rest_url,
                                     other_scripts=self.other_scripts,
                                     ev_element=self.ev_element)
        return self.model

    def append_warning(self, msg):
        """Append a warning message to the model to expose issues."""
        assert self.model is not None, "You must already have run make_model!"
        addendum = ('\t(CAUTION: %s occurred when '
                    'creating this page.)' % msg)
        self.model = self.model.replace(self.title, self.title + addendum)
        return self.model

    def save_model(self, fname):
        """Save the assembled HTML into a file.

        Parameters
        ----------
        fname : str
            The path to the file to save the HTML into.
        """
        if self.model is None:
            self.make_model()

        with open(fname, 'wb') as fh:
            fh.write(self.model.encode('utf-8'))

    @staticmethod
    def _format_evidence_text(stmt):
        """Returns evidence metadata with highlighted evidence text.

        Parameters
        ----------
        stmt : indra.Statement
            The Statement with Evidence to be formatted.

        Returns
        -------
        list of dicts
            List of dictionaries corresponding to each Evidence object in the
            Statement's evidence list. Each dictionary has keys 'source_api',
            'pmid' and 'text', drawn from the corresponding fields in the
            Evidence objects. The text entry of the dict includes
            `<span>` tags identifying the agents referenced by the Statement.
        """
        def get_role(ag_ix):
            if isinstance(stmt, Complex) or \
               isinstance(stmt, SelfModification) or \
               isinstance(stmt, ActiveForm) or isinstance(stmt, Conversion) or\
               isinstance(stmt, Translocation):
                return 'other'
            else:
                assert len(stmt.agent_list()) == 2, (len(stmt.agent_list()),
                                                     type(stmt))
                return 'subject' if ag_ix == 0 else 'object'

        ev_list = []
        for ix, ev in enumerate(stmt.evidence):
            # Expand the source api to include the sub-database
            if ev.source_api == 'biopax' and \
               'source_sub_id' in ev.annotations and \
               ev.annotations['source_sub_id']:
               source_api = '%s:%s' % (ev.source_api,
                                       ev.annotations['source_sub_id'])
            else:
                source_api = ev.source_api
            # Prepare the evidence text
            if ev.text is None:
                format_text = None
            else:
                indices = []
                for ix, ag in enumerate(stmt.agent_list()):
                    if ag is None:
                        continue
                    # If the statement has been preassembled, it will have
                    # this entry in annotations
                    try:
                        ag_text = ev.annotations['agents']['raw_text'][ix]
                        if ag_text is None:
                            raise KeyError
                    # Otherwise we try to get the agent text from db_refs
                    except KeyError:
                        ag_text = ag.db_refs.get('TEXT')
                    if ag_text is None:
                        continue
                    role = get_role(ix)
                    # Get the tag with the correct badge
                    tag_start = '' % role
                    tag_close = ''
                    # Build up a set of indices
                    indices += [(m.start(), m.start() + len(ag_text),
                                 ag_text, tag_start, tag_close)
                                 for m in re.finditer(re.escape(ag_text),
                                                      ev.text)]
                format_text = tag_text(ev.text, indices)

            ev_list.append({'source_api': source_api,
                            'pmid': ev.pmid,
                            'text_refs': ev.text_refs,
                            'text': format_text,
                            'source_hash': ev.source_hash })

        return ev_list

    @staticmethod
    def _format_stmt_text(stmt):
        # Get the English assembled statement
        ea = EnglishAssembler([stmt])
        english = ea.make_model()
        if not english:
            english = str(stmt)
        indices = []
        for ag in stmt.agent_list():
            if ag is None or not ag.name:
                continue
            url = id_url(ag)
            if url is None:
                continue
            # Build up a set of indices
            tag_start = "" % url
            tag_close = ""
            # FIXME: the EnglishAssembler capitalizes the first letter of
            # each sentence. In some cases this causes no match here
            # and not produce agent links.
            indices += [(m.start(), m.start() + len(ag.name), ag.name,
                         tag_start, tag_close)
                         for m in re.finditer(re.escape(ag.name), english)]
        return tag_text(english, indices)

def id_url(ag):
    # Return identifier URLs in a prioritized order
    for db_name in ('HGNC', 'FPLX', 'UP', 'IP', 'PF', 'NXPFA',
                    'MIRBASEM', 'MIRBASE',
                    'MESH', 'GO',
                    'HMDB', 'PUBCHEM', 'CHEBI',
                    'NCIT',
                    'UN', 'HUME', 'CWMS', 'SOFIA'):
        if db_name in ag.db_refs:
            # Handle a special case where a list of IDs is given
            if isinstance(ag.db_refs[db_name], list):
                db_id = ag.db_refs[db_name][0]
                if db_name == 'CHEBI':
                    if not db_id.startswith('CHEBI'):
                        db_id = 'CHEBI:%s' % db_id
                elif db_name in ('UN', 'HUME'):
                    db_id = db_id[0]
            else:
                db_id = ag.db_refs[db_name]
            return get_identifiers_url(db_name, db_id)

def tag_text(text, tag_info_list):
    """Apply start/end tags to spans of the given text.

    Parameters
    ----------
    text : str
        Text to be tagged
    tag_info_list : list of tuples
        Each tuple refers to a span of the given text. Fields are `(start_ix,
        end_ix, substring, start_tag, close_tag)`, where substring, start_tag,
        and close_tag are strings. If any of the given spans of text overlap,
        the longest span is used.

    Returns
    -------
    str
        String where the specified substrings have been surrounded by the
        given start and close tags.
    """
    # Check to tags for overlap and if there is any, return the subsumed
    # range. Return None if no overlap.
    def overlap(t1, t2):
        if range(max(t1[0], t2[0]), min(t1[1]-1, t2[1]-1)+1):
            if t1[1] - t1[0] >= t2[1] - t2[0]:
                return t2
            else:
                return t1
        else:
            return None
    # Remove subsumed tags
    for t1, t2 in list(itertools.combinations(tag_info_list, 2)):
        subsumed_tag = overlap(t1, t2)
        if subsumed_tag is not None:
            # Delete the subsumed tag from the list
            try:
                tag_ix = tag_info_list.index(subsumed_tag)
                del tag_info_list[tag_ix]
            # Ignore case where tag has already been deleted
            except ValueError:
                pass
    # Sort the indices by their start position
    tag_info_list.sort(key=lambda x: x[0])
    # Now, add the marker text for each occurrence of the strings
    format_text = ''
    start_pos = 0
    for i, j, ag_text, tag_start, tag_close in tag_info_list:
        # Add the text before this agent, if any
        format_text += text[start_pos:i]
        # Add wrapper for this entity
        format_text += tag_start + ag_text + tag_close
        # Now set the next start position
        start_pos = j
    # Add the last section of text
    format_text += text[start_pos:]
    return format_text
<|endoftext|>


<commit_before>
import math

math.sin(math.pi / 2)

a_list = ['a', 'b', 'c']
a_list.extend(['d', 'e', 'f'])

a_list = ['aa', 'bb', 'cc', 'dd', 'aa']

print("Count: {}".format(a_list.count('aa')))

a_set = {1, 2}
a_set.add(4)

print(a_set)

a_set.update({2, 4, 6, 10, 12, 20})
print(a_set)

a_set.discard(10)

b_set = a_set.copy()

b_set.add(7)
b_set.discard(6)

print(a_set)
print(b_set)

print(b_set.intersection(a_set))

import os

print(os.path.expanduser('~'))

pathname = '/Users/pilgrim/diveintopython3/examples/humansize.py'

(dirname, filename) = os.path.split(pathname)
print(os.path.splitext(filename))

print(os.getcwd())

pattern = "^M?M?"

import re

print(pattern)
print(re.search(pattern, 'M'))

phonePattern = re.compile(r'^(\d{3})-(\d{3})-(\d{4})-(\d+)$')
print(phonePattern.search('800-555-1212-1234').groups())

def rules(rules_filename):
    with open(rules_filename, encoding='utf-8') as pattern_file:
        for line in pattern_file:
            pattern, search, replace = line.split(None, 2)
            yield build_match_and_apply_functions(pattern, search, replace)

def plural(noun, rules_filename='plural5-rules.txt'):
    for matches_rule, apply_rule in rules(rules_filename):
        if matches_rule(noun):
            return apply_rule(noun)
    raise ValueError('no matching rule for {0}'.format(noun))
<commit_msg>[Python] Advanced - Iterators<commit_after>
import math

math.sin(math.pi / 2)

a_list = ['a', 'b', 'c']
a_list.extend(['d', 'e', 'f'])

a_list = ['aa', 'bb', 'cc', 'dd', 'aa']

print("Count: {}".format(a_list.count('aa')))

a_set = {1, 2}
a_set.add(4)

print(a_set)

a_set.update({2, 4, 6, 10, 12, 20})
print(a_set)

a_set.discard(10)

b_set = a_set.copy()

b_set.add(7)
b_set.discard(6)

print(a_set)
print(b_set)

print(b_set.intersection(a_set))

import os

print(os.path.expanduser('~'))

pathname = '/Users/pilgrim/diveintopython3/examples/humansize.py'

(dirname, filename) = os.path.split(pathname)
print(os.path.splitext(filename))

print(os.getcwd())

pattern = "^M?M?"

import re

print(pattern)
print(re.search(pattern, 'M'))

phonePattern = re.compile(r'^(\d{3})-(\d{3})-(\d{4})-(\d+)$')
print(phonePattern.search('800-555-1212-1234').groups())

def rules(rules_filename):
    with open(rules_filename, encoding='utf-8') as pattern_file:
        for line in pattern_file:
            pattern, search, replace = line.split(None, 2)
            yield build_match_and_apply_functions(pattern, search, replace)

def plural(noun, rules_filename='plural5-rules.txt'):
    for matches_rule, apply_rule in rules(rules_filename):
        if matches_rule(noun):
            return apply_rule(noun)
    raise ValueError('no matching rule for {0}'.format(noun))

class Fib:
    '''iterator that yields numbers in the Fibonacci sequence'''

    def __init__(self, max):
        self.max = max

    def __iter__(self):
        self.a = 0
        self.b = 1
        return self

    def __next__(self):
        fib = self.a
        if fib > self.max:
            raise StopIteration
        self.a, self.b = self.b, self.a + self.b
        return fib
<|endoftext|>


<commit_before>import logging
from indra.ontology.standardize import get_standard_name
from indra.ontology.bio import bio_ontology
from indra.statements import *
from .minerva_client import get_ids_to_refs, default_map_name
from .id_mapping import indra_db_refs_from_minerva_refs

logger = logging.getLogger(__name__)

class SifProcessor:
    """Processor that extracts INDRA Statements from SIF strings.

    Parameters
    ----------
    model_id_to_sif_strs : dict
        A dictionary mapping a model ID (int) to a list of strings in SIF
        format. Example: {799: ['csa2 POSITIVE sa9', 'csa11 NEGATIVE sa30']}

    Attributes
    ----------
    statements : list[indra.statements.Statement]
        A list of INDRA Statements extracted from the SIF strings.
    """
    def __init__(self, model_id_to_sif_strs, map_name=default_map_name):
        self.model_id_to_sif_strs = model_id_to_sif_strs
        self.map_name = map_name
        self.statements = []

    def extract_statements(self):
        for model_id, sif_strs in self.model_id_to_sif_strs.items():
            self.statements += self.process_model(model_id, sif_strs)
        logger.info('Got %d total statements from %d models'
                    % (len(self.statements), len(self.model_id_to_sif_strs)))

    def process_model(self, model_id, sif_strs):
        logger.info('Processing model %d' % model_id)
        ids_to_refs, complex_members = get_ids_to_refs(model_id, self.map_name)
        stmts = []
        for sif_str in sif_strs:
            stmt = self.get_stmt(sif_str, ids_to_refs, complex_members,
                                 model_id)
            if stmt:
                stmts.append(stmt)
        logger.info('Got %d statements from model %d' % (len(stmts), model_id))
        return stmts

    def get_stmt(self, sif_str, ids_to_refs, complex_members, model_id):
        if sif_str.startswith('#') or sif_str == '':
            return
        clean_str = sif_str.strip('\n')
        subj_id, rel_type, obj_id = clean_str.split(' ')
        subj = get_agent(subj_id, ids_to_refs, complex_members)
        obj = get_agent(obj_id, ids_to_refs, complex_members)
        if rel_type == 'POSITIVE':
            stmt = Activation(subj, obj)
        elif rel_type == 'NEGATIVE':
            stmt = Inhibition(subj, obj)
        else:
            raise ValueError('Unknown relation type: %s' % rel_type)
        evid = Evidence(source_api='minerva',
                        annotations={'sif_str': sif_str,
                                     'minerva_model_id': model_id})
        stmt.evidence = [evid]
        return stmt

def get_agent(element_id, ids_to_refs, complex_members):
    """Get an agent for a MINERVA element.

    Parameters
    ----------
    element_id : str
        ID of an element used in MINERVA API and raw SIF files.
    ids_to_refs : dict
        A dictionary mapping element IDs to MINERVA provided references. Note
        that this mapping is unique per model (same IDs can be mapped to
        different refs in different models).
    complex_members : dict
        A dictionary mapping element ID of a complex element to element IDs of
        its members.

    Returns
    -------
    agent : indra.statements.agent.Agent
        INDRA agent created from given refs.
    """
    # Get references from MINERVA and filter to accepted namespaces
    accepted_ns = default_ns_order + ['TEXT']
    refs = ids_to_refs.get(element_id)
    filtered_refs = [ref for ref in refs if ref[0] in accepted_ns]
    # If it's a complex and doesn't have complex level grounding
    if element_id in complex_members and len(filtered_refs) == 1:
        # Sort to always have the same main agent
        member_ids = complex_members[element_id]
        agents = [get_agent(member_id, ids_to_refs, complex_members)
                  for member_id in member_ids]
        agents = sorted(agents, key=lambda ag: ag.name)
        # Try to get a FamPlex family
        fam = get_family(agents)
        if fam:
            # Combine TEXT from MINERVA and found FPLX ID
            agent_refs = filtered_refs + [fam]
            return get_agent_from_refs(agent_refs)
        # Otherwise treat a list of agents as an agent with bound conditions
        else:
            main_agent = agents[0]
            if len(agents) > 1:
                for ag in agents[1:]:
                    main_agent.bound_conditions.append(BoundCondition(ag))
            return main_agent
    # Now we have either individual agents or complexes with complex level
    # grounding (e.g. from GO or MESH)
    else:
        return get_agent_from_refs(filtered_refs)

def get_family(agents):
    """Get a FamPlex family if all of its members are given."""
    family_sets = []
    ag_groundings = []
    for ag in agents:
        gr = ag.get_grounding()
        ag_groundings.append(gr)
        parents = bio_ontology.get_parents(*gr)
        families = {p for p in parents if p[0] == 'FPLX'}
        family_sets.append(families)
    common_families = family_sets[0].intersection(*family_sets)
    if not common_families:
        return
    for fam in common_families:
        children = bio_ontology.get_children(*fam)
        # Check if all family members are present
        if set(children) == set(ag_groundings):
            return fam

def get_agent_from_refs(refs):
    """Get an agent given MINERVA refs."""
    db_refs = indra_db_refs_from_minerva_refs(refs)
    name = get_standard_name(db_refs)
    if not name:
        name = db_refs.get('TEXT')
    if name and db_refs:
        return Agent(name, db_refs=db_refs)
<commit_msg>Convert to indra db refs before filtering<commit_after>import logging
from indra.ontology.standardize import get_standard_name
from indra.ontology.bio import bio_ontology
from indra.statements import *
from .minerva_client import get_ids_to_refs, default_map_name
from .id_mapping import indra_db_refs_from_minerva_refs

logger = logging.getLogger(__name__)

class SifProcessor:
    """Processor that extracts INDRA Statements from SIF strings.

    Parameters
    ----------
    model_id_to_sif_strs : dict
        A dictionary mapping a model ID (int) to a list of strings in SIF
        format. Example: {799: ['csa2 POSITIVE sa9', 'csa11 NEGATIVE sa30']}

    Attributes
    ----------
    statements : list[indra.statements.Statement]
        A list of INDRA Statements extracted from the SIF strings.
    """
    def __init__(self, model_id_to_sif_strs, map_name=default_map_name):
        self.model_id_to_sif_strs = model_id_to_sif_strs
        self.map_name = map_name
        self.statements = []

    def extract_statements(self):
        for model_id, sif_strs in self.model_id_to_sif_strs.items():
            self.statements += self.process_model(model_id, sif_strs)
        logger.info('Got %d total statements from %d models'
                    % (len(self.statements), len(self.model_id_to_sif_strs)))

    def process_model(self, model_id, sif_strs):
        logger.info('Processing model %d' % model_id)
        ids_to_refs, complex_members = get_ids_to_refs(model_id, self.map_name)
        stmts = []
        for sif_str in sif_strs:
            stmt = self.get_stmt(sif_str, ids_to_refs, complex_members,
                                 model_id)
            if stmt:
                stmts.append(stmt)
        logger.info('Got %d statements from model %d' % (len(stmts), model_id))
        return stmts

    def get_stmt(self, sif_str, ids_to_refs, complex_members, model_id):
        if sif_str.startswith('#') or sif_str == '':
            return
        clean_str = sif_str.strip('\n')
        subj_id, rel_type, obj_id = clean_str.split(' ')
        subj = get_agent(subj_id, ids_to_refs, complex_members)
        obj = get_agent(obj_id, ids_to_refs, complex_members)
        if rel_type == 'POSITIVE':
            stmt = Activation(subj, obj)
        elif rel_type == 'NEGATIVE':
            stmt = Inhibition(subj, obj)
        else:
            raise ValueError('Unknown relation type: %s' % rel_type)
        evid = Evidence(source_api='minerva',
                        annotations={'sif_str': sif_str,
                                     'minerva_model_id': model_id})
        stmt.evidence = [evid]
        return stmt

def get_agent(element_id, ids_to_refs, complex_members):
    """Get an agent for a MINERVA element.

    Parameters
    ----------
    element_id : str
        ID of an element used in MINERVA API and raw SIF files.
    ids_to_refs : dict
        A dictionary mapping element IDs to MINERVA provided references. Note
        that this mapping is unique per model (same IDs can be mapped to
        different refs in different models).
    complex_members : dict
        A dictionary mapping element ID of a complex element to element IDs of
        its members.

    Returns
    -------
    agent : indra.statements.agent.Agent
        INDRA agent created from given refs.
    """
    # Get references from MINERVA and filter to accepted namespaces
    accepted_ns = default_ns_order + ['TEXT']
    refs = ids_to_refs.get(element_id)
    db_refs = indra_db_refs_from_minerva_refs(refs)
    filtered_refs = {db_ns: db_id for (db_ns, db_id) in db_refs.items()
                     if db_ns in accepted_ns}
    # If it's a complex and doesn't have complex level grounding
    if element_id in complex_members and len(filtered_refs) == 1:
        # Sort to always have the same main agent
        member_ids = complex_members[element_id]
        agents = [get_agent(member_id, ids_to_refs, complex_members)
                  for member_id in member_ids]
        agents = sorted(agents, key=lambda ag: ag.name)
        # Try to get a FamPlex family
        fam = get_family(agents)
        if fam:
            # Combine TEXT from MINERVA and found FPLX ID
            filtered_refs['FPLX'] = fam
            return get_agent_from_refs(filtered_refs)
        # Otherwise treat a list of agents as an agent with bound conditions
        else:
            main_agent = agents[0]
            if len(agents) > 1:
                for ag in agents[1:]:
                    main_agent.bound_conditions.append(BoundCondition(ag))
            return main_agent
    # Now we have either individual agents or complexes with complex level
    # grounding (e.g. from GO, MESH, UNIPROT)
    else:
        return get_agent_from_refs(filtered_refs)

def get_family(agents):
    """Get a FamPlex family if all of its members are given."""
    family_sets = []
    ag_groundings = []
    for ag in agents:
        gr = ag.get_grounding()
        ag_groundings.append(gr)
        parents = bio_ontology.get_parents(*gr)
        families = {p for p in parents if p[0] == 'FPLX'}
        family_sets.append(families)
    common_families = family_sets[0].intersection(*family_sets)
    if not common_families:
        return
    for fam in common_families:
        children = bio_ontology.get_children(*fam)
        # Check if all family members are present
        if set(children) == set(ag_groundings):
            return fam

def get_agent_from_refs(db_refs):
    """Get an agent given its db_refs."""
    name = get_standard_name(db_refs)
    if not name:
        name = db_refs.get('TEXT')
    if name and db_refs:
        return Agent(name, db_refs=db_refs)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import urllib2
import os
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
os.environ['DJANGO_SETTINGS_MODULE'] = "insacloud.settings"
sys.path.append("..")
from services.models import Event
from django.core.files import File
=======
=======
>>>>>>> Reorganisation
sys.path.append("..")
sys.path.append("config")
sys.path.append("services")

<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
#from models import Event
>>>>>>> Reorganisation
=======
=======
>>>>>>> Removent FileDao unused -> integrated in EventDao
os.environ['DJANGO_SETTINGS_MODULE'] = "insacloud.settings"
sys.path.append("..")
from services.models import Event
from django.core.files import File
>>>>>>> Import is fonctionnal: adding to the database works
=======
#from models import Event
>>>>>>> Reorganisation
=======
os.environ['DJANGO_SETTINGS_MODULE'] = "insacloud.settings"
from services.models import Event
from django.core.files import File
>>>>>>> Import is fonctionnal: adding to the database works

def TryAndAddNewEvent(id_source_new,
					  source,
					  date_start,
					  date_end,
					  category,
					  title,
					  location,
					  venue,
					  longitude,
					  latitude,
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> Import is fonctionnal: adding to the database works
=======
>>>>>>> Import is fonctionnal: adding to the database works
					  poster_url):
	try:
		existent = Event.objects.get(id_source=id_source_new)
	except Event.DoesNotExist:
		existent = None

	if 	existent == None:

		from django.core.files import File

		path = "./tmp/"

		if not os.path.exists(path):
			os.makedirs(path)

		filename, file_extension = os.path.splitext(poster_url)

		file = '%s/%s%s' % (path, id_source_new, file_extension)

		response = urllib2.urlopen(poster_url)

		#open the file for writing
		fh = open(file, "w")

		# read from request while writing to file
		fh.write(response.read())
		fh.close()

		event = Event()

		event.id_source = id_source_new
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
		event.source = source
=======
>>>>>>> Import is fonctionnal: adding to the database works
=======
		event.source = source
>>>>>>> ++
=======
>>>>>>> Import is fonctionnal: adding to the database works
=======
		event.source = source
>>>>>>> ++
		event.date_start = date_start
		event.date_end = date_end
		event.category = category
		event.title = title
		event.location = location
		event.venue = venue
		event.longitude = longitude
		event.latitude = latitude

		with open(file, 'r') as f:
			image_file = File(f)
			event.poster.save('%s%s' % (id_source_new, file_extension), image_file, True)

		os.remove(file)

		print "\n\n\nOK--- Event Added to database\n\n\n"

	else:
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
		print "\n\n\nWARNING--- Event already exists (IGNORED)\n\n\n"
=======
=======
>>>>>>> Reorganisation
					  poster):

	# existent = Event.objects.filter(id_source=id_source_new)
	# if 	existent == None:
		# event = Event(id_source_new,
						  # source,
						  # date_start,
						  # date_end,
						  # category,
						  # title,
						  # location,
						  # venue,
						  # longitude,
						  # latitude,
						  # poster)
		# event.save()
		# print "OK--- Event Add to database"

	# else:
		print "WARNING--- Event already exists"
<<<<<<< HEAD

>>>>>>> Reorganisation
=======
		print "\n\n\nWARNING--- Event already exists\n\n\n"
>>>>>>> Import is fonctionnal: adding to the database works
=======
		print "\n\n\nWARNING--- Event already exists (IGNORED)\n\n\n"
>>>>>>> ImportService self runnable
=======

>>>>>>> Reorganisation
=======
		print "\n\n\nWARNING--- Event already exists\n\n\n"
>>>>>>> Import is fonctionnal: adding to the database works
<commit_msg>ImportService self runnable<commit_after>#!/usr/bin/env python
import sys
import urllib2
import os
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
os.environ['DJANGO_SETTINGS_MODULE'] = "insacloud.settings"
sys.path.append("..")
from services.models import Event
from django.core.files import File
=======
=======
>>>>>>> Reorganisation
sys.path.append("..")
sys.path.append("config")
sys.path.append("services")

<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
#from models import Event
>>>>>>> Reorganisation
=======
=======
>>>>>>> Removent FileDao unused -> integrated in EventDao
os.environ['DJANGO_SETTINGS_MODULE'] = "insacloud.settings"
sys.path.append("..")
from services.models import Event
from django.core.files import File
>>>>>>> Import is fonctionnal: adding to the database works
=======
#from models import Event
>>>>>>> Reorganisation
=======
os.environ['DJANGO_SETTINGS_MODULE'] = "insacloud.settings"
from services.models import Event
from django.core.files import File
>>>>>>> Import is fonctionnal: adding to the database works

def TryAndAddNewEvent(id_source_new,
					  source,
					  date_start,
					  date_end,
					  category,
					  title,
					  location,
					  venue,
					  longitude,
					  latitude,
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> Import is fonctionnal: adding to the database works
=======
>>>>>>> Import is fonctionnal: adding to the database works
					  poster_url):
	try:
		existent = Event.objects.get(id_source=id_source_new)
	except Event.DoesNotExist:
		existent = None

	if 	existent == None:

		from django.core.files import File

		path = "./tmp/"

		if not os.path.exists(path):
			os.makedirs(path)

		filename, file_extension = os.path.splitext(poster_url)

		file = '%s/%s%s' % (path, id_source_new, file_extension)

		response = urllib2.urlopen(poster_url)

		#open the file for writing
		fh = open(file, "w")

		# read from request while writing to file
		fh.write(response.read())
		fh.close()

		event = Event()

		event.id_source = id_source_new
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
		event.source = source
=======
>>>>>>> Import is fonctionnal: adding to the database works
=======
		event.source = source
>>>>>>> ++
=======
>>>>>>> Import is fonctionnal: adding to the database works
=======
		event.source = source
>>>>>>> ++
		event.date_start = date_start
		event.date_end = date_end
		event.category = category
		event.title = title
		event.location = location
		event.venue = venue
		event.longitude = longitude
		event.latitude = latitude

		with open(file, 'r') as f:
			image_file = File(f)
			event.poster.save('%s%s' % (id_source_new, file_extension), image_file, True)

		os.remove(file)

		print "\n\n\nOK--- Event Added to database\n\n\n"

	else:
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
<<<<<<< HEAD
		print "\n\n\nWARNING--- Event already exists (IGNORED)\n\n\n"
=======
=======
>>>>>>> Reorganisation
					  poster):

	# existent = Event.objects.filter(id_source=id_source_new)
	# if 	existent == None:
		# event = Event(id_source_new,
						  # source,
						  # date_start,
						  # date_end,
						  # category,
						  # title,
						  # location,
						  # venue,
						  # longitude,
						  # latitude,
						  # poster)
		# event.save()
		# print "OK--- Event Add to database"

	# else:
		print "WARNING--- Event already exists"
<<<<<<< HEAD

>>>>>>> Reorganisation
=======
		print "\n\n\nWARNING--- Event already exists\n\n\n"
>>>>>>> Import is fonctionnal: adding to the database works
=======
		print "\n\n\nWARNING--- Event already exists (IGNORED)\n\n\n"
>>>>>>> ImportService self runnable
=======

>>>>>>> Reorganisation
=======
		print "\n\n\nWARNING--- Event already exists\n\n\n"
>>>>>>> Import is fonctionnal: adding to the database works
=======
		print "\n\n\nWARNING--- Event already exists (IGNORED)\n\n\n"
>>>>>>> ImportService self runnable
<|endoftext|>


<commit_before># TmLibrary - TissueMAPS library for distibuted image analysis routines.
# Copyright (C) 2016  Markus D. Herrmann, University of Zurich and Robin Hafen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
'''Implementation of classes for reading microscope image and metadata files
provided in a format specfic to the
`Yokogawa CellVoyager 7000 <http://www.yokogawa.com/solutions/products-platforms/life-science/high-throughput-cytological-discovery-system/cv7000s/>`_
microscope.
'''
import re
import logging
import bioformats
from natsort import natsorted
from collections import defaultdict
from lxml import etree

from tmlib import utils
from tmlib.workflow.illuminati import stitch
from tmlib.workflow.metaconfig.base import MetadataReader
from tmlib.workflow.metaconfig.base import MetadataHandler
from tmlib.workflow.metaconfig.omexml import XML_DECLARATION

logger = logging.getLogger(__name__)

#: Regular expression pattern to identify image files
IMAGE_FILE_REGEX_PATTERN = r'[^_]+_(?P<w>[A-Z]\d{2})_T(?P<t>\d+)F(?P<s>\d+)L\d+A\d+Z(?P<z>\d+)C(?P<c>\d+)\.'

#: Supported extensions for metadata files
METADATA_FILE_REGEX_PATTERN = r'.*\.(mlf|mrf)$'

class CellvoyagerMetadataHandler(MetadataHandler):

    '''Class for handling metadata specific to the Yokogawa Cellvoyager 7000
    microscope.
    '''

    def __init__(self, omexml_images, omexml_metadata=None):
        '''
        Parameters
        ----------
        omexml_images: Dict[str, bioformats.omexml.OMEXML]
            metadata extracted from microscope image files
        omexml_metadata: bioformats.omexml.OMEXML
            metadata extracted from microscope metadata files
        '''
        super(CellvoyagerMetadataHandler, self).__init__(
            omexml_images, omexml_metadata
        )

    @staticmethod
    def _calculate_coordinates(positions, n):
        # y axis is inverted
        return stitch.calc_grid_coordinates_from_positions(
            positions, n, reverse_rows=True
        )

class CellvoyagerMetadataReader(MetadataReader):

    '''Class for reading metadata from files formats specific to the Yokogawa
    CellVoyager 7000 microscope.

    Yokogawa doesn't store the position and channel information at the level
    of individual images, but in additional metadata files with *.mlf* and
    *.mrf* extensions.
    Unfortunately, these file formats are not supported by Bio-Formats,
    see `CellVoyagerReader <http://www.openmicroscopy.org/site/support/bio-formats5.1/formats/cellvoyager.html>`_.
    For compatibility with the OME data model, the `CellvoyagerMetadataReader`
    reads the XML from "MeasurementDetail.mrf" and "MeasurmentData.mlf" files,
    extracts the relevant data and stores them in an OMEXML object according to
    the Bio-Formats convention.
    '''

    def read(self, microscope_metadata_files, microscope_image_filenames):
        '''Reads metadata from "mlf" and "mrf" metadata files in case they
        are provided.

        Parameters
        ----------
        microscope_metadata_files: List[str]
            absolute path to microscope metadata files
        microscope_image_filenames: List[str]
            names of the corresponding microscope image files

        Returns
        -------
        bioformats.omexml.OMEXML
            OMEXML image metadata

        '''
        microscope_image_filenames = natsorted(microscope_image_filenames)
        metadata = bioformats.OMEXML(XML_DECLARATION)
        if len(microscope_metadata_files) == 0:
            logger.warn('no microscope metadata files found')
            return None
        elif len(microscope_metadata_files) != 2:
            logger.warn('expected two microscope metadata files')
            return None
        for f in microscope_metadata_files:
            if f.endswith('mlf'):
                mlf_filename = f
            elif f.endswith('mrf'):
                mrf_filename = f

        mrf_tree = etree.parse(mrf_filename)
        mrf_root = mrf_tree.getroot()

        # Obtain the positional information for each image acquisition site
        # from the ".mlf" file:
        mlf_tree = etree.parse(mlf_filename)
        mlf_root = mlf_tree.getroot()
        record_elements = mlf_root.xpath(
            './/bts:MeasurementRecord', namespaces=mlf_root.nsmap
        )
        mlf_ns = mlf_root.nsmap['bts']

        metadata.image_count = len([
            e for e in record_elements
            if e.attrib['{%s}Type' % mlf_ns] != 'ERR'
        ])
        lookup = defaultdict(list)

        for e in record_elements:
            # Translate positional information into well identifier string
            well_row = utils.map_number_to_letter(
                int(e.attrib['{%s}Row' % mlf_ns]))
            well_col = int(e.attrib['{%s}Column' % mlf_ns])
            well_id = '%s%.2d' % (well_row, well_col)
            if e.attrib['{%s}Type' % mlf_ns] == 'ERR':
                field_index = int(e.attrib['{%s}FieldIndex' % mlf_ns])
                logger.error(
                    'erroneous acquisition - no channel and z-position '
                    'information available at well %s field %d'
                    % (well_id, field_index)
                )
                continue
            # This microscope stores each plane in a separate file. Therefore,
            # we can use the filename to match images.
            name = e.text
            index = microscope_image_filenames.index(name)
            img = metadata.image(index)
            img.AcquisitionDate = e.attrib['{%s}Time' % mlf_ns]

            # Image files always contain only a single plane
            img.Pixels.SizeT = 1
            img.Pixels.SizeC = 1
            img.Pixels.SizeZ = 1
            img.Pixels.plane_count = 1
            # A name has to be set as a flag for the handler to update
            # the metadata
            img.Name = name
            img.Pixels.Channel(0).Name = channel_name
            img.Pixels.Plane(0).PositionX = float(e.attrib['{%s}X' % mlf_ns])
            img.Pixels.Plane(0).PositionY = float(e.attrib['{%s}Y' % mlf_ns])
            img.Pixels.Plane(0).PositionZ = float(e.attrib['{%s}Z' % mlf_ns])
            img.Pixels.Plane(0).TheZ = int(e.attrib['{%s}ZIndex' % mlf_ns])
            img.Pixels.Plane(0).TheT = int(e.attrib['{%s}TimelineIndex' % mlf_ns])

            idx = microscope_image_filenames.index(img.Name)
            lookup[well_id].append(idx)

        # Obtain the general experiment information and well plate format
        # specifications from the ".mrf" file:
        name = mrf_root.attrib['{%s}Title' % mrf_ns]
        plate = metadata.PlatesDucktype(metadata.root_node).newPlate(name=name)
        plate.RowNamingConvention = 'letter'
        plate.ColumnNamingConvention = 'number'
        plate.Rows = mrf_root.attrib['{%s}RowCount' % mrf_ns]
        plate.Columns = mrf_root.attrib['{%s}ColumnCount' % mrf_ns]
        wells = lookup.keys()
        for w in set(wells):
            # Create a *Well* element for each imaged well in the plate
            row = utils.map_letter_to_number(w[0]) - 1
            col = int(w[1:]) - 1
            well = metadata.WellsDucktype(plate).new(row=row, column=col)
            well_samples = metadata.WellSampleDucktype(well.node)
            for i, ref in enumerate(lookup[w]):
                # Create a *WellSample* element for each acquisition site
                well_samples.new(index=i)
                well_samples[i].ImageRef = ref

        return metadata
<commit_msg>Fix bug related to metadata config for cellvoyager<commit_after># TmLibrary - TissueMAPS library for distibuted image analysis routines.
# Copyright (C) 2016  Markus D. Herrmann, University of Zurich and Robin Hafen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
'''Implementation of classes for reading microscope image and metadata files
provided in a format specfic to the
`Yokogawa CellVoyager 7000 <http://www.yokogawa.com/solutions/products-platforms/life-science/high-throughput-cytological-discovery-system/cv7000s/>`_
microscope.
'''
import re
import logging
import bioformats
from natsort import natsorted
from collections import defaultdict
from lxml import etree

from tmlib import utils
from tmlib.workflow.illuminati import stitch
from tmlib.workflow.metaconfig.base import MetadataReader
from tmlib.workflow.metaconfig.base import MetadataHandler
from tmlib.workflow.metaconfig.omexml import XML_DECLARATION

logger = logging.getLogger(__name__)

#: Regular expression pattern to identify image files
IMAGE_FILE_REGEX_PATTERN = r'[^_]+_(?P<w>[A-Z]\d{2})_T(?P<t>\d+)F(?P<s>\d+)L\d+A\d+Z(?P<z>\d+)C(?P<c>\d+)\.'

#: Supported extensions for metadata files
METADATA_FILE_REGEX_PATTERN = r'.*\.(mlf|mrf)$'

class CellvoyagerMetadataHandler(MetadataHandler):

    '''Class for handling metadata specific to the Yokogawa Cellvoyager 7000
    microscope.
    '''

    def __init__(self, omexml_images, omexml_metadata=None):
        '''
        Parameters
        ----------
        omexml_images: Dict[str, bioformats.omexml.OMEXML]
            metadata extracted from microscope image files
        omexml_metadata: bioformats.omexml.OMEXML
            metadata extracted from microscope metadata files
        '''
        super(CellvoyagerMetadataHandler, self).__init__(
            omexml_images, omexml_metadata
        )

    @staticmethod
    def _calculate_coordinates(positions, n):
        # y axis is inverted
        return stitch.calc_grid_coordinates_from_positions(
            positions, n, reverse_rows=True
        )

class CellvoyagerMetadataReader(MetadataReader):

    '''Class for reading metadata from files formats specific to the Yokogawa
    CellVoyager 7000 microscope.

    Yokogawa doesn't store the position and channel information at the level
    of individual images, but in additional metadata files with *.mlf* and
    *.mrf* extensions.
    Unfortunately, these file formats are not supported by Bio-Formats,
    see `CellVoyagerReader <http://www.openmicroscopy.org/site/support/bio-formats5.1/formats/cellvoyager.html>`_.
    For compatibility with the OME data model, the `CellvoyagerMetadataReader`
    reads the XML from "MeasurementDetail.mrf" and "MeasurmentData.mlf" files,
    extracts the relevant data and stores them in an OMEXML object according to
    the Bio-Formats convention.
    '''

    def read(self, microscope_metadata_files, microscope_image_filenames):
        '''Reads metadata from "mlf" and "mrf" metadata files in case they
        are provided.

        Parameters
        ----------
        microscope_metadata_files: List[str]
            absolute path to microscope metadata files
        microscope_image_filenames: List[str]
            names of the corresponding microscope image files

        Returns
        -------
        bioformats.omexml.OMEXML
            OMEXML image metadata

        '''
        microscope_image_filenames = natsorted(microscope_image_filenames)
        metadata = bioformats.OMEXML(XML_DECLARATION)
        if len(microscope_metadata_files) == 0:
            logger.warn('no microscope metadata files found')
            return None
        elif len(microscope_metadata_files) != 2:
            logger.warn('expected two microscope metadata files')
            return None
        for f in microscope_metadata_files:
            if f.endswith('mlf'):
                mlf_filename = f
            elif f.endswith('mrf'):
                mrf_filename = f

        mrf_tree = etree.parse(mrf_filename)
        mrf_root = mrf_tree.getroot()
        mrf_ns = mrf_root.nsmap['bts']

        # Obtain the positional information for each image acquisition site
        # from the ".mlf" file:
        mlf_tree = etree.parse(mlf_filename)
        mlf_root = mlf_tree.getroot()
        record_elements = mlf_root.xpath(
            './/bts:MeasurementRecord', namespaces=mlf_root.nsmap
        )
        mlf_ns = mlf_root.nsmap['bts']

        metadata.image_count = len([
            e for e in record_elements
            if e.attrib['{%s}Type' % mlf_ns] != 'ERR'
        ])
        lookup = defaultdict(list)

        for e in record_elements:
            # Translate positional information into well identifier string
            well_row = utils.map_number_to_letter(
                int(e.attrib['{%s}Row' % mlf_ns]))
            well_col = int(e.attrib['{%s}Column' % mlf_ns])
            well_id = '%s%.2d' % (well_row, well_col)
            if e.attrib['{%s}Type' % mlf_ns] == 'ERR':
                field_index = int(e.attrib['{%s}FieldIndex' % mlf_ns])
                logger.error(
                    'erroneous acquisition - no channel and z-position '
                    'information available at well %s field %d'
                    % (well_id, field_index)
                )
                continue
            # This microscope stores each plane in a separate file. Therefore,
            # we can use the filename to match images.
            name = e.text
            index = microscope_image_filenames.index(name)
            img = metadata.image(index)
            img.AcquisitionDate = e.attrib['{%s}Time' % mlf_ns]

            # Image files always contain only a single plane
            img.Pixels.SizeT = 1
            img.Pixels.SizeC = 1
            img.Pixels.SizeZ = 1
            img.Pixels.plane_count = 1
            # A name has to be set as a flag for the handler to update
            # the metadata
            img.Name = name
            img.Pixels.Channel(0).Name = e.attrib['{%s}Ch' % mlf_ns]
            img.Pixels.Plane(0).PositionX = float(e.attrib['{%s}X' % mlf_ns])
            img.Pixels.Plane(0).PositionY = float(e.attrib['{%s}Y' % mlf_ns])
            img.Pixels.Plane(0).PositionZ = float(e.attrib['{%s}Z' % mlf_ns])
            img.Pixels.Plane(0).TheZ = int(e.attrib['{%s}ZIndex' % mlf_ns])
            img.Pixels.Plane(0).TheT = int(e.attrib['{%s}TimelineIndex' % mlf_ns])

            idx = microscope_image_filenames.index(img.Name)
            lookup[well_id].append(idx)

        # Obtain the general experiment information and well plate format
        # specifications from the ".mrf" file:
        name = mrf_root.attrib['{%s}Title' % mrf_ns]
        plate = metadata.PlatesDucktype(metadata.root_node).newPlate(name=name)
        plate.RowNamingConvention = 'letter'
        plate.ColumnNamingConvention = 'number'
        plate.Rows = mrf_root.attrib['{%s}RowCount' % mrf_ns]
        plate.Columns = mrf_root.attrib['{%s}ColumnCount' % mrf_ns]
        wells = lookup.keys()
        for w in set(wells):
            # Create a *Well* element for each imaged well in the plate
            row = utils.map_letter_to_number(w[0]) - 1
            col = int(w[1:]) - 1
            well = metadata.WellsDucktype(plate).new(row=row, column=col)
            well_samples = metadata.WellSampleDucktype(well.node)
            for i, ref in enumerate(lookup[w]):
                # Create a *WellSample* element for each acquisition site
                well_samples.new(index=i)
                well_samples[i].ImageRef = ref

        return metadata
<|endoftext|>


<commit_before>#partly based on Thibaut Munzer's script

import os
import time
import cPickle
import uuid
from importlib import import_module
from ..job import Job
import copy
import path

job_queue_class={
	'local':'local.LocalJobQueue',
	'local_multiprocess':'local.MultiProcessJobQueue',
	'avakas':'avakas.AvakasJobQueue',
	'plafrim':'plafrim.PlafrimJobQueue'
}

def get_jobqueue(jq_type='local', name =None, **jq_cfg2):
	if name is not None and os.path.isfile('jobs/'+name+'.jq'):
		with open('jobs/'+name+'.jq','r') as f:
			jq = cPickle.loads(f.read())
	tempstr = jq_type
	if tempstr in job_queue_class.keys():
		tempstr = job_queue_class[tempstr]
	templist = tempstr.split('.')
	temppath = '.'.join(templist[:-1])
	tempclass = templist[-1]
	_tempmod = import_module('.'+temppath,package=__name__)
	return getattr(_tempmod,tempclass)(name=name, **jq_cfg2)

class JobQueue(object):
	def __init__(self, erase=True, auto_update=True, name=None):
		self.job_list = []
		self.erase = erase
		self.auto_update = auto_update
		self.past_exec_time = 0
		self.uuid = str(uuid.uuid1())
		if name is None:
			self.name = self.uuid
		else:
			self.name = name

	def save(self):
		with open('jobs/'+self.name+'.jq','w') as f:
			f.write(cPickle.dumps(self,cPickle.HIGHEST_PROTOCOL))

	def add_job(self, job):
		eq_filter = [j for j in self.job_list if (j == job)]
		lt_filter = [j for j in eq_filter if eq_filter and (j < job)]
		if not eq_filter:
			self.status = 'pending'
			self.job_list.append(job)
		elif lt_filter:
			self.status = 'dependencies not satisfied'
		else:
			print 'Job already in queue!'
		job.save()
		self.save()

	def update_queue(self):
		if self.auto_update:
			self.check_virtualenvs()
		for j in [x for x in self.job_list]:
			if j.status == 'dependencies not satisfied':
				j.re_init()
				self.job_list.remove(j)
				self.add_job(j)
			if j.status == 'pending' and self.avail_workers()>0:
				self.submit_job(j)
				j.save()
			elif j.status == 'running':
				if not self.check_job_running(j):
					self.retrieve_job(j)
					if j.status == 'pending':
						j.status = 'missubmitted'
			elif j.status == 'dependencies not satisfied':
				for dep in j.gen_depend():
					print 'Adding dependency for job ' + j.job_dir
					self.add_job(dep)
			if j.status == 'unfinished':
				j.fix()
			elif j.status == 'done':
				with path.Path(j.get_path()):
					j.get_data()
				j.unpack_data()
				j.data = None
				self.past_exec_time += j.exec_time
				self.job_list.remove(j)
				if (not self.erase) and (not j.erase):
					j.clean()
			elif j.status == 'missubmitted':
				print('Missubmitted job: '+'_'.join([j.descr,j.uuid]))
			elif j.status == 'dependencies not satisfied':
				print('Dependencies not satisfied for job: '+j.job_dir)
			self.save()

	def auto_finish_queue(self,t=60):
		while [j for j in job_list if j.status != 'missubmitted']:
			self.update_queue()
			print time.gmtime(),' Queue updated'
			time.sleep(t)

	def check_virtualenvs(self):
		envs = {}
		for j in self.job_list:
			env = str(j.virtual_env)
			if env not in envs.keys():
				envs[env] = copy.deepcopy(j.requirements)
			else:
				envs[env] += copy.deepcopy(j.requirements)
		for env in envs.keys():
			self.update_virtualenv(env, requirements=list(set(envs[env])))

	def cancel_job(self, job, clean=False):
		if clean:
			job.clean()
		else:
			job.status = 'canceled'
			job.save()

	def total_time(self):
		t = 0
		for j in job_list:
			t += j.estimated_time

	def exec_time(self):
		t = self.past_exec_time
		for j in job_list:
			t += j.exec_time

	def submit_job(self, job):
		pass

	def check_job_running(self, job):
		pass

	def set_virtualenv(self, virtual_env, requirements):
		pass

	def update_virtualenv(self, virtual_env, requirements):
		pass

	def retrieve_job(self, job):
		pass

	def avail_workers(self):
		return 1
<commit_msg>updated JobQueue.add_job check policy to avoid by default in depth equality of jobs, and just check uuid<commit_after>#partly based on Thibaut Munzer's script

import os
import time
import cPickle
import uuid
from importlib import import_module
from ..job import Job
import copy
import path

job_queue_class={
	'local':'local.LocalJobQueue',
	'local_multiprocess':'local.MultiProcessJobQueue',
	'avakas':'avakas.AvakasJobQueue',
	'plafrim':'plafrim.PlafrimJobQueue'
}

def get_jobqueue(jq_type='local', name =None, **jq_cfg2):
	if name is not None and os.path.isfile('jobs/'+name+'.jq'):
		with open('jobs/'+name+'.jq','r') as f:
			jq = cPickle.loads(f.read())
	tempstr = jq_type
	if tempstr in job_queue_class.keys():
		tempstr = job_queue_class[tempstr]
	templist = tempstr.split('.')
	temppath = '.'.join(templist[:-1])
	tempclass = templist[-1]
	_tempmod = import_module('.'+temppath,package=__name__)
	return getattr(_tempmod,tempclass)(name=name, **jq_cfg2)

class JobQueue(object):
	def __init__(self, erase=True, auto_update=True, name=None, deep_check=False):
		self.job_list = []
		self.erase = erase
		self.auto_update = auto_update
		self.past_exec_time = 0
		self.uuid = str(uuid.uuid1())
		if name is None:
			self.name = self.uuid
		else:
			self.name = name
		self.deep_check = deep_check

	def save(self):
		with open('jobs/'+self.name+'.jq','w') as f:
			f.write(cPickle.dumps(self,cPickle.HIGHEST_PROTOCOL))

	def add_job(self, job, deep_check=None):
		if deep_check is None:
			deep_check = self.deep_check
		if deep_check:
			eq_filter = [j for j in self.job_list if (j == job)]
		else:
			eq_filter = [j for j in self.job_list if (j.uuid == job.uuid)]
		lt_filter = [j for j in eq_filter if eq_filter and (j < job)]
		if not eq_filter:
			self.status = 'pending'
			self.job_list.append(job)
		elif lt_filter:
			self.status = 'dependencies not satisfied'
		else:
			print 'Job already in queue!'
		job.save()
		self.save()

	def update_queue(self):
		if self.auto_update:
			self.check_virtualenvs()
		for j in [x for x in self.job_list]:
			if j.status == 'dependencies not satisfied':
				j.re_init()
				self.job_list.remove(j)
				self.add_job(j)
			if j.status == 'pending' and self.avail_workers()>0:
				self.submit_job(j)
				j.save()
			elif j.status == 'running':
				if not self.check_job_running(j):
					self.retrieve_job(j)
					if j.status == 'pending':
						j.status = 'missubmitted'
			elif j.status == 'dependencies not satisfied':
				for dep in j.gen_depend():
					print 'Adding dependency for job ' + j.job_dir
					self.add_job(dep)
			if j.status == 'unfinished':
				j.fix()
			elif j.status == 'done':
				with path.Path(j.get_path()):
					j.get_data()
				j.unpack_data()
				j.data = None
				self.past_exec_time += j.exec_time
				self.job_list.remove(j)
				if (not self.erase) and (not j.erase):
					j.clean()
			elif j.status == 'missubmitted':
				print('Missubmitted job: '+'_'.join([j.descr,j.uuid]))
			elif j.status == 'dependencies not satisfied':
				print('Dependencies not satisfied for job: '+j.job_dir)
			self.save()

	def auto_finish_queue(self,t=60):
		while [j for j in job_list if j.status != 'missubmitted']:
			self.update_queue()
			print time.gmtime(),' Queue updated'
			time.sleep(t)

	def check_virtualenvs(self):
		envs = {}
		for j in self.job_list:
			env = str(j.virtual_env)
			if env not in envs.keys():
				envs[env] = copy.deepcopy(j.requirements)
			else:
				envs[env] += copy.deepcopy(j.requirements)
		for env in envs.keys():
			self.update_virtualenv(env, requirements=list(set(envs[env])))

	def cancel_job(self, job, clean=False):
		if clean:
			job.clean()
		else:
			job.status = 'canceled'
			job.save()

	def total_time(self):
		t = 0
		for j in job_list:
			t += j.estimated_time

	def exec_time(self):
		t = self.past_exec_time
		for j in job_list:
			t += j.exec_time

	def submit_job(self, job):
		pass

	def check_job_running(self, job):
		pass

	def set_virtualenv(self, virtual_env, requirements):
		pass

	def update_virtualenv(self, virtual_env, requirements):
		pass

	def retrieve_job(self, job):
		pass

	def avail_workers(self):
		return 1
<|endoftext|>


<commit_before>#partly based on Thibaut Munzer's script

import os
import time
import cPickle
import uuid
from importlib import import_module
from ..job import Job
import copy
import path

job_queue_class={
	'local':'local.LocalJobQueue',
	'local_multiprocess':'local.MultiProcessJobQueue',
	'avakas':'avakas.AvakasJobQueue',
	'plafrim':'plafrim.PlafrimJobQueue',
	'torque':'torque.TorqueJobQueue',
	'slurm':'slurm.SlurmJobQueue',
	'oldslurm':'slurm.OldSlurmJobQueue',
	'anyone':'anyone.AnyoneJobQueue'
}

def get_jobqueue(jq_type='local', name =None, **jq_cfg2):
	if name is not None and os.path.isfile('jobs/'+name+'.jq'):
		with open('jobs/'+name+'.jq','r') as f:
			jq = cPickle.loads(f.read())
			if 'db' in jq_cfg2.keys():
				jq.db = jq_cfg2['db']
	else:
		tempstr = jq_type
		if tempstr in job_queue_class.keys():
			tempstr = job_queue_class[tempstr]
		templist = tempstr.split('.')
		temppath = '.'.join(templist[:-1])
		tempclass = templist[-1]
		_tempmod = import_module('.'+temppath,package=__name__)
		jq = getattr(_tempmod,tempclass)(name=name, **jq_cfg2)
	return jq

class JobQueue(object):
	def __init__(self, erase=False, auto_update=True, name=None, deep_check=False, verbose=False,path='job_queues/', reinit_missubmitted_times=0):
		self.verbose = verbose
		self.job_list = []
		self.erase = erase
		self.update_needed = False
		self.auto_update = auto_update
		self.past_exec_time = 0
		self.uuid = str(uuid.uuid1())
		if name is None:
			self.name = self.uuid
		else:
			self.name = name
		self.deep_check = deep_check
		self.executed_jobs = 0
		self.reinit_missubmitted_times = reinit_missubmitted_times
		self.jobqueue_dir = '_'.join([time.strftime('%Y-%m-%d_%H-%M-%S'), self.__class__.__name__, self.uuid])
		self.path = os.path.join(path,self.jobqueue_dir)

	def save(self):
		if not os.path.isdir(self.path):#'jobs'):
			os.makedirs(self.path)#'jobs')
		with open(os.path.join(self.path,self.name+'.jq','w')) as f:#'jobs/'+self.name+'.jq','w') as f:
			f.write(cPickle.dumps(self,cPickle.HIGHEST_PROTOCOL))

	def check_backups(self):
		self.backups_status = {'present':[],'locked':[]}

	def add_job(self, job, deep_check=None,save=True):
		if job.status == 'already done':
			job.clean()
			return []
		elif job.status == 'dependencies not satisfied':
			deps = job.gen_depend()
			for d in deps:
				uuid_l = self.add_job(d,save=False)
				job.deps += uuid_l
		eq_filter = [j for j in self.job_list if (j == job)]
		lt_filter = [j for j in eq_filter if (j < job)]
		ge_filter = [j for j in eq_filter if (j >= job)]
		if not eq_filter:
			self.job_list.append(job)
			job.move(new_path=self.path)
			job.reinit_missubmitted_times = self.reinit_missubmitted_times
			self.update_needed = True
			#job.status = 'pending'
			job.save()
			ans = [job.uuid]
		elif lt_filter and not ge_filter:
			job.status = 'dependencies not satisfied'
			self.job_list.append(job)
			job.move(new_path=self.path)
			self.update_needed = True
			job.deps += [jj.uuid for jj in lt_filter]
			#job.status = 'pending'
			job.save()
			ans = [job.uuid]
		else:
			if self.verbose:
				print 'Job already in queue!'
			job.clean()
			ans = [jj.uuid for jj in eq_filter]
		if save:
			self.save()
		job.close_connections()
		return ans

	def update_queue(self):
		self.save_status(message='Starting queue udpate')
		if self.auto_update and self.update_needed:
			self.check_virtualenvs()
			self.update_needed = False
			self.save_status(message='Requirements installed')
			self.save()
		self.check_running_jobs()
		self.check_backups()

		for j in [x for x in self.job_list]:
			if j.status == 'finished running':
				self.retrieve_job(j)
				if j.status in ['pending','running']:
					j.status = 'missubmitted'
			#elif j.status == 'dependencies not satisfied':
				#for dep in j.gen_depend():
				#	print 'Adding dependency for job ' + j.job_dir
				#	self.add_job(dep)
			j.close_connections()

		retrieved_list = self.global_retrieval()
		for j in retrieved_list:
			if j.status in ['pending','running']:
					j.status = 'missubmitted'

		self.reinit_missubmitted()

		for j in [x for x in self.job_list]:
			if j.status == 'unfinished':
				j.fix()
				if hasattr(self, 'extended_jobs'):
					self.extended_jobs += 1
				else:
					self.extended_jobs = 1
			elif j.status == 'done':
				if j.get_data_at_unpack:
					with path.Path(j.get_path()):
						j.get_data()
				j.unpack_data()
				j.data = None
				self.past_exec_time += j.exec_time
				self.executed_jobs += 1
				#if self.erase:
				j.status = 'to be cleaned'
			j.close_connections()

		for j in [x for x in self.job_list]:
			if j.status == 'dependencies not satisfied':
				job_uuids = [jj.uuid for jj in self.job_list if jj.status not in ['done','to be cleaned']] # maybe manage dependencies differently: stay in 'done' status , or 'unpacked', and clean only if deps do not need it anymore
				for dep_uuid in [dep_uuid for dep_uuid in j.deps]:
					if dep_uuid not in job_uuids:
						j.deps.remove(dep_uuid)
				if not j.deps:
					j.re_init()
				#self.job_list.remove(j)
				#self.add_job(j)
			j.close_connections()

		for j in [x for x in self.job_list]:
			if j.status == 'pending' and self.avail_workers()>0:
				j.save()
				self.submit_job(j)
				j.save()
			j.close_connections()

		self.global_submit()

		for j in [x for x in self.job_list]:
			if j.status == 'to be cleaned':
				self.job_list.remove(j)
				if self.erase:
					j.clean()
			elif self.verbose and j.status == 'missubmitted':
				print('Missubmitted job: '+'_'.join([j.descr,j.uuid]))
			elif self.verbose and j.status == 'dependencies not satisfied':
				print('Dependencies not satisfied for job: '+j.job_dir)
			elif self.verbose and j.status == 'script error':
				print('Script error for job: '+j.job_dir)
			j.close_connections()

		self.save()
		print self.get_status_string()
		self.save_status()
		if self.job_list and not [j for j in self.job_list if j.status not in ['missubmitted', 'script error', 'dependencies not satisfied']]:
			raise Exception('Queue blocked, only missubmitted jobs, script errors or waiting for dependencies jobs')

	def get_status_string(self,message='Queue updated'):
		return time.strftime("[%Y %m %d %H:%M:%S]: "+message+"\n"+str(self), time.localtime())

	def save_status(self,message='Queue updated'):
		if not os.path.isdir(self.path):#'jobs'):
			os.makedirs(self.path)#'jobs')
		with open(os.path.join(self.path,self.name+'.jq_status'),'a') as f_status:#'jobs/'+self.name+'.jq_status','a') as f_status:
			f_status.write(self.get_status_string(message=message))

	def __str__(self):
		total = 0
		ans = {}
		for j in self.job_list:
			total +=1
			if not j.status in ans.keys():
				ans[j.status] = 1
			else:
				ans[j.status] += 1
		exec_time = self.past_exec_time
		exec_time_j = int(exec_time/86400)
		exec_time -= exec_time_j * 86400
		exec_time_h = int(exec_time/3600)
		exec_time -= exec_time_h * 3600
		exec_time_m = int(exec_time/60)
		exec_time -= exec_time_m * 60
		str_exec = ''
		if exec_time_j:
			str_exec += str(int(exec_time_j)) + ' days '
		if exec_time_h:
			str_exec += str(int(exec_time_h))+' h '
		if exec_time_m:
			str_exec += str(int(exec_time_m))+' min '
		str_exec +=str(exec_time)+' s'
		str_ans = '    total: '+str(total)+'\n    '+'\n    '.join([str(key)+': '+str(val) for key,val in ans.items()])
		if not hasattr(self,'restarted_jobs'):
			self.restarted_jobs = 0
		if not hasattr(self,'extended_jobs'):
			self.extended_jobs = 0
		str_ans += '\n\n    execution time: '+str_exec+'\n    jobs done: '+str(self.executed_jobs)+'\n    jobs restarted: '+str(self.restarted_jobs)+'\n    jobs extended: '+str(self.extended_jobs)+'\n'
		return str_ans

	def auto_finish_queue(self,t=60,coeff=1,call_between=None):
		self.update_queue()
		step = t
		state = str(self)
		while [j for j in self.job_list if (j.status != 'missubmitted' and j.status != 'dependencies not satisfied')]:
			time.sleep(step)
			self.update_queue()
			if str(self) == state:
				step *= coeff
			else:
				state = str(self)
				step = t
			if call_between is not None:
				call_between()
			if self.job_list and not [j for j in self.job_list if j.status != 'to be cleaned']:
				self.update_queue()

	def check_virtualenvs(self):
		envs = {}
		for j in self.job_list:
			env = str(j.virtual_env)
			if env not in envs.keys():
				envs[env] = copy.deepcopy(j.requirements)
			else:
				envs[env] += copy.deepcopy(j.requirements)
		for env in envs.keys():
			self.update_virtualenv(env, requirements=list(set(envs[env])))

	def cancel_job(self, job, clean=False):
		if self.erase:
			job.clean()
		else:
			job.status = 'canceled'
			job.save()

	def total_time(self):
		t = 0
		for j in self.job_list:
			t += j.estimated_time

	def exec_time(self):
		t = self.past_exec_time
		for j in self.job_list:
			t += j.exec_time

	def reinit_missubmitted(self,job=None, force=False):
		if job is None:
			jlist = self.job_list
		else:
			jlist = [job]
		for j in jlist:
			if j.status == 'missubmitted':
				if force or ( hasattr(j,'reinit_missubmitted_times') and j.reinit_missubmitted_times >0):
					#j.update()
					j.restart()
					if hasattr(j,'reinit_missubmitted_times'):
						j.reinit_missubmitted_times -= 1
					j.status = 'pending'
					if hasattr(self,'restarted_jobs'):
						self.restarted_jobs += 1
					else:
						self.restarted_jobs = 1

	def submit_job(self, job):
		pass

	def global_submit(self):
		pass

	def check_running_jobs(self):
		pass

	def set_virtualenv(self, virtual_env, requirements):
		pass

	def update_virtualenv(self, virtual_env, requirements):
		pass

	def retrieve_job(self, job):
		pass

	def global_retrieval(self):
		return []

	def avail_workers(self):
		return 1

	def get_errors(self):
		errors = []
		for j in self.job_list:
			if j.status in ['missubmitted','script error']:
				errors.append((j.job_dir,j.get_error()))
		return errors

<commit_msg>added per jobqueue folder support<commit_after>#partly based on Thibaut Munzer's script

import os
import time
import cPickle
import uuid
from importlib import import_module
from ..job import Job
import copy
import path

job_queue_class={
	'local':'local.LocalJobQueue',
	'local_multiprocess':'local.MultiProcessJobQueue',
	'avakas':'avakas.AvakasJobQueue',
	'plafrim':'plafrim.PlafrimJobQueue',
	'torque':'torque.TorqueJobQueue',
	'slurm':'slurm.SlurmJobQueue',
	'oldslurm':'slurm.OldSlurmJobQueue',
	'anyone':'anyone.AnyoneJobQueue'
}

def get_jobqueue(jq_type='local', name =None, **jq_cfg2):
	if name is not None and os.path.isfile('jobs/'+name+'.jq'):
		with open('jobs/'+name+'.jq','r') as f:
			jq = cPickle.loads(f.read())
			if 'db' in jq_cfg2.keys():
				jq.db = jq_cfg2['db']
	else:
		tempstr = jq_type
		if tempstr in job_queue_class.keys():
			tempstr = job_queue_class[tempstr]
		templist = tempstr.split('.')
		temppath = '.'.join(templist[:-1])
		tempclass = templist[-1]
		_tempmod = import_module('.'+temppath,package=__name__)
		jq = getattr(_tempmod,tempclass)(name=name, **jq_cfg2)
	return jq

class JobQueue(object):
	def __init__(self, erase=False, auto_update=True, name=None, deep_check=False, verbose=False,path='job_queues/', reinit_missubmitted_times=0):
		self.verbose = verbose
		self.job_list = []
		self.erase = erase
		self.update_needed = False
		self.auto_update = auto_update
		self.past_exec_time = 0
		self.uuid = str(uuid.uuid1())
		if name is None:
			self.name = self.uuid
		else:
			self.name = name
		self.deep_check = deep_check
		self.executed_jobs = 0
		self.reinit_missubmitted_times = reinit_missubmitted_times
		self.jobqueue_dir = '_'.join([time.strftime('%Y-%m-%d_%H-%M-%S'), self.__class__.__name__, self.uuid])
		self.path = os.path.join(path,self.jobqueue_dir)

	def save(self):
		if not os.path.isdir(self.path):#'jobs'):
			os.makedirs(self.path)#'jobs')
		with open(os.path.join(self.path,self.name+'.jq'),'w') as f:#'jobs/'+self.name+'.jq','w') as f:
			f.write(cPickle.dumps(self,cPickle.HIGHEST_PROTOCOL))

	def check_backups(self):
		self.backups_status = {'present':[],'locked':[]}

	def add_job(self, job, deep_check=None,save=True):
		if job.status == 'already done':
			job.clean()
			return []
		elif job.status == 'dependencies not satisfied':
			deps = job.gen_depend()
			for d in deps:
				uuid_l = self.add_job(d,save=False)
				job.deps += uuid_l
		eq_filter = [j for j in self.job_list if (j == job)]
		lt_filter = [j for j in eq_filter if (j < job)]
		ge_filter = [j for j in eq_filter if (j >= job)]
		if not eq_filter:
			self.job_list.append(job)
			job.move(new_path=self.path)
			job.reinit_missubmitted_times = self.reinit_missubmitted_times
			self.update_needed = True
			#job.status = 'pending'
			job.save()
			ans = [job.uuid]
		elif lt_filter and not ge_filter:
			job.status = 'dependencies not satisfied'
			self.job_list.append(job)
			job.move(new_path=self.path)
			self.update_needed = True
			job.deps += [jj.uuid for jj in lt_filter]
			#job.status = 'pending'
			job.save()
			ans = [job.uuid]
		else:
			if self.verbose:
				print 'Job already in queue!'
			job.clean()
			ans = [jj.uuid for jj in eq_filter]
		if save:
			self.save()
		job.close_connections()
		return ans

	def update_queue(self):
		self.save_status(message='Starting queue udpate')
		if self.auto_update and self.update_needed:
			self.check_virtualenvs()
			self.update_needed = False
			self.save_status(message='Requirements installed')
			self.save()
		self.check_running_jobs()
		self.check_backups()

		for j in [x for x in self.job_list]:
			if j.status == 'finished running':
				self.retrieve_job(j)
				if j.status in ['pending','running']:
					j.status = 'missubmitted'
			#elif j.status == 'dependencies not satisfied':
				#for dep in j.gen_depend():
				#	print 'Adding dependency for job ' + j.job_dir
				#	self.add_job(dep)
			j.close_connections()

		retrieved_list = self.global_retrieval()
		for j in retrieved_list:
			if j.status in ['pending','running']:
					j.status = 'missubmitted'

		self.reinit_missubmitted()

		for j in [x for x in self.job_list]:
			if j.status == 'unfinished':
				j.fix()
				if hasattr(self, 'extended_jobs'):
					self.extended_jobs += 1
				else:
					self.extended_jobs = 1
			elif j.status == 'done':
				if j.get_data_at_unpack:
					with path.Path(j.get_path()):
						j.get_data()
				j.unpack_data()
				j.data = None
				self.past_exec_time += j.exec_time
				self.executed_jobs += 1
				#if self.erase:
				j.status = 'to be cleaned'
			j.close_connections()

		for j in [x for x in self.job_list]:
			if j.status == 'dependencies not satisfied':
				job_uuids = [jj.uuid for jj in self.job_list if jj.status not in ['done','to be cleaned']] # maybe manage dependencies differently: stay in 'done' status , or 'unpacked', and clean only if deps do not need it anymore
				for dep_uuid in [dep_uuid for dep_uuid in j.deps]:
					if dep_uuid not in job_uuids:
						j.deps.remove(dep_uuid)
				if not j.deps:
					j.re_init()
				#self.job_list.remove(j)
				#self.add_job(j)
			j.close_connections()

		for j in [x for x in self.job_list]:
			if j.status == 'pending' and self.avail_workers()>0:
				j.save()
				self.submit_job(j)
				j.save()
			j.close_connections()

		self.global_submit()

		for j in [x for x in self.job_list]:
			if j.status == 'to be cleaned':
				self.job_list.remove(j)
				if self.erase:
					j.clean()
			elif self.verbose and j.status == 'missubmitted':
				print('Missubmitted job: '+'_'.join([j.descr,j.uuid]))
			elif self.verbose and j.status == 'dependencies not satisfied':
				print('Dependencies not satisfied for job: '+j.job_dir)
			elif self.verbose and j.status == 'script error':
				print('Script error for job: '+j.job_dir)
			j.close_connections()

		self.save()
		print self.get_status_string()
		self.save_status()
		if self.job_list and not [j for j in self.job_list if j.status not in ['missubmitted', 'script error', 'dependencies not satisfied']]:
			raise Exception('Queue blocked, only missubmitted jobs, script errors or waiting for dependencies jobs')

	def get_status_string(self,message='Queue updated'):
		return time.strftime("[%Y %m %d %H:%M:%S]: "+message+"\n"+str(self), time.localtime())

	def save_status(self,message='Queue updated'):
		if not os.path.isdir(self.path):#'jobs'):
			os.makedirs(self.path)#'jobs')
		with open(os.path.join(self.path,self.name+'.jq_status'),'a') as f_status:#'jobs/'+self.name+'.jq_status','a') as f_status:
			f_status.write(self.get_status_string(message=message))

	def __str__(self):
		total = 0
		ans = {}
		for j in self.job_list:
			total +=1
			if not j.status in ans.keys():
				ans[j.status] = 1
			else:
				ans[j.status] += 1
		exec_time = self.past_exec_time
		exec_time_j = int(exec_time/86400)
		exec_time -= exec_time_j * 86400
		exec_time_h = int(exec_time/3600)
		exec_time -= exec_time_h * 3600
		exec_time_m = int(exec_time/60)
		exec_time -= exec_time_m * 60
		str_exec = ''
		if exec_time_j:
			str_exec += str(int(exec_time_j)) + ' days '
		if exec_time_h:
			str_exec += str(int(exec_time_h))+' h '
		if exec_time_m:
			str_exec += str(int(exec_time_m))+' min '
		str_exec +=str(exec_time)+' s'
		str_ans = '    total: '+str(total)+'\n    '+'\n    '.join([str(key)+': '+str(val) for key,val in ans.items()])
		if not hasattr(self,'restarted_jobs'):
			self.restarted_jobs = 0
		if not hasattr(self,'extended_jobs'):
			self.extended_jobs = 0
		str_ans += '\n\n    execution time: '+str_exec+'\n    jobs done: '+str(self.executed_jobs)+'\n    jobs restarted: '+str(self.restarted_jobs)+'\n    jobs extended: '+str(self.extended_jobs)+'\n'
		return str_ans

	def auto_finish_queue(self,t=60,coeff=1,call_between=None):
		self.update_queue()
		step = t
		state = str(self)
		while [j for j in self.job_list if (j.status != 'missubmitted' and j.status != 'dependencies not satisfied')]:
			time.sleep(step)
			self.update_queue()
			if str(self) == state:
				step *= coeff
			else:
				state = str(self)
				step = t
			if call_between is not None:
				call_between()
			if self.job_list and not [j for j in self.job_list if j.status != 'to be cleaned']:
				self.update_queue()

	def check_virtualenvs(self):
		envs = {}
		for j in self.job_list:
			env = str(j.virtual_env)
			if env not in envs.keys():
				envs[env] = copy.deepcopy(j.requirements)
			else:
				envs[env] += copy.deepcopy(j.requirements)
		for env in envs.keys():
			self.update_virtualenv(env, requirements=list(set(envs[env])))

	def cancel_job(self, job, clean=False):
		if self.erase:
			job.clean()
		else:
			job.status = 'canceled'
			job.save()

	def total_time(self):
		t = 0
		for j in self.job_list:
			t += j.estimated_time

	def exec_time(self):
		t = self.past_exec_time
		for j in self.job_list:
			t += j.exec_time

	def reinit_missubmitted(self,job=None, force=False):
		if job is None:
			jlist = self.job_list
		else:
			jlist = [job]
		for j in jlist:
			if j.status == 'missubmitted':
				if force or ( hasattr(j,'reinit_missubmitted_times') and j.reinit_missubmitted_times >0):
					#j.update()
					j.restart()
					if hasattr(j,'reinit_missubmitted_times'):
						j.reinit_missubmitted_times -= 1
					j.status = 'pending'
					if hasattr(self,'restarted_jobs'):
						self.restarted_jobs += 1
					else:
						self.restarted_jobs = 1

	def submit_job(self, job):
		pass

	def global_submit(self):
		pass

	def check_running_jobs(self):
		pass

	def set_virtualenv(self, virtual_env, requirements):
		pass

	def update_virtualenv(self, virtual_env, requirements):
		pass

	def retrieve_job(self, job):
		pass

	def global_retrieval(self):
		return []

	def avail_workers(self):
		return 1

	def get_errors(self):
		errors = []
		for j in self.job_list:
			if j.status in ['missubmitted','script error']:
				errors.append((j.job_dir,j.get_error()))
		return errors

<|endoftext|>