<commit_before>import pandas
import logging
import requests
# Python3
try:
    from io import StringIO
# Python2
except ImportError:
    from StringIO import StringIO

logger = logging.getLogger('cbio')

cbio_url = 'http://www.cbioportal.org/webservice.do'
ccle_study = 'cellline_ccle_broad'

def send_request(data, skiprows=0):
    """Return a data frame from a web service request to cBio portal.

    Sends a web service requrest to the cBio portal with arguments given in
    the dictionary data and returns a Pandas data frame on success.

    More information about the service here:
    http://www.cbioportal.org/web_api.jsp

    Parameters
    ----------
    data : dict
        A dict of parameters for the query.
    skiprows : Optional[int]
        Number of rows to skip when reading dataframe. This is useful to align
        headers.

    Returns
    -------
    df : Pandas DataFrame
        return the response from cBioPortal as a Pandas DataFrame
    """
    res = requests.get(cbio_url, params=data)
    if res.status_code == 200:
        csv_StringIO = StringIO(res.text)
        df = pandas.read_csv(csv_StringIO, sep='\t', skiprows=skiprows)
        return df
    else:
        logger.error('Request returned with code %d' % res.status_code)

def get_mutations(study_id, gene_list, mutation_type=None,
                  case_id=None):
    """Return mutations as a list of genes and list of amino acid changes.

    Parameters
    ----------
    study_id : str
        The ID of the cBio study.
        Example: 'cellline_ccle_broad' or 'paad_icgc'
    gene_list : list[str]
        A list of genes with their HGNC symbols.
        Example: ['BRAF', 'KRAS']
    mutation_type : Optional[str]
        The type of mutation to filter to.
        mutation_type can be one of: missense, nonsense, frame_shift_ins,
                                     frame_shift_del, splice_site
    case_id : Optional[str]
        The case ID within the study to filter to.

    Returns
    -------
    mutations : tuple[list]
        A tuple of two lists, the first one containing a list of genes, and
        the second one a list of amino acid changes in those genes.
    """
    genetic_profile = get_genetic_profiles(study_id, 'mutation')[0]
    gene_list_str = ','.join(gene_list)

    data = {'cmd': 'getMutationData',
            'case_set_id': study_id,
            'genetic_profile_id': genetic_profile,
            'gene_list': gene_list_str}
    df = send_request(data, skiprows=1)
    if case_id:
        df = df[df['case_id'] == case_id]
    res = _filter_data_frame(df, ['gene_symbol', 'amino_acid_change'],
                                   'mutation_type', mutation_type)
    mutations = {'gene_symbol': res['gene_symbol'].values(),
                 'amino_acid_change': res['amino_acid_change'].values()}
    return mutations

def get_num_sequenced(study_id):
    """Return number of sequenced tumors for given study.

    This is useful for calculating mutation statistics in terms of the
    prevalence of certain mutations within a type of cancer.

    Parameters
    ----------
    study_id : str
        The ID of the cBio study.
        Example: 'paad_icgc'

    Returns
    -------
    num_case : int
        The number of sequenced tumors in the given study
    """
    data = {'cmd': 'getCaseLists',
            'cancer_study_id': study_id}
    df = send_request(data)
    row_filter = df['case_list_id'].str.contains('sequenced', case=False)
    num_case = len(df[row_filter]['case_ids'].tolist()[0].split(' '))
    return num_case

def get_genetic_profiles(study_id, profile_filter=None):
    """Return all the genetic profiles (data sets) for a given study.

    Genetic profiles are different types of data for a given study. For
    instance the study 'cellline_ccle_broad' has profiles such as
    'cellline_ccle_broad_mutations' for mutations, 'cellline_ccle_broad_CNA'
    for copy number alterations, etc.

    Parameters
    ----------
    study_id : str
        The ID of the cBio study.
        Example: 'paad_icgc'
    profile_filter : Optional[str]
        A string used to filter the profiles to return.
        The genetic profiles can include "mutation", "CNA", "rppa",
        "methylation", etc.

    Returns
    -------
    genetic_profiles : list[str]
        A list of genetic profiles available  for the given study.
    """
    data = {'cmd': 'getGeneticProfiles',
            'cancer_study_id': study_id}
    df = send_request(data)
    res = _filter_data_frame(df, ['genetic_profile_id'],
                                  'genetic_alteration_type', profile_filter)
    genetic_profiles = res['genetic_profile_id'].values()
    return genetic_profiles

def get_cancer_studies(study_filter=None):
    """Return a list of cancer study identifiers, optionally filtered.

    There are typically multiple studies for a given type of cancer and
    a filter can be used to constrain the returned list.

    Parameters
    ----------
    study_filter : Optional[str]
        A string used to filter the study IDs to return. Example: "paad"

    Returns
    -------
    study_ids : list[str]
        A list of study IDs.
        For instance "paad" as a filter would result in a list
        of study IDs with paad in their name like "paad_icgc", "paad_tcga",
        etc.
    """
    data = {'cmd': 'getCancerStudies'}
    df = send_request(data)
    res = _filter_data_frame(df, ['cancer_study_id'],
                             'cancer_study_id', study_filter)
    study_ids = res['cancer_study_id'].values()
    return study_ids

def get_cancer_types(cancer_filter=None):
    """Return a list of cancer types, optionally filtered.

    Parameters
    ----------
    cancer_filter : Optional[str]
        A string used to filter cancer types. Its value is the name or
        part of the name of a type of cancer. Example: "melanoma",
        "pancreatic", "non-small cell lung"

    Returns
    -------
    type_ids : list[str]
        A list of cancer types matching the filter.
        Example: for cancer_filter="pancreatic", the result includes
        "panet" (neuro-endocrine) and "paad" (adenocarcinoma)
    """
    data = {'cmd': 'getTypesOfCancer'}
    df = send_request(data)
    res = _filter_data_frame(df, ['type_of_cancer_id'], 'name', filter_str)
    type_ids = res['type_of_cancer_id'].values()
    return type_ids

def get_mutations_ccle(gene_list, cell_lines, mutation_type=None):
    """Return a dict of mutations in given genes and cell lines from CCLE.

    This is a specialized call to get_mutations tailored to CCLE cell lines.

    Parameters
    ----------
    gene_list : list[str]
        A list of HGNC gene symbols to get mutations in
    cell_lines : list[str]
        A list of CCLE cell line names to get mutations for.
    mutation_type : Optional[str]
        The type of mutation to filter to.
        mutation_type can be one of: missense, nonsense, frame_shift_ins,
                                     frame_shift_del, splice_site

    Returns
    -------
    mutations : dict
        The result from cBioPortal as a dict in the format
        {cell_line : {gene : [mutation1, mutation2, ...] }}

        Example:
        {'LOXIMVI_SKIN': {'BRAF': ['V600E', 'I208V']},
         'SKMEL30_SKIN': {'BRAF': ['D287H', 'E275K']}}
    """
    mutations = {cl: {g: [] for g in gene_list} for cl in cell_lines}
    for cell_line in cell_lines:
        mutations_cl = get_mutations(ccle_study, gene_list,
                                     mutation_type=mutation_type,
                                     case_id=cell_line)
        for gene, aa_change in zip(mutations_cl['gene_symbol'],
                                   mutations_cl['amino_acid_change']):
            mutations[cell_line][gene].append(aa_change)
    return mutations

def check_ccle_lines_for_mutation(gene, amino_acid_change):
    """Return cell lines with a given mutation.

    Check which cell lines in CCLE have a particular mutation and
    return their names in a list.

    Parameters
    ----------
    gene : str
        as HGNC ID
    amino_acid_change : str
        example - V600E

    Returns
    -------
    cell_lines : list
        return the response from cBioPortal as a list of cell lines
    """
    data = {'cmd': 'getMutationData',
            'case_set_id': ccle_study,
            'genetic_profile_id': ccle_study + '_mutations',
            'gene_list': gene}
    df = send_request(data, skiprows=1)
    df = df[df['amino_acid_change'] == amino_acid_change]
    cell_lines = df['case_id'].unique().tolist()
    cell_lines = [x.split('_')[0] for x in cell_lines]
    return cell_lines

def _filter_data_frame(df, data_col, filter_col, filter_str=None):
    """Return a filtered data frame as a dictionary."""
    if filter_str is not None:
        relevant_cols = data_col + [filter_col]
        df.dropna(inplace=True, subset=relevant_cols)
        row_filter = df[filter_col].str.contains(filter_str, case=False)
        data_list = df[row_filter][data_col].to_dict()
    else:
        data_list = df[data_col].to_dict()
    return data_list
<commit_msg>Rename, document getting CCLE lines with mutation<commit_after>import pandas
import logging
import requests
# Python3
try:
    from io import StringIO
# Python2
except ImportError:
    from StringIO import StringIO

logger = logging.getLogger('cbio')

cbio_url = 'http://www.cbioportal.org/webservice.do'
ccle_study = 'cellline_ccle_broad'

def send_request(data, skiprows=0):
    """Return a data frame from a web service request to cBio portal.

    Sends a web service requrest to the cBio portal with arguments given in
    the dictionary data and returns a Pandas data frame on success.

    More information about the service here:
    http://www.cbioportal.org/web_api.jsp

    Parameters
    ----------
    data : dict
        A dict of parameters for the query.
    skiprows : Optional[int]
        Number of rows to skip when reading dataframe. This is useful to align
        headers.

    Returns
    -------
    df : Pandas DataFrame
        return the response from cBioPortal as a Pandas DataFrame
    """
    res = requests.get(cbio_url, params=data)
    if res.status_code == 200:
        csv_StringIO = StringIO(res.text)
        df = pandas.read_csv(csv_StringIO, sep='\t', skiprows=skiprows)
        return df
    else:
        logger.error('Request returned with code %d' % res.status_code)

def get_mutations(study_id, gene_list, mutation_type=None,
                  case_id=None):
    """Return mutations as a list of genes and list of amino acid changes.

    Parameters
    ----------
    study_id : str
        The ID of the cBio study.
        Example: 'cellline_ccle_broad' or 'paad_icgc'
    gene_list : list[str]
        A list of genes with their HGNC symbols.
        Example: ['BRAF', 'KRAS']
    mutation_type : Optional[str]
        The type of mutation to filter to.
        mutation_type can be one of: missense, nonsense, frame_shift_ins,
                                     frame_shift_del, splice_site
    case_id : Optional[str]
        The case ID within the study to filter to.

    Returns
    -------
    mutations : tuple[list]
        A tuple of two lists, the first one containing a list of genes, and
        the second one a list of amino acid changes in those genes.
    """
    genetic_profile = get_genetic_profiles(study_id, 'mutation')[0]
    gene_list_str = ','.join(gene_list)

    data = {'cmd': 'getMutationData',
            'case_set_id': study_id,
            'genetic_profile_id': genetic_profile,
            'gene_list': gene_list_str}
    df = send_request(data, skiprows=1)
    if case_id:
        df = df[df['case_id'] == case_id]
    res = _filter_data_frame(df, ['gene_symbol', 'amino_acid_change'],
                                   'mutation_type', mutation_type)
    mutations = {'gene_symbol': res['gene_symbol'].values(),
                 'amino_acid_change': res['amino_acid_change'].values()}
    return mutations

def get_num_sequenced(study_id):
    """Return number of sequenced tumors for given study.

    This is useful for calculating mutation statistics in terms of the
    prevalence of certain mutations within a type of cancer.

    Parameters
    ----------
    study_id : str
        The ID of the cBio study.
        Example: 'paad_icgc'

    Returns
    -------
    num_case : int
        The number of sequenced tumors in the given study
    """
    data = {'cmd': 'getCaseLists',
            'cancer_study_id': study_id}
    df = send_request(data)
    row_filter = df['case_list_id'].str.contains('sequenced', case=False)
    num_case = len(df[row_filter]['case_ids'].tolist()[0].split(' '))
    return num_case

def get_genetic_profiles(study_id, profile_filter=None):
    """Return all the genetic profiles (data sets) for a given study.

    Genetic profiles are different types of data for a given study. For
    instance the study 'cellline_ccle_broad' has profiles such as
    'cellline_ccle_broad_mutations' for mutations, 'cellline_ccle_broad_CNA'
    for copy number alterations, etc.

    Parameters
    ----------
    study_id : str
        The ID of the cBio study.
        Example: 'paad_icgc'
    profile_filter : Optional[str]
        A string used to filter the profiles to return.
        The genetic profiles can include "mutation", "CNA", "rppa",
        "methylation", etc.

    Returns
    -------
    genetic_profiles : list[str]
        A list of genetic profiles available  for the given study.
    """
    data = {'cmd': 'getGeneticProfiles',
            'cancer_study_id': study_id}
    df = send_request(data)
    res = _filter_data_frame(df, ['genetic_profile_id'],
                                  'genetic_alteration_type', profile_filter)
    genetic_profiles = res['genetic_profile_id'].values()
    return genetic_profiles

def get_cancer_studies(study_filter=None):
    """Return a list of cancer study identifiers, optionally filtered.

    There are typically multiple studies for a given type of cancer and
    a filter can be used to constrain the returned list.

    Parameters
    ----------
    study_filter : Optional[str]
        A string used to filter the study IDs to return. Example: "paad"

    Returns
    -------
    study_ids : list[str]
        A list of study IDs.
        For instance "paad" as a filter would result in a list
        of study IDs with paad in their name like "paad_icgc", "paad_tcga",
        etc.
    """
    data = {'cmd': 'getCancerStudies'}
    df = send_request(data)
    res = _filter_data_frame(df, ['cancer_study_id'],
                             'cancer_study_id', study_filter)
    study_ids = res['cancer_study_id'].values()
    return study_ids

def get_cancer_types(cancer_filter=None):
    """Return a list of cancer types, optionally filtered.

    Parameters
    ----------
    cancer_filter : Optional[str]
        A string used to filter cancer types. Its value is the name or
        part of the name of a type of cancer. Example: "melanoma",
        "pancreatic", "non-small cell lung"

    Returns
    -------
    type_ids : list[str]
        A list of cancer types matching the filter.
        Example: for cancer_filter="pancreatic", the result includes
        "panet" (neuro-endocrine) and "paad" (adenocarcinoma)
    """
    data = {'cmd': 'getTypesOfCancer'}
    df = send_request(data)
    res = _filter_data_frame(df, ['type_of_cancer_id'], 'name', filter_str)
    type_ids = res['type_of_cancer_id'].values()
    return type_ids

def get_mutations_ccle(gene_list, cell_lines, mutation_type=None):
    """Return a dict of mutations in given genes and cell lines from CCLE.

    This is a specialized call to get_mutations tailored to CCLE cell lines.

    Parameters
    ----------
    gene_list : list[str]
        A list of HGNC gene symbols to get mutations in
    cell_lines : list[str]
        A list of CCLE cell line names to get mutations for.
    mutation_type : Optional[str]
        The type of mutation to filter to.
        mutation_type can be one of: missense, nonsense, frame_shift_ins,
                                     frame_shift_del, splice_site

    Returns
    -------
    mutations : dict
        The result from cBioPortal as a dict in the format
        {cell_line : {gene : [mutation1, mutation2, ...] }}

        Example:
        {'LOXIMVI_SKIN': {'BRAF': ['V600E', 'I208V']},
         'SKMEL30_SKIN': {'BRAF': ['D287H', 'E275K']}}
    """
    mutations = {cl: {g: [] for g in gene_list} for cl in cell_lines}
    for cell_line in cell_lines:
        mutations_cl = get_mutations(ccle_study, gene_list,
                                     mutation_type=mutation_type,
                                     case_id=cell_line)
        for gene, aa_change in zip(mutations_cl['gene_symbol'],
                                   mutations_cl['amino_acid_change']):
            mutations[cell_line][gene].append(aa_change)
    return mutations

def get_ccle_lines_mutation(gene, amino_acid_change):
    """Return cell lines with a given point mutation in a given gene.

    Checks which cell lines in CCLE have a particular point mutation
    in a given gene and return their names in a list.

    Parameters
    ----------
    gene : str
        The HGNC symbol of the mutated gene in whose product the amino
        acid change occurs. Example: "BRAF"
    amino_acid_change : str
        The amino acid change of interest. Example: "V600E"

    Returns
    -------
    cell_lines : list
        A list of CCLE cell lines in which the given mutation occurs.
    """
    data = {'cmd': 'getMutationData',
            'case_set_id': ccle_study,
            'genetic_profile_id': ccle_study + '_mutations',
            'gene_list': gene}
    df = send_request(data, skiprows=1)
    df = df[df['amino_acid_change'] == amino_acid_change]
    cell_lines = df['case_id'].unique().tolist()
    return cell_lines

def _filter_data_frame(df, data_col, filter_col, filter_str=None):
    """Return a filtered data frame as a dictionary."""
    if filter_str is not None:
        relevant_cols = data_col + [filter_col]
        df.dropna(inplace=True, subset=relevant_cols)
        row_filter = df[filter_col].str.contains(filter_str, case=False)
        data_list = df[row_filter][data_col].to_dict()
    else:
        data_list = df[data_col].to_dict()
    return data_list
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import re
import logging
import os.path
import requests
from lxml import etree
from lxml.etree import QName
import xml.etree.ElementTree as ET

from indra.literature import pubmed_client
from indra.util import UnicodeXMLTreeBuilder as UTB

# Python 2
try:
    basestring
# Python 3
except:
    basestring = str

logger = logging.getLogger(__name__)

pmc_url = 'https://www.ncbi.nlm.nih.gov/pmc/oai/oai.cgi'
pmid_convert_url = 'https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/'

# Paths to resource files
pmids_fulltext_path = os.path.join(os.path.dirname(__file__),
                                   'pmids_fulltext.txt')
pmids_oa_xml_path = os.path.join(os.path.dirname(__file__),
                                 'pmids_oa_xml.txt')
pmids_oa_txt_path = os.path.join(os.path.dirname(__file__),
                                 'pmids_oa_txt.txt')
pmids_auth_xml_path = os.path.join(os.path.dirname(__file__),
                                   'pmids_auth_xml.txt')
# Define global dict containing lists of PMIDs among mineable PMCs
# to be lazily initialized
pmids_fulltext_dict = {}

def id_lookup(paper_id, idtype=None):
    """This function takes a Pubmed ID, Pubmed Central ID, or DOI
    and use the Pubmed ID mapping
    service and looks up all other IDs from one
    of these. The IDs are returned in a dictionary."""
    if idtype is not None and idtype not in ('pmid', 'pmcid', 'doi'):
        raise ValueError("Invalid idtype %s; must be 'pmid', 'pmcid', "
                         "or 'doi'." % idtype)
    if paper_id.upper().startswith('PMC'):
        idtype = 'pmcid'
    # Strip off any prefix
    if paper_id.upper().startswith('PMID'):
        paper_id = paper_id[4:]
    elif paper_id.upper().startswith('DOI'):
        paper_id = paper_id[3:]
    data = {'ids': paper_id}
    if idtype is not None:
        data['idtype'] = idtype
    try:
        tree = pubmed_client.send_request(pmid_convert_url, data)
    except Exception as e:
        logger.error('Error looking up PMID in PMC: %s' % e)
        return {}
    if tree is None:
        return {}
    record = tree.find('record')
    if record is None:
        return {}
    doi = record.attrib.get('doi')
    pmid = record.attrib.get('pmid')
    pmcid = record.attrib.get('pmcid')
    ids = {'doi': doi,
           'pmid': pmid,
           'pmcid': pmcid}
    return ids

def get_ids(search_term, retmax=1000):
    return pubmed_client.get_ids(search_term, retmax=retmax, db='pmc')

def get_xml(pmc_id):
    """Returns XML for the article corresponding to a PMC ID."""
    if pmc_id.upper().startswith('PMC'):
        pmc_id = pmc_id[3:]
    # Request params
    params = {}
    params['verb'] = 'GetRecord'
    params['identifier'] = 'oai:pubmedcentral.nih.gov:%s' % pmc_id
    params['metadataPrefix'] = 'pmc'
    # Submit the request
    res = requests.get(pmc_url, params)
    if not res.status_code == 200:
        logger.warning("Couldn't download %s" % pmc_id)
        return None
    # Read the bytestream
    xml_bytes = res.content
    # Check for any XML errors; xml_str should still be bytes
    tree = ET.XML(xml_bytes, parser=UTB())
    xmlns = "http://www.openarchives.org/OAI/2.0/"
    err_tag = tree.find('{%s}error' % xmlns)
    if err_tag is not None:
        err_code = err_tag.attrib['code']
        err_text = err_tag.text
        logger.warning('PMC client returned with error %s: %s'
                       % (err_code, err_text))
        return None
    # If no error, return the XML as a unicode string
    else:
        return xml_bytes.decode('utf-8')

def extract_text(xml_string):
    """Get text from the body of the given NLM XML string.

    Parameters
    ----------
    xml_string : str
        String containing valid NLM XML.

    Returns
    -------
    str
        Extracted plaintext.
    """
    tree = etree.fromstring(xml_string.encode('utf-8'))

    paragraphs = []
    # In NLM xml, all plaintext is within <p> tags, and is the only thing
    # that can be contained in <p> tags. To handle to possibility of namespaces
    # uses regex to search for tags either of the form 'p' or '{<namespace>}p'
    for element in tree.iter():
        if isinstance(element.tag, basestring) and \
           re.search('(^|})p$', element.tag) and element.text:
            paragraphs.append(' '.join(element.text.split()))
    if paragraphs:
        return ' '.join(paragraphs).strip()
    else:
        return None

def filter_pmids(pmid_list, source_type):
    """Filter a list of PMIDs for ones with full text from PMC.

    Parameters
    ----------
    pmid_list : list of str
        List of PMIDs to filter.
    source_type : string
        One of 'fulltext', 'oa_xml', 'oa_txt', or 'auth_xml'.

    Returns
    -------
    list of str
        PMIDs available in the specified source/format type.
    """
    global pmids_fulltext_dict
    # Check args
    if source_type not in ('fulltext', 'oa_xml', 'oa_txt', 'auth_xml'):
        raise ValueError("source_type must be one of: 'fulltext', 'oa_xml', "
                         "'oa_txt', or 'auth_xml'.")
    # Check if we've loaded this type, and lazily initialize
    if pmids_fulltext_dict.get(source_type) is None:
        fulltext_list_path = os.path.join(os.path.dirname(__file__),
                                          'pmids_%s.txt' % source_type)
        with open(fulltext_list_path, 'rb') as f:
            fulltext_list = set([line.strip().decode('utf-8')
                                 for line in f.readlines()])
            pmids_fulltext_dict[source_type] = fulltext_list
    return list(set(pmid_list).intersection(
                                pmids_fulltext_dict.get(source_type)))
<commit_msg>Fix bug that caused some xml content to be missed<commit_after>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import re
import logging
import os.path
import requests
from lxml import etree
from lxml.etree import QName
import xml.etree.ElementTree as ET

from indra.literature import pubmed_client
from indra.util import UnicodeXMLTreeBuilder as UTB

# Python 2
try:
    basestring
# Python 3
except:
    basestring = str

logger = logging.getLogger(__name__)

pmc_url = 'https://www.ncbi.nlm.nih.gov/pmc/oai/oai.cgi'
pmid_convert_url = 'https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/'

# Paths to resource files
pmids_fulltext_path = os.path.join(os.path.dirname(__file__),
                                   'pmids_fulltext.txt')
pmids_oa_xml_path = os.path.join(os.path.dirname(__file__),
                                 'pmids_oa_xml.txt')
pmids_oa_txt_path = os.path.join(os.path.dirname(__file__),
                                 'pmids_oa_txt.txt')
pmids_auth_xml_path = os.path.join(os.path.dirname(__file__),
                                   'pmids_auth_xml.txt')
# Define global dict containing lists of PMIDs among mineable PMCs
# to be lazily initialized
pmids_fulltext_dict = {}

def id_lookup(paper_id, idtype=None):
    """This function takes a Pubmed ID, Pubmed Central ID, or DOI
    and use the Pubmed ID mapping
    service and looks up all other IDs from one
    of these. The IDs are returned in a dictionary."""
    if idtype is not None and idtype not in ('pmid', 'pmcid', 'doi'):
        raise ValueError("Invalid idtype %s; must be 'pmid', 'pmcid', "
                         "or 'doi'." % idtype)
    if paper_id.upper().startswith('PMC'):
        idtype = 'pmcid'
    # Strip off any prefix
    if paper_id.upper().startswith('PMID'):
        paper_id = paper_id[4:]
    elif paper_id.upper().startswith('DOI'):
        paper_id = paper_id[3:]
    data = {'ids': paper_id}
    if idtype is not None:
        data['idtype'] = idtype
    try:
        tree = pubmed_client.send_request(pmid_convert_url, data)
    except Exception as e:
        logger.error('Error looking up PMID in PMC: %s' % e)
        return {}
    if tree is None:
        return {}
    record = tree.find('record')
    if record is None:
        return {}
    doi = record.attrib.get('doi')
    pmid = record.attrib.get('pmid')
    pmcid = record.attrib.get('pmcid')
    ids = {'doi': doi,
           'pmid': pmid,
           'pmcid': pmcid}
    return ids

def get_ids(search_term, retmax=1000):
    return pubmed_client.get_ids(search_term, retmax=retmax, db='pmc')

def get_xml(pmc_id):
    """Returns XML for the article corresponding to a PMC ID."""
    if pmc_id.upper().startswith('PMC'):
        pmc_id = pmc_id[3:]
    # Request params
    params = {}
    params['verb'] = 'GetRecord'
    params['identifier'] = 'oai:pubmedcentral.nih.gov:%s' % pmc_id
    params['metadataPrefix'] = 'pmc'
    # Submit the request
    res = requests.get(pmc_url, params)
    if not res.status_code == 200:
        logger.warning("Couldn't download %s" % pmc_id)
        return None
    # Read the bytestream
    xml_bytes = res.content
    # Check for any XML errors; xml_str should still be bytes
    tree = ET.XML(xml_bytes, parser=UTB())
    xmlns = "http://www.openarchives.org/OAI/2.0/"
    err_tag = tree.find('{%s}error' % xmlns)
    if err_tag is not None:
        err_code = err_tag.attrib['code']
        err_text = err_tag.text
        logger.warning('PMC client returned with error %s: %s'
                       % (err_code, err_text))
        return None
    # If no error, return the XML as a unicode string
    else:
        return xml_bytes.decode('utf-8')

def extract_text(xml_string):
    """Get text from the body of the given NLM XML string.

    Parameters
    ----------
    xml_string : str
        String containing valid NLM XML.

    Returns
    -------
    str
        Extracted plaintext.
    """
    tree = etree.fromstring(xml_string.encode('utf-8'))

    paragraphs = []
    # In NLM xml, all plaintext is within <p> tags, and is the only thing
    # that can be contained in <p> tags. To handle to possibility of namespaces
    # uses regex to search for tags either of the form 'p' or '{<namespace>}p'
    for element in tree.iter():
        if isinstance(element.tag, basestring) and \
           re.search('(^|})[p|title]$', element.tag) and element.text:
            paragraphs.append(' '.join(element.itertext()).strip())
    if paragraphs:
        return ' '.join(paragraphs).strip()
    else:
        return None

def filter_pmids(pmid_list, source_type):
    """Filter a list of PMIDs for ones with full text from PMC.

    Parameters
    ----------
    pmid_list : list of str
        List of PMIDs to filter.
    source_type : string
        One of 'fulltext', 'oa_xml', 'oa_txt', or 'auth_xml'.

    Returns
    -------
    list of str
        PMIDs available in the specified source/format type.
    """
    global pmids_fulltext_dict
    # Check args
    if source_type not in ('fulltext', 'oa_xml', 'oa_txt', 'auth_xml'):
        raise ValueError("source_type must be one of: 'fulltext', 'oa_xml', "
                         "'oa_txt', or 'auth_xml'.")
    # Check if we've loaded this type, and lazily initialize
    if pmids_fulltext_dict.get(source_type) is None:
        fulltext_list_path = os.path.join(os.path.dirname(__file__),
                                          'pmids_%s.txt' % source_type)
        with open(fulltext_list_path, 'rb') as f:
            fulltext_list = set([line.strip().decode('utf-8')
                                 for line in f.readlines()])
            pmids_fulltext_dict[source_type] = fulltext_list
    return list(set(pmid_list).intersection(
                                pmids_fulltext_dict.get(source_type)))
<|endoftext|>


<commit_before>import pygraphviz as pgv
import itertools
from copy import copy
from indra.statements import *
from indra.databases import uniprot_client

class Preassembler(object):

    def __init__(self, entity_hierarchy, mod_hierarchy, stmts=None):
        self.entity_hierarchy = entity_hierarchy
        self.mod_hierarchy = mod_hierarchy
        if stmts:
            self.stmts = stmts
        else:
            self.stmts = []
        self.unique_stmts = []
        self.related_stmts = []

    def add_statements(self, stmts):
        """Add to the current list of statements."""
        self.stmts += stmts

    def combine_duplicates(self):
        """Combine evidence from duplicate Statements.

        Statements are deemed to be duplicates if they have the same key
        returned by the matches_key() method of the Statement class.

        This function keeps the first instance of each set of duplicate
        statements and merges the evidence lists from all of the other
        statements.

        Returns a list of unique Statements and stores it in
        self.unique_stmts.
        """

        self.unique_stmts = []
        # Group statements according to whether they are matches (differing
        # only in their evidence).
        # Sort the statements in place by matches_key()
        self.stmts.sort(key=lambda x: x.matches_key())
        for key, duplicates in itertools.groupby(self.stmts,
                                                 key=lambda x: x.matches_key()):
            # Get the first statement and add the evidence of all subsequent
            # Statements to it
            for stmt_ix, stmt in enumerate(duplicates):
                if stmt_ix == 0:
                    first_stmt = stmt
                else:
                    first_stmt.evidence += stmt.evidence
            # This should never be None or anything else
            assert isinstance(first_stmt, Statement)
            self.unique_stmts.append(first_stmt)
        return self.unique_stmts

    def combine_related(self):
        """Connect related statements based on their refinement relationships.

        This function takes as a starting point the unique statements (with
        duplicates removed) and returns a modified flat list of statements
        containing only those statements which do not represent a refinement of
        other existing statements. In other words, the more general versions of
        a given statement do not appear at the top level, but instead are
        listed in the supports field of the top-level statements.

        If self.unique_stmts has not been initialized with the de-duplicated
        statements, self.combine_duplicates is called.

        The procedure for combining statements in this way involves a series
        of steps:

        1. The statements are grouped by whether they are of the same type
           (e.g., Phosphorylation) and involve the same entities (e.g.,
           BRAF and MAP2K1).
        2. The groups of statements are then further compared to see if one
           group involves superfamily entities of another group. If so, the
           statements involving the superfamily are added into the group of
           statements involving the more concrete entities to create an
           "extended group."
        3. The statements within each extended group are then compared; if one
           statement represents a refinement of the other (as defined by the
           refinement_of() method implemented for the Statement), then the more
           refined statement is added to the `supports` field of the more
           general statement, and the more general statement is added to the
           `supported_by` field of the more refined statement.
        4. A new flat list of statements is created that contains only those
           statements that have no `supports` entries (statements containing
           such entries are not eliminated, because they will be retrievable
           from the `supported_by` fields of other statements. This list
           is returned to the caller.

        .. note:: Subfamily relationships must be consistent across arguments

            For now, we will require that merges can only occur if the isa
            relationships are all in the same direction for all the agents in a
            Statement. For example, the two statement groups: RAF_family ->
            MEK1 and BRAF -> MEK_family would not be merged, since BRAF isa
            RAF_family, but MEK_family is not a MEK1. In the future this
            restriction could be revisited.

        Returns
        -------
        list of Statements.
            The returned list contains Statements representing the more
            concrete/refined versions of the Statements involving particular
            entities.

        Example
        -------
        A more general statement with no information about a Phosphorylation
        site is identified as supporting a more specific statement::

            >>> from indra.preassembler.hierarchy_manager import \
                    entity_hierarchy as eh, modification_hierarchy as mh
            >>> braf = Agent('BRAF')
            >>> map2k1 = Agent('MAP2K1')
            >>> st1 = Phosphorylation(braf, map2k1, 'Phosphorylation', None)
            >>> st2 = Phosphorylation(braf, map2k1, 'Phosphorylation', '218')
            >>> pa = Preassembler(eh, mh, [st1, st2])
            >>> combined_stmts = pa.combine_related()
            >>> combined_stmts
            [Phosphorylation(BRAF, MAP2K1, Phosphorylation, 218)]
            >>> combined_stmts[0].supported_by
            [Phosphorylation(BRAF, MAP2K1, Phosphorylation, None)]
            >>> combined_stmts[0].supported_by[0].supports
            [Phosphorylation(BRAF, MAP2K1, Phosphorylation, 218)]
        """

        # If unique_stmts is not initialized, call combine_duplicates.
        if not self.unique_stmts:
            self.combine_duplicates()
        # Group statements according to whether they have matching entities,
        # and store the resulting lists in a dict, indexed by the key defined
        # by the statement type and its entities:
        # Sort the statements in place by entities_match_key():
        self.unique_stmts.sort(key=lambda x: x.entities_match_key())
        groups = {grouper[0]: list(grouper[1])
                  for grouper in itertools.groupby(self.unique_stmts,
                                          key=lambda x: x.entities_match_key())}
        # The ext_groups dict is where we store the extended groups, those
        # statements which involve either the same entities or entities with
        # family relationships.
        ext_groups = copy(groups)
        # We examine pairs of Statement groups, looking for "isa" relationships:
        for g1_key, g2_key in itertools.permutations(groups.keys(), 2):
            g1 = groups[g1_key]
            g2 = groups[g2_key]
            # If we have two groups G1 and G2, each containing Statements with
            # some number of Agent arguments, e.g.  G1_Stmt(Ag1, Ag2, Ag3) and
            # G2_Stmt(Ag4, Ag5, Ag6), we need to know whether each of the
            # corresponding Agent arguments are related for all of the
            # Statements in the two groups.  That is, each of the arguments
            # between G1 and G2 must either be an entity match (as determined
            # by the entities_match test of the respective agents) or have an
            # "isa" relationship, as determined by the HierarchyManager in use.
            # If the elements in both groups are related, then the
            # Statements in the group with the superfamily relationship
            # are added into the group with the more specific entities. However,
            # the superfamily group is not removed from the group list, as
            # it may be a superfamily supporting Statements from another group.

            # Get the first statement from each group (we've already determined
            # that all of the statements within each group have an entity
            # match, so we only need the first:
            g1_stmt = g1[0]
            g2_stmt = g2[0]
            # Check that the statements are of the same type; if not, no merge.
            if type(g1_stmt) is not type(g2_stmt):
                continue
            # Check that all of the agents match or have an isa relationship.
            # Because the statements are of the same type, they should have the
            # same number of agents as arguments.  First, let's keep track of
            # our checks that g1 is the "primary" group, i.e., it is the one
            # with the more refined/grounded entities. We build a list with one
            # boolean entry for each argument, where a True value indicates
            # that the arguments at that position imply that g1 is the primary
            # group.
            agent_pairs = zip(g1_stmt.agent_list(), g2_stmt.agent_list())
            g1_is_refinement = [ag1.entity_matches(ag2) or
                                self.entity_hierarchy.isa(ag1.name, ag2.name)
                                for ag1, ag2 in agent_pairs]
            # If g1_is_refinement is all True values, that means everything in
            # the group1 statements isa thing in the group2 statements.
            if all(g1_is_refinement):
                g1_ext_list = ext_groups[g1_key]
                ext_groups[g1_key] = g1_ext_list + g2
            continue
        # At this point we have, in ext_groups, a dict of lists of Statements
        # indexed by their entity_matches key, but now the groups contain not
        # only statements with matching entities, but also entities related by
        # isa relationships. The next step is to process each group, checking
        # each statement against each other statement to determine supports and
        # supported_by relationships. This is determined by calling the
        # refinement_of method on pairs of statements.
        # Iterate over each of the extended groups:
        for ext_group in ext_groups.values():
            # Iterate over pairs of statements in the group:
            for stmt1, stmt2 in itertools.permutations(ext_group, 2):
                if stmt1.refinement_of(stmt2, self.entity_hierarchy,
                                       self.mod_hierarchy):
                    stmt1.supported_by.append(stmt2)
                    stmt2.supports.append(stmt1)
        # Now that the groups have been processed, we need to find the
        # non-subsumed statements, those that have no supports relationships.
        self.related_stmts = [stmt for ext_group in ext_groups.values()
                               for stmt in ext_group
                               if not stmt.supports]
        return self.related_stmts

    def check_sequence(self):
        """Iterate over all Statements and check whether references to
        residues and sequence positions are consistent with sequence
        information in the UniProt database"""
        # TODO: extend to all Agent modifications
        for st in self.stmts:
            if isinstance(st, Phosphorylation):
                sub_id = st.sub.db_refs['UP']
                sub = uniprot_client.query_protein(sub_id)
                if st.mod == 'PhosphorylationSerine':
                    residue = 'S'
                elif st.mod == 'PhosphorylationThreonine':
                    residue = 'T'
                elif st.mod == 'PhosphorylationTyrosine':
                    residue = 'Y'
                else:
                    continue
                ver = uniprot_client.verify_location(sub, residue, location=st.mod_pos)
                if ver is False:
                    print 'Sequence check failed for %s:' % st
                    print '-> Position %s on %s is not %s.' %\
                          (st.mod_pos, st.sub.name, residue)

def render_stmt_graph(statements, agent_style=None):
    """Renders the supports/supported_by relationships of a set of statements
    and returns a pygraphviz graph.

    Example
    -------
    Pattern for getting statements and rendering as a Graphviz graph::

        bp = biopax_api.process_pc_pathsfromto(['BRAF'], ['MAP2K1'])
        bp.get_phosphorylation()

        pa = Preassembler(eh, mh, bp.statements)
        pa.combine_related()

        graph = render_stmt_graph(pa.related_stmts)
        graph.write('braf.dot')
        graph.draw('braf.pdf', prog='dot')
    """

    # Set the default agent formatting properties
    if agent_style is None:
        agent_style = {'color': 'lightgray', 'style': 'filled',
                       'fontname': 'arial'}
    # Sets to store all of the nodes and edges as we recursively process all
    # of the statements
    nodes = set([])
    edges = set([])
    # Recursive function for processing all statements
    def process_stmt(stmt):
        nodes.add(stmt)
        for sby_ix, sby_stmt in enumerate(stmt.supported_by):
            edges.add((str(stmt.matches_key()), str(sby_stmt.matches_key())))
            process_stmt(sby_stmt)
    # Process all of the top-level statements, getting the supporting statements
    # recursively
    for stmt in statements:
        process_stmt(stmt)
    # Add the nodes and edges to the graph
    graph = pgv.AGraph(name='statements', directed=True)
    for node in nodes:
        graph.add_node(str(node.matches_key()), label=str(node), **agent_style)
    graph.add_edges_from(edges)
    return graph

<commit_msg>Handle single/list of UniProt IDs in sequence check<commit_after>import pygraphviz as pgv
import itertools
from copy import copy
from indra.statements import *
from indra.databases import uniprot_client

class Preassembler(object):

    def __init__(self, entity_hierarchy, mod_hierarchy, stmts=None):
        self.entity_hierarchy = entity_hierarchy
        self.mod_hierarchy = mod_hierarchy
        if stmts:
            self.stmts = stmts
        else:
            self.stmts = []
        self.unique_stmts = []
        self.related_stmts = []

    def add_statements(self, stmts):
        """Add to the current list of statements."""
        self.stmts += stmts

    def combine_duplicates(self):
        """Combine evidence from duplicate Statements.

        Statements are deemed to be duplicates if they have the same key
        returned by the matches_key() method of the Statement class.

        This function keeps the first instance of each set of duplicate
        statements and merges the evidence lists from all of the other
        statements.

        Returns a list of unique Statements and stores it in
        self.unique_stmts.
        """

        self.unique_stmts = []
        # Group statements according to whether they are matches (differing
        # only in their evidence).
        # Sort the statements in place by matches_key()
        self.stmts.sort(key=lambda x: x.matches_key())
        for key, duplicates in itertools.groupby(self.stmts,
                                                 key=lambda x: x.matches_key()):
            # Get the first statement and add the evidence of all subsequent
            # Statements to it
            for stmt_ix, stmt in enumerate(duplicates):
                if stmt_ix == 0:
                    first_stmt = stmt
                else:
                    first_stmt.evidence += stmt.evidence
            # This should never be None or anything else
            assert isinstance(first_stmt, Statement)
            self.unique_stmts.append(first_stmt)
        return self.unique_stmts

    def combine_related(self):
        """Connect related statements based on their refinement relationships.

        This function takes as a starting point the unique statements (with
        duplicates removed) and returns a modified flat list of statements
        containing only those statements which do not represent a refinement of
        other existing statements. In other words, the more general versions of
        a given statement do not appear at the top level, but instead are
        listed in the supports field of the top-level statements.

        If self.unique_stmts has not been initialized with the de-duplicated
        statements, self.combine_duplicates is called.

        The procedure for combining statements in this way involves a series
        of steps:

        1. The statements are grouped by whether they are of the same type
           (e.g., Phosphorylation) and involve the same entities (e.g.,
           BRAF and MAP2K1).
        2. The groups of statements are then further compared to see if one
           group involves superfamily entities of another group. If so, the
           statements involving the superfamily are added into the group of
           statements involving the more concrete entities to create an
           "extended group."
        3. The statements within each extended group are then compared; if one
           statement represents a refinement of the other (as defined by the
           refinement_of() method implemented for the Statement), then the more
           refined statement is added to the `supports` field of the more
           general statement, and the more general statement is added to the
           `supported_by` field of the more refined statement.
        4. A new flat list of statements is created that contains only those
           statements that have no `supports` entries (statements containing
           such entries are not eliminated, because they will be retrievable
           from the `supported_by` fields of other statements. This list
           is returned to the caller.

        .. note:: Subfamily relationships must be consistent across arguments

            For now, we will require that merges can only occur if the isa
            relationships are all in the same direction for all the agents in a
            Statement. For example, the two statement groups: RAF_family ->
            MEK1 and BRAF -> MEK_family would not be merged, since BRAF isa
            RAF_family, but MEK_family is not a MEK1. In the future this
            restriction could be revisited.

        Returns
        -------
        list of Statements.
            The returned list contains Statements representing the more
            concrete/refined versions of the Statements involving particular
            entities.

        Example
        -------
        A more general statement with no information about a Phosphorylation
        site is identified as supporting a more specific statement::

            >>> from indra.preassembler.hierarchy_manager import \
                    entity_hierarchy as eh, modification_hierarchy as mh
            >>> braf = Agent('BRAF')
            >>> map2k1 = Agent('MAP2K1')
            >>> st1 = Phosphorylation(braf, map2k1, 'Phosphorylation', None)
            >>> st2 = Phosphorylation(braf, map2k1, 'Phosphorylation', '218')
            >>> pa = Preassembler(eh, mh, [st1, st2])
            >>> combined_stmts = pa.combine_related()
            >>> combined_stmts
            [Phosphorylation(BRAF, MAP2K1, Phosphorylation, 218)]
            >>> combined_stmts[0].supported_by
            [Phosphorylation(BRAF, MAP2K1, Phosphorylation, None)]
            >>> combined_stmts[0].supported_by[0].supports
            [Phosphorylation(BRAF, MAP2K1, Phosphorylation, 218)]
        """

        # If unique_stmts is not initialized, call combine_duplicates.
        if not self.unique_stmts:
            self.combine_duplicates()
        # Group statements according to whether they have matching entities,
        # and store the resulting lists in a dict, indexed by the key defined
        # by the statement type and its entities:
        # Sort the statements in place by entities_match_key():
        self.unique_stmts.sort(key=lambda x: x.entities_match_key())
        groups = {grouper[0]: list(grouper[1])
                  for grouper in itertools.groupby(self.unique_stmts,
                                          key=lambda x: x.entities_match_key())}
        # The ext_groups dict is where we store the extended groups, those
        # statements which involve either the same entities or entities with
        # family relationships.
        ext_groups = copy(groups)
        # We examine pairs of Statement groups, looking for "isa" relationships:
        for g1_key, g2_key in itertools.permutations(groups.keys(), 2):
            g1 = groups[g1_key]
            g2 = groups[g2_key]
            # If we have two groups G1 and G2, each containing Statements with
            # some number of Agent arguments, e.g.  G1_Stmt(Ag1, Ag2, Ag3) and
            # G2_Stmt(Ag4, Ag5, Ag6), we need to know whether each of the
            # corresponding Agent arguments are related for all of the
            # Statements in the two groups.  That is, each of the arguments
            # between G1 and G2 must either be an entity match (as determined
            # by the entities_match test of the respective agents) or have an
            # "isa" relationship, as determined by the HierarchyManager in use.
            # If the elements in both groups are related, then the
            # Statements in the group with the superfamily relationship
            # are added into the group with the more specific entities. However,
            # the superfamily group is not removed from the group list, as
            # it may be a superfamily supporting Statements from another group.

            # Get the first statement from each group (we've already determined
            # that all of the statements within each group have an entity
            # match, so we only need the first:
            g1_stmt = g1[0]
            g2_stmt = g2[0]
            # Check that the statements are of the same type; if not, no merge.
            if type(g1_stmt) is not type(g2_stmt):
                continue
            # Check that all of the agents match or have an isa relationship.
            # Because the statements are of the same type, they should have the
            # same number of agents as arguments.  First, let's keep track of
            # our checks that g1 is the "primary" group, i.e., it is the one
            # with the more refined/grounded entities. We build a list with one
            # boolean entry for each argument, where a True value indicates
            # that the arguments at that position imply that g1 is the primary
            # group.
            agent_pairs = zip(g1_stmt.agent_list(), g2_stmt.agent_list())
            g1_is_refinement = [ag1.entity_matches(ag2) or
                                self.entity_hierarchy.isa(ag1.name, ag2.name)
                                for ag1, ag2 in agent_pairs]
            # If g1_is_refinement is all True values, that means everything in
            # the group1 statements isa thing in the group2 statements.
            if all(g1_is_refinement):
                g1_ext_list = ext_groups[g1_key]
                ext_groups[g1_key] = g1_ext_list + g2
            continue
        # At this point we have, in ext_groups, a dict of lists of Statements
        # indexed by their entity_matches key, but now the groups contain not
        # only statements with matching entities, but also entities related by
        # isa relationships. The next step is to process each group, checking
        # each statement against each other statement to determine supports and
        # supported_by relationships. This is determined by calling the
        # refinement_of method on pairs of statements.
        # Iterate over each of the extended groups:
        for ext_group in ext_groups.values():
            # Iterate over pairs of statements in the group:
            for stmt1, stmt2 in itertools.permutations(ext_group, 2):
                if stmt1.refinement_of(stmt2, self.entity_hierarchy,
                                       self.mod_hierarchy):
                    stmt1.supported_by.append(stmt2)
                    stmt2.supports.append(stmt1)
        # Now that the groups have been processed, we need to find the
        # non-subsumed statements, those that have no supports relationships.
        self.related_stmts = [stmt for ext_group in ext_groups.values()
                               for stmt in ext_group
                               if not stmt.supports]
        return self.related_stmts

    def check_sequence(self):
        """Iterate over all Statements and check whether references to
        residues and sequence positions are consistent with sequence
        information in the UniProt database"""
        # TODO: extend to all Agent modifications
        for st in self.stmts:
            if isinstance(st, Phosphorylation):
                # Skip statements with no specified position
                if st.mod_pos is None:
                    continue
                # Is looking at the first element enough?
                sub_id = st.sub.db_refs['UP']
                if not isinstance(sub_id, basestring):
                    sub_id = sub_id[0]
                sub = uniprot_client.query_protein(sub_id)
                if st.mod == 'PhosphorylationSerine':
                    residue = 'S'
                elif st.mod == 'PhosphorylationThreonine':
                    residue = 'T'
                elif st.mod == 'PhosphorylationTyrosine':
                    residue = 'Y'
                else:
                    continue
                ver = uniprot_client.verify_location(sub, residue,
                                                     location=st.mod_pos)
                if ver is False:
                    print '%s:' % st
                    print '-> Sequence check failed; position %s on %s is not %s.' %\
                          (st.mod_pos, st.sub.name, residue)

def render_stmt_graph(statements, agent_style=None):
    """Renders the supports/supported_by relationships of a set of statements
    and returns a pygraphviz graph.

    Example
    -------
    Pattern for getting statements and rendering as a Graphviz graph::

        bp = biopax_api.process_pc_pathsfromto(['BRAF'], ['MAP2K1'])
        bp.get_phosphorylation()

        pa = Preassembler(eh, mh, bp.statements)
        pa.combine_related()

        graph = render_stmt_graph(pa.related_stmts)
        graph.write('braf.dot')
        graph.draw('braf.pdf', prog='dot')
    """

    # Set the default agent formatting properties
    if agent_style is None:
        agent_style = {'color': 'lightgray', 'style': 'filled',
                       'fontname': 'arial'}
    # Sets to store all of the nodes and edges as we recursively process all
    # of the statements
    nodes = set([])
    edges = set([])
    # Recursive function for processing all statements
    def process_stmt(stmt):
        nodes.add(stmt)
        for sby_ix, sby_stmt in enumerate(stmt.supported_by):
            edges.add((str(stmt.matches_key()), str(sby_stmt.matches_key())))
            process_stmt(sby_stmt)
    # Process all of the top-level statements, getting the supporting statements
    # recursively
    for stmt in statements:
        process_stmt(stmt)
    # Add the nodes and edges to the graph
    graph = pgv.AGraph(name='statements', directed=True)
    for node in nodes:
        graph.add_node(str(node.matches_key()), label=str(node), **agent_style)
    graph.add_edges_from(edges)
    return graph

<|endoftext|>


<commit_before>import ctypes
import pkg_resources
import os
import sys
from typing import List, Optional

from setuptools import setup

VERSION = '10.4.0'

# List of packages supported by this version of CuPy.
PACKAGES = [
    'cupy-cuda102',
    'cupy-cuda110',
    'cupy-cuda111',
    'cupy-cuda112',
    'cupy-cuda113',
    'cupy-cuda114',
    'cupy-cuda115',
    'cupy-cuda116',
    'cupy-rocm-4-0',
    'cupy-rocm-4-2',
    'cupy-rocm-4-3',
    'cupy-rocm-5-0',
]

# List of packages NOT supported by this version of CuPy.
PACKAGES_OUTDATED = [
    'cupy-cuda80',
    'cupy-cuda90',
    'cupy-cuda91',
    'cupy-cuda92',
    'cupy-cuda100',
    'cupy-cuda101',
]

# List of sdist packages.
PACKAGES_SDIST = [
    'cupy',
]

class AutoDetectionFailed(Exception):
    pass

def _log(msg: str) -> None:
    sys.stdout.write(f'[cupy-wheel] {msg}\n')
    sys.stdout.flush()

def _get_version_from_library(
        libnames: List[str], funcname: str) -> Optional[int]:
    """Returns the library version from list of candidate libraries."""

    for libname in libnames:
        try:
            _log(f'Looking for library: {libname}')
            runtime_so = ctypes.CDLL(libname)
            break
        except Exception as e:
            _log(f'Failed to open {libname}: {e}')
            return None
    else:
        _log('No more candidate library to find')
        return None

    func = getattr(runtime_so, funcname, None)
    if func is None:
        raise AutoDetectionFailed(
            f'{libname}: {func} could not be found')
    func.restype = ctypes.c_int  # cudaError_t
    func.argtypes = [ctypes.POINTER(ctypes.c_int)]
    version_ptr = ctypes.c_int()
    retval = func(version_ptr)
    if retval != 0:
        raise AutoDetectionFailed(
            f'{libname}: {func} returned error: {retval}')
    version = version_ptr.value
    _log(f'Detected version: {version}')
    return version

def _get_cuda_version() -> Optional[int]:
    """Returns the detected CUDA version or None."""

    if sys.platform == 'linux':
        libnames = ['libcudart.so']
    elif sys.platform == 'win32':
        libnames = ['cudart64.dll']
    else:
        _log(f'CUDA detection unsupported on platform: {sys.platform}')
        return None
    _log(f'Trying to detect CUDA version from libraries: {libnames}')
    version = _get_version_from_library(libnames, 'cudaRuntimeGetVersion')
    return version

def _get_rocm_version() -> Optional[int]:
    """Returns the detected ROCm version or None."""
    if sys.platform == 'linux':
        libnames = ['libamdhip64.so']
    else:
        _log(f'ROCm detection unsupported on platform: {sys.platform}')
        return None
    version = _get_version_from_library(libnames, 'hipRuntimeGetVersion')
    return version

def _find_installed_packages() -> List[str]:
    """Returns the list of CuPy packages installed in the environment."""

    found = []
    for pkg in (PACKAGES + PACKAGES_OUTDATED + PACKAGES_SDIST):
        try:
            pkg_resources.get_distribution(pkg)
            found.append(pkg)
        except pkg_resources.DistributionNotFound:
            pass
    return found

def _cuda_version_to_package(ver: int) -> str:
    if ver < 10020:
        raise AutoDetectionFailed(
            f'Your CUDA version ({ver}) is too old.')
    elif ver < 11000:
        # CUDA 10.2
        suffix = '102'
    elif ver < 11010:
        # CUDA 11.0
        suffix = '110'
    elif ver < 11020:
        # CUDA 11.1
        suffix = '111'
    elif ver < 11030:
        # CUDA 11.2
        suffix = '112'
    elif ver < 11040:
        # CUDA 11.3
        suffix = '113'
    elif ver < 11050:
        # CUDA 11.4
        suffix = '114'
    elif ver < 11060:
        # CUDA 11.5
        suffix = '115'
    elif ver < 11070:
        # CUDA 11.6
        suffix = '116'
    else:
        raise AutoDetectionFailed(
            f'Your CUDA version ({ver}) is too new.')
    return f'cupy-cuda{suffix}'

def _rocm_version_to_package(ver: int) -> str:
    if 400 <= ver < 410:
        # ROCm 4.0
        suffix = '4-0'
    elif 420 <= ver < 430:
        # ROCm 4.2
        suffix = '4-2'
    elif 4_03_00000 <= ver < 4_04_00000:
        # ROCm 4.3
        suffix = '4-3'
    elif 5_00_00000 <= ver < 5_01_00000:
        # ROCm 5.0
        suffix = '5-0'
    else:
        raise AutoDetectionFailed(
            f'Your ROCm version ({ver}) is unsupported.')
    return f'cupy-rocm-{suffix}'

def infer_best_package() -> str:
    """Returns the appropriate CuPy wheel package name for the environment."""

    # Find the existing CuPy wheel installation.
    installed = _find_installed_packages()
    if 1 < len(installed):
        raise AutoDetectionFailed(
            'You have multiple CuPy packages installed: \n'
            f'  {installed}\n'
            'Please uninstall all of them first, then try reinstalling.')

    elif 1 == len(installed):
        if installed[0] in PACKAGES_SDIST:
            raise AutoDetectionFailed(
                'You already have CuPy installed via source'
                ' (pip install cupy).')
        if installed[0] in PACKAGES_OUTDATED:
            raise AutoDetectionFailed(
                f'You have CuPy package "{installed[0]}" installed, but the'
                f' package is not available for version {VERSION}.')
        return installed[0]

    # Try CUDA.
    version = _get_cuda_version()
    if version is not None:
        return _cuda_version_to_package(version)

    # Try ROCm.
    version = _get_rocm_version()
    if version is not None:
        return _rocm_version_to_package(version)

    raise AutoDetectionFailed(
        'Unable to detect NVIDIA CUDA or AMD ROCm installation.')

#
# Entrypoint
#

def main() -> None:
    if os.environ.get('CUPY_UNIVERSAL_PKG_BUILD', None) is None:
        package = infer_best_package()
        requires = f'{package}=={VERSION}'
        _log(f'Installing package: {requires}')
        install_requires = [requires]
    else:
        _log('Building cupy-wheel package for release.')
        install_requires = []

    setup(
        name='cupy-wheel',
        version=VERSION,
        install_requires=install_requires,
    )

if __name__ == '__main__':
    main()
<commit_msg>fix Windows DLL name<commit_after>import ctypes
import pkg_resources
import os
import sys
from typing import List, Optional

from setuptools import setup

VERSION = '10.4.0'

# List of packages supported by this version of CuPy.
PACKAGES = [
    'cupy-cuda102',
    'cupy-cuda110',
    'cupy-cuda111',
    'cupy-cuda112',
    'cupy-cuda113',
    'cupy-cuda114',
    'cupy-cuda115',
    'cupy-cuda116',
    'cupy-rocm-4-0',
    'cupy-rocm-4-2',
    'cupy-rocm-4-3',
    'cupy-rocm-5-0',
]

# List of packages NOT supported by this version of CuPy.
PACKAGES_OUTDATED = [
    'cupy-cuda80',
    'cupy-cuda90',
    'cupy-cuda91',
    'cupy-cuda92',
    'cupy-cuda100',
    'cupy-cuda101',
]

# List of sdist packages.
PACKAGES_SDIST = [
    'cupy',
]

class AutoDetectionFailed(Exception):
    pass

def _log(msg: str) -> None:
    sys.stdout.write(f'[cupy-wheel] {msg}\n')
    sys.stdout.flush()

def _get_version_from_library(
        libnames: List[str], funcname: str) -> Optional[int]:
    """Returns the library version from list of candidate libraries."""

    for libname in libnames:
        try:
            _log(f'Looking for library: {libname}')
            runtime_so = ctypes.CDLL(libname)
            break
        except Exception as e:
            _log(f'Failed to open {libname}: {e}')
            return None
    else:
        _log('No more candidate library to find')
        return None

    func = getattr(runtime_so, funcname, None)
    if func is None:
        raise AutoDetectionFailed(
            f'{libname}: {func} could not be found')
    func.restype = ctypes.c_int  # cudaError_t
    func.argtypes = [ctypes.POINTER(ctypes.c_int)]
    version_ptr = ctypes.c_int()
    retval = func(version_ptr)
    if retval != 0:
        raise AutoDetectionFailed(
            f'{libname}: {func} returned error: {retval}')
    version = version_ptr.value
    _log(f'Detected version: {version}')
    return version

def _get_cuda_version() -> Optional[int]:
    """Returns the detected CUDA version or None."""

    if sys.platform == 'linux':
        libnames = ['libcudart.so']
    elif sys.platform == 'win32':
        libnames = ['cudart64_110.dll', 'cudart64_102.dll']
    else:
        _log(f'CUDA detection unsupported on platform: {sys.platform}')
        return None
    _log(f'Trying to detect CUDA version from libraries: {libnames}')
    version = _get_version_from_library(libnames, 'cudaRuntimeGetVersion')
    return version

def _get_rocm_version() -> Optional[int]:
    """Returns the detected ROCm version or None."""
    if sys.platform == 'linux':
        libnames = ['libamdhip64.so']
    else:
        _log(f'ROCm detection unsupported on platform: {sys.platform}')
        return None
    version = _get_version_from_library(libnames, 'hipRuntimeGetVersion')
    return version

def _find_installed_packages() -> List[str]:
    """Returns the list of CuPy packages installed in the environment."""

    found = []
    for pkg in (PACKAGES + PACKAGES_OUTDATED + PACKAGES_SDIST):
        try:
            pkg_resources.get_distribution(pkg)
            found.append(pkg)
        except pkg_resources.DistributionNotFound:
            pass
    return found

def _cuda_version_to_package(ver: int) -> str:
    if ver < 10020:
        raise AutoDetectionFailed(
            f'Your CUDA version ({ver}) is too old.')
    elif ver < 11000:
        # CUDA 10.2
        suffix = '102'
    elif ver < 11010:
        # CUDA 11.0
        suffix = '110'
    elif ver < 11020:
        # CUDA 11.1
        suffix = '111'
    elif ver < 11030:
        # CUDA 11.2
        suffix = '112'
    elif ver < 11040:
        # CUDA 11.3
        suffix = '113'
    elif ver < 11050:
        # CUDA 11.4
        suffix = '114'
    elif ver < 11060:
        # CUDA 11.5
        suffix = '115'
    elif ver < 11070:
        # CUDA 11.6
        suffix = '116'
    else:
        raise AutoDetectionFailed(
            f'Your CUDA version ({ver}) is too new.')
    return f'cupy-cuda{suffix}'

def _rocm_version_to_package(ver: int) -> str:
    if 400 <= ver < 410:
        # ROCm 4.0
        suffix = '4-0'
    elif 420 <= ver < 430:
        # ROCm 4.2
        suffix = '4-2'
    elif 4_03_00000 <= ver < 4_04_00000:
        # ROCm 4.3
        suffix = '4-3'
    elif 5_00_00000 <= ver < 5_01_00000:
        # ROCm 5.0
        suffix = '5-0'
    else:
        raise AutoDetectionFailed(
            f'Your ROCm version ({ver}) is unsupported.')
    return f'cupy-rocm-{suffix}'

def infer_best_package() -> str:
    """Returns the appropriate CuPy wheel package name for the environment."""

    # Find the existing CuPy wheel installation.
    installed = _find_installed_packages()
    if 1 < len(installed):
        raise AutoDetectionFailed(
            'You have multiple CuPy packages installed: \n'
            f'  {installed}\n'
            'Please uninstall all of them first, then try reinstalling.')

    elif 1 == len(installed):
        if installed[0] in PACKAGES_SDIST:
            raise AutoDetectionFailed(
                'You already have CuPy installed via source'
                ' (pip install cupy).')
        if installed[0] in PACKAGES_OUTDATED:
            raise AutoDetectionFailed(
                f'You have CuPy package "{installed[0]}" installed, but the'
                f' package is not available for version {VERSION}.')
        return installed[0]

    # Try CUDA.
    version = _get_cuda_version()
    if version is not None:
        return _cuda_version_to_package(version)

    # Try ROCm.
    version = _get_rocm_version()
    if version is not None:
        return _rocm_version_to_package(version)

    raise AutoDetectionFailed(
        'Unable to detect NVIDIA CUDA or AMD ROCm installation.')

#
# Entrypoint
#

def main() -> None:
    if os.environ.get('CUPY_UNIVERSAL_PKG_BUILD', None) is None:
        package = infer_best_package()
        requires = f'{package}=={VERSION}'
        _log(f'Installing package: {requires}')
        install_requires = [requires]
    else:
        _log('Building cupy-wheel package for release.')
        install_requires = []

    setup(
        name='cupy-wheel',
        version=VERSION,
        install_requires=install_requires,
    )

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""Functions useful to all element classes."""

import sys
import argparse
from subprocess import Popen, PIPE, STDOUT, CalledProcessError
# ==============================================================================
__version__ = "0.1"

__copyright__ = "Copyright 2017, devops.center"
__credits__ = ["Bob Lozano", "Gregg Jensen"]
__license__ = ' \
   # Copyright 2014-2017 devops.center llc                                    \
   #                                                                          \
   # Licensed under the Apache License, Version 2.0 (the "License");          \
   # you may not use this file except in compliance with the License.         \
   # You may obtain a copy of the License at                                  \
   #                                                                          \
   #   http://www.apache.org/licenses/LICENSE-2.0                             \
   #                                                                          \
   # Unless required by applicable law or agreed to in writing, software      \
   # distributed under the License is distributed on an "AS IS" BASIS,        \
   # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. \
   # See the License for the specific language governing permissions and      \
   # limitations under the License.                                           \
   # '
__status__ = "Development"
# ==============================================================================

class ElementBase(object):
    """Serves base functionality for all element classes."""

    def __init__(self, nameIn, argList):
        """Constructor for the base class."""
        self.name = nameIn
#        if "CONFIG_FILE" in argList:
#            self.readConfigFile(argList["CONFIG_FILE"])
        if "APP_NAME" in argList:
            self.appName = argList["APP_NAME"]
        if "PROFILE" in argList:
            self.profile = argList["PROFILE"]
        if "REGION" in argList:
            self.region = argList["REGION"]
        if "ENV" in argList:
            self.env = argList["ENV"]
        if "SUFFIX" in argList:
            self.suffix = argList["SUFFIX"]
        if "STACK" in argList:
            self.stack = argList["STACK"]
        if "STACK_DIR" in argList:
            self.stackDir = argList["STACK_DIR"]
        if "PGVERSION" in argList:
            self.postgresVersion = argList["PGVERSION"]
        # print("ElementBase args: {}".format(argList))

#    def readConfigFile(self, theFileName):
#        """Read the config file for the key/value pairs listing the paths."""
#        # check to see if the config file exists
#        try:
#            tmpFileHandle = open(theFileName, 'r')
#            tmpFileHandle.close()
#        except IOError:
#            print("Unable to access the config file: {}".format(theFileName))
#            sys.exit(1)
#
#        self.config = ConfigParser()
#        self.config.read(theFileName)
#
#        for keyName, aValue in self.config.items("settings"):
#            if not hasattr(self, keyName):
#                setattr(self, keyName, aValue)

    def runScript(self, shellScript):
        """Execute the passed in shell script."""
        print(self.__class__.__name__ + " EXECUTING: " + shellScript)
        # subprocess.call(shellScript, shell=True)
        with Popen(shellScript, stdout=PIPE, bufsize=1,
                   universal_newlines=True) as p:
            for line in p.stdout:
                print(line, end='')  # process line here

        if p.returncode != 0:
            raise CalledProcessError(p.returncode, p.args)
            sys.exit(1)

    def priorToRun(self):
        """Execute steps prior to running."""
        return

    def postRunning(self):
        """Execute steps after the run has completed."""
        return

def checkArgs():
    """Check the command line arguments."""
    parser = argparse.ArgumentParser(
        description=('comment'))
    parser.parse_args()

def main(argv):
    """Main code goes here."""
    checkArgs()

if __name__ == "__main__":
    main(sys.argv[1:])

# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
<commit_msg>add shell=True to get the environment<commit_after>#!/usr/bin/env python3
"""Functions useful to all element classes."""

import sys
import argparse
from subprocess import Popen, PIPE, STDOUT, CalledProcessError
# ==============================================================================
__version__ = "0.1"

__copyright__ = "Copyright 2017, devops.center"
__credits__ = ["Bob Lozano", "Gregg Jensen"]
__license__ = ' \
   # Copyright 2014-2017 devops.center llc                                    \
   #                                                                          \
   # Licensed under the Apache License, Version 2.0 (the "License");          \
   # you may not use this file except in compliance with the License.         \
   # You may obtain a copy of the License at                                  \
   #                                                                          \
   #   http://www.apache.org/licenses/LICENSE-2.0                             \
   #                                                                          \
   # Unless required by applicable law or agreed to in writing, software      \
   # distributed under the License is distributed on an "AS IS" BASIS,        \
   # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. \
   # See the License for the specific language governing permissions and      \
   # limitations under the License.                                           \
   # '
__status__ = "Development"
# ==============================================================================

class ElementBase(object):
    """Serves base functionality for all element classes."""

    def __init__(self, nameIn, argList):
        """Constructor for the base class."""
        self.name = nameIn
#        if "CONFIG_FILE" in argList:
#            self.readConfigFile(argList["CONFIG_FILE"])
        if "APP_NAME" in argList:
            self.appName = argList["APP_NAME"]
        if "PROFILE" in argList:
            self.profile = argList["PROFILE"]
        if "REGION" in argList:
            self.region = argList["REGION"]
        if "ENV" in argList:
            self.env = argList["ENV"]
        if "SUFFIX" in argList:
            self.suffix = argList["SUFFIX"]
        if "STACK" in argList:
            self.stack = argList["STACK"]
        if "STACK_DIR" in argList:
            self.stackDir = argList["STACK_DIR"]
        if "PGVERSION" in argList:
            self.postgresVersion = argList["PGVERSION"]
        # print("ElementBase args: {}".format(argList))

#    def readConfigFile(self, theFileName):
#        """Read the config file for the key/value pairs listing the paths."""
#        # check to see if the config file exists
#        try:
#            tmpFileHandle = open(theFileName, 'r')
#            tmpFileHandle.close()
#        except IOError:
#            print("Unable to access the config file: {}".format(theFileName))
#            sys.exit(1)
#
#        self.config = ConfigParser()
#        self.config.read(theFileName)
#
#        for keyName, aValue in self.config.items("settings"):
#            if not hasattr(self, keyName):
#                setattr(self, keyName, aValue)

    def runScript(self, shellScript):
        """Execute the passed in shell script."""
        print(self.__class__.__name__ + " EXECUTING: " + shellScript)
        # subprocess.call(shellScript, shell=True)
        with Popen(shellScript, shell=True, stdout=PIPE, bufsize=1,
                   universal_newlines=True) as p:
            for line in p.stdout:
                print(line, end='')  # process line here

        if p.returncode != 0:
            raise CalledProcessError(p.returncode, p.args)
            sys.exit(1)

    def priorToRun(self):
        """Execute steps prior to running."""
        return

    def postRunning(self):
        """Execute steps after the run has completed."""
        return

def checkArgs():
    """Check the command line arguments."""
    parser = argparse.ArgumentParser(
        description=('comment'))
    parser.parse_args()

def main(argv):
    """Main code goes here."""
    checkArgs()

if __name__ == "__main__":
    main(sys.argv[1:])

# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
<|endoftext|>


<commit_before>import datetime

import magic
import pytz
from laboratory import settings
import simplejson as json
from dateutil.relativedelta import relativedelta
import directions.models as directions
from directory.models import Fractions
from laboratory.settings import (
    DEATH_RESEARCH_PK,
    DEF_LABORATORY_AUTH_PK,
    DEF_LABORATORY_LEGAL_AUTH_PK,
)

from results.sql_func import get_paraclinic_results_by_direction, get_laboratory_results_by_directions
from users.models import DoctorProfile
from utils.dates import normalize_dots_date
from directions.models import Napravleniya
from utils.nsi_directories import NSI

def get_json_protocol_data(pk):
    result_protocol = get_paraclinic_results_by_direction(pk)
    data = {}
    document = {}
    for r in result_protocol:
        if "{" in r.value and "}" in r.value:
            try:
                val = json.loads(r.value)
                if not val or not isinstance(val, dict):
                    pass
            except Exception:
                val = r.value
        else:
            val = r.value
        if "rows" in val:
            for k in val['rows']:
                count = 0
                for el in k:
                    if "{" in el and "}" in el:
                        try:
                            el = json.loads(el)
                            if not el or not isinstance(el, dict):
                                pass
                            else:
                                k[count] = el
                        except Exception:
                            pass
                    count += 1
        if isinstance(val, str):
            if val.strip() in ('-', ''):
                val = ""
        if r.title == "Страховая ОМС":
            nsi_smo_code = NSI.get("1.2.643.5.1.13.13.99.2.183_smo_id", None)
            if val and nsi_smo_code:
                smo_id = nsi_smo_code["values"][val.get("code", "")]
                val["id"] = smo_id

        data[r.title] = val

    iss = directions.Issledovaniya.objects.get(napravleniye_id=pk)
    if iss.research_id == DEATH_RESEARCH_PK:
        data_direct_death = data.get("а) Болезнь или состояние, непосредственно приведшее к смерти", None)
        if data_direct_death:
            period_befor_death = data_direct_death["rows"][0][0]
            type_period_befor_death = data_direct_death["rows"][0][1]
            date_death = data["Дата смерти"]
            time_death = data.get("Время смерти", "00:00")
            if period_befor_death and type_period_befor_death:
                data["Начало патологии"] = start_pathological_process(f"{date_death} {time_death}", int(period_befor_death), type_period_befor_death)
    doctor_confirm_obj = iss.doc_confirmation
    author_data = author_doctor(doctor_confirm_obj)

    legal_auth = data.get("Подпись от организации", None)
    legal_auth_data = legal_auth_get(legal_auth)
    hosp_obj = doctor_confirm_obj.hospital
    hosp_oid = hosp_obj.oid

    document["id"] = pk
    time_confirm = iss.time_confirmation
    confirmed_time = time_confirm.astimezone(pytz.timezone(settings.TIME_ZONE)).strftime('%Y%m%d%H%M')
    document["confirmedAt"] = f"{confirmed_time}+0800"
    document["legalAuthenticator"] = legal_auth_data
    document["author"] = author_data
    document["content"] = data
    document["content"]["Код ОКПО"] = hosp_obj.okpo
    document["oidMo"] = hosp_oid
    document["organization"] = organization_get(hosp_obj)
    document["orgName"] = hosp_obj.title
    document["tel"] = hosp_obj.phones

    return document

def get_json_labortory_data(pk):
    result_protocol = get_laboratory_results_by_directions([pk])
    document = {}
    confirmed_at = ""
    date_reiceve = ""
    data = []
    prev_research_title = ""
    count = 0
    tests = []
    author_data = None
    iss = None
    for k in result_protocol:
        iss = directions.Issledovaniya.objects.get(pk=k.iss_id)
        next_research_title = iss.research.title
        if (prev_research_title != next_research_title) and count != 0:
            if len(tests) > 0:
                data.append({"title": prev_research_title, "tests": tests, "confirmedAt": confirmed_at, "receivedAt": date_reiceve, "author_data": author_data})
            tests = []
        author_data = author_doctor(iss.doc_confirmation)
        time_confirm = iss.time_confirmation
        confirmed_time = time_confirm.astimezone(pytz.timezone(settings.TIME_ZONE)).strftime('%Y%m%d%H%M')
        fraction_id = k.fraction_id
        frac_obj = Fractions.objects.get(pk=fraction_id)
        if not frac_obj.unit or not frac_obj.fsli:
            continue
        unit_obj = frac_obj.unit
        unit_val = {"code": unit_obj.code, "full_title": unit_obj.title, "ucum": unit_obj.ucum, "short_title": unit_obj.short_title}
        flsi_param = {"code": frac_obj.fsli, "title": frac_obj.title}
        result_val = k.value.replace(",", ".") if k.value else ""
        confirmed_at = f"{confirmed_time}+0800"
        date_reiceve = normalize_dots_date(k.date_confirm).replace("-", "")
        date_reiceve = f"{date_reiceve}0800+0800"
        tests.append({"unit_val": unit_val, "flsi_param": flsi_param, "result_val": result_val})
        prev_research_title = next_research_title
        count += 1
    if len(tests) > 0:
        data.append({"title": prev_research_title, "tests": tests, "confirmedAt": confirmed_at, "receivedAt": date_reiceve, "author_data": author_data})

    hosp_obj = iss.doc_confirmation.hospital
    hosp_oid = hosp_obj.oid

    document["id"] = pk
    legal_auth_data = legal_auth_get({"id": iss.legal_authenticator_id})
    document["legalAuthenticator"] = legal_auth_data
    document["author"] = author_data
    document["content"] = {}
    document["content"]["Лаборатория"] = data
    document["content"]["Код ОКПО"] = hosp_obj.okpo
    document["content"]["payment"] = {"code": "1", "title": "Средства обязательного медицинского страхования"}
    document["oidMo"] = hosp_oid
    document["orgName"] = hosp_obj.title
    direction_obj = Napravleniya.objects.get(pk=pk)
    direction_create = direction_obj.data_sozdaniya
    direction_create = direction_create.astimezone(pytz.timezone(settings.TIME_ZONE)).strftime('%Y%m%d%H%M')
    document["createdAt"] = f"{direction_create}+0800"
    document["lastConfirmedAt"] = f"{confirmed_at}"
    document["confirmedAt"] = f"{confirmed_at}"
    document["organization"] = organization_get(hosp_obj)

    return document

def organization_get(hosp_obj_f):
    return {
        "name": hosp_obj_f.title,
        "tel": hosp_obj_f.phones if hosp_obj_f.phones else "",
        "address": {
            "text": hosp_obj_f.address if hosp_obj_f.address else "г. Иркутск",
            "subjectCode": "38",
            "subjectName": "Иркутская область",
        },
    }

def author_doctor(doctor_confirm_obj, is_recursion=False):
    author = {}
    author["id"] = doctor_confirm_obj.pk
    author["positionCode"] = doctor_confirm_obj.position.n3_id if doctor_confirm_obj.position else ""
    author["positionName"] = doctor_confirm_obj.position.title if doctor_confirm_obj.position else ""
    author["snils"] = doctor_confirm_obj.snils if doctor_confirm_obj.snils else ""
    if "" in [author["positionCode"], author["positionName"], author["snils"]] and DEF_LABORATORY_AUTH_PK and not is_recursion:
        return author_doctor(DoctorProfile.objects.get(pk=DEF_LABORATORY_AUTH_PK), True)
    author["name"] = {}
    author["name"]["family"] = doctor_confirm_obj.family
    author["name"]["name"] = doctor_confirm_obj.name
    author["name"]["patronymic"] = doctor_confirm_obj.patronymic
    return author

def legal_auth_get(legal_auth_doc, is_recursion=False):
    legal_auth = {"id": "", "snils": "", "positionCode": "", "positionName": "", "name": {"family": "", "name": "", "patronymic": ""}}
    if legal_auth_doc and legal_auth_doc["id"]:
        id_doc = legal_auth_doc["id"]
        legal_doctor = DoctorProfile.objects.get(pk=id_doc)
        legal_auth["id"] = legal_doctor.pk
        legal_auth["snils"] = legal_doctor.snils
        legal_auth["positionCode"] = legal_doctor.position.n3_id
        legal_auth["positionName"] = legal_doctor.position.title
        legal_auth["name"]["family"] = legal_doctor.family
        legal_auth["name"]["name"] = legal_doctor.name
        legal_auth["name"]["patronymic"] = legal_doctor.patronymic
    if (legal_auth["positionCode"] not in [7] or "" in [legal_auth["positionCode"], legal_auth["positionName"], legal_auth["snils"]]) and DEF_LABORATORY_LEGAL_AUTH_PK and not is_recursion:
        return legal_auth_get({"id": DEF_LABORATORY_LEGAL_AUTH_PK}, True)
    return legal_auth

def start_pathological_process(date_death, time_data, type_period):
    dt = datetime.datetime.strptime(date_death, '%Y-%m-%d %H:%M')
    period = {
        "часов": relativedelta(hours=-time_data),
        "минут": relativedelta(minutes=-time_data),
        "дней": relativedelta(days=-time_data),
        "суток": relativedelta(days=-time_data),
        "месяцев": relativedelta(months=-time_data),
        "лет": relativedelta(years=-time_data),
    }
    delta = dt + period[type_period]
    delta.strftime("%Y%m%d%H:%M:%S")
    return f"{delta.strftime('%Y%m%d%H%M')}+0800"

def check_type_file(file):
    type_file = magic.from_buffer(open(file).read(2048)).lower()
    if "pdf" in type_file or "jpeg" in type_file:
        return True
    return False
<commit_msg>rename arg file to file_path<commit_after>import datetime

import magic
import pytz
from laboratory import settings
import simplejson as json
from dateutil.relativedelta import relativedelta
import directions.models as directions
from directory.models import Fractions
from laboratory.settings import (
    DEATH_RESEARCH_PK,
    DEF_LABORATORY_AUTH_PK,
    DEF_LABORATORY_LEGAL_AUTH_PK,
)

from results.sql_func import get_paraclinic_results_by_direction, get_laboratory_results_by_directions
from users.models import DoctorProfile
from utils.dates import normalize_dots_date
from directions.models import Napravleniya
from utils.nsi_directories import NSI

def get_json_protocol_data(pk):
    result_protocol = get_paraclinic_results_by_direction(pk)
    data = {}
    document = {}
    for r in result_protocol:
        if "{" in r.value and "}" in r.value:
            try:
                val = json.loads(r.value)
                if not val or not isinstance(val, dict):
                    pass
            except Exception:
                val = r.value
        else:
            val = r.value
        if "rows" in val:
            for k in val['rows']:
                count = 0
                for el in k:
                    if "{" in el and "}" in el:
                        try:
                            el = json.loads(el)
                            if not el or not isinstance(el, dict):
                                pass
                            else:
                                k[count] = el
                        except Exception:
                            pass
                    count += 1
        if isinstance(val, str):
            if val.strip() in ('-', ''):
                val = ""
        if r.title == "Страховая ОМС":
            nsi_smo_code = NSI.get("1.2.643.5.1.13.13.99.2.183_smo_id", None)
            if val and nsi_smo_code:
                smo_id = nsi_smo_code["values"][val.get("code", "")]
                val["id"] = smo_id

        data[r.title] = val

    iss = directions.Issledovaniya.objects.get(napravleniye_id=pk)
    if iss.research_id == DEATH_RESEARCH_PK:
        data_direct_death = data.get("а) Болезнь или состояние, непосредственно приведшее к смерти", None)
        if data_direct_death:
            period_befor_death = data_direct_death["rows"][0][0]
            type_period_befor_death = data_direct_death["rows"][0][1]
            date_death = data["Дата смерти"]
            time_death = data.get("Время смерти", "00:00")
            if period_befor_death and type_period_befor_death:
                data["Начало патологии"] = start_pathological_process(f"{date_death} {time_death}", int(period_befor_death), type_period_befor_death)
    doctor_confirm_obj = iss.doc_confirmation
    author_data = author_doctor(doctor_confirm_obj)

    legal_auth = data.get("Подпись от организации", None)
    legal_auth_data = legal_auth_get(legal_auth)
    hosp_obj = doctor_confirm_obj.hospital
    hosp_oid = hosp_obj.oid

    document["id"] = pk
    time_confirm = iss.time_confirmation
    confirmed_time = time_confirm.astimezone(pytz.timezone(settings.TIME_ZONE)).strftime('%Y%m%d%H%M')
    document["confirmedAt"] = f"{confirmed_time}+0800"
    document["legalAuthenticator"] = legal_auth_data
    document["author"] = author_data
    document["content"] = data
    document["content"]["Код ОКПО"] = hosp_obj.okpo
    document["oidMo"] = hosp_oid
    document["organization"] = organization_get(hosp_obj)
    document["orgName"] = hosp_obj.title
    document["tel"] = hosp_obj.phones

    return document

def get_json_labortory_data(pk):
    result_protocol = get_laboratory_results_by_directions([pk])
    document = {}
    confirmed_at = ""
    date_reiceve = ""
    data = []
    prev_research_title = ""
    count = 0
    tests = []
    author_data = None
    iss = None
    for k in result_protocol:
        iss = directions.Issledovaniya.objects.get(pk=k.iss_id)
        next_research_title = iss.research.title
        if (prev_research_title != next_research_title) and count != 0:
            if len(tests) > 0:
                data.append({"title": prev_research_title, "tests": tests, "confirmedAt": confirmed_at, "receivedAt": date_reiceve, "author_data": author_data})
            tests = []
        author_data = author_doctor(iss.doc_confirmation)
        time_confirm = iss.time_confirmation
        confirmed_time = time_confirm.astimezone(pytz.timezone(settings.TIME_ZONE)).strftime('%Y%m%d%H%M')
        fraction_id = k.fraction_id
        frac_obj = Fractions.objects.get(pk=fraction_id)
        if not frac_obj.unit or not frac_obj.fsli:
            continue
        unit_obj = frac_obj.unit
        unit_val = {"code": unit_obj.code, "full_title": unit_obj.title, "ucum": unit_obj.ucum, "short_title": unit_obj.short_title}
        flsi_param = {"code": frac_obj.fsli, "title": frac_obj.title}
        result_val = k.value.replace(",", ".") if k.value else ""
        confirmed_at = f"{confirmed_time}+0800"
        date_reiceve = normalize_dots_date(k.date_confirm).replace("-", "")
        date_reiceve = f"{date_reiceve}0800+0800"
        tests.append({"unit_val": unit_val, "flsi_param": flsi_param, "result_val": result_val})
        prev_research_title = next_research_title
        count += 1
    if len(tests) > 0:
        data.append({"title": prev_research_title, "tests": tests, "confirmedAt": confirmed_at, "receivedAt": date_reiceve, "author_data": author_data})

    hosp_obj = iss.doc_confirmation.hospital
    hosp_oid = hosp_obj.oid

    document["id"] = pk
    legal_auth_data = legal_auth_get({"id": iss.legal_authenticator_id})
    document["legalAuthenticator"] = legal_auth_data
    document["author"] = author_data
    document["content"] = {}
    document["content"]["Лаборатория"] = data
    document["content"]["Код ОКПО"] = hosp_obj.okpo
    document["content"]["payment"] = {"code": "1", "title": "Средства обязательного медицинского страхования"}
    document["oidMo"] = hosp_oid
    document["orgName"] = hosp_obj.title
    direction_obj = Napravleniya.objects.get(pk=pk)
    direction_create = direction_obj.data_sozdaniya
    direction_create = direction_create.astimezone(pytz.timezone(settings.TIME_ZONE)).strftime('%Y%m%d%H%M')
    document["createdAt"] = f"{direction_create}+0800"
    document["lastConfirmedAt"] = f"{confirmed_at}"
    document["confirmedAt"] = f"{confirmed_at}"
    document["organization"] = organization_get(hosp_obj)

    return document

def organization_get(hosp_obj_f):
    return {
        "name": hosp_obj_f.title,
        "tel": hosp_obj_f.phones if hosp_obj_f.phones else "",
        "address": {
            "text": hosp_obj_f.address if hosp_obj_f.address else "г. Иркутск",
            "subjectCode": "38",
            "subjectName": "Иркутская область",
        },
    }

def author_doctor(doctor_confirm_obj, is_recursion=False):
    author = {}
    author["id"] = doctor_confirm_obj.pk
    author["positionCode"] = doctor_confirm_obj.position.n3_id if doctor_confirm_obj.position else ""
    author["positionName"] = doctor_confirm_obj.position.title if doctor_confirm_obj.position else ""
    author["snils"] = doctor_confirm_obj.snils if doctor_confirm_obj.snils else ""
    if "" in [author["positionCode"], author["positionName"], author["snils"]] and DEF_LABORATORY_AUTH_PK and not is_recursion:
        return author_doctor(DoctorProfile.objects.get(pk=DEF_LABORATORY_AUTH_PK), True)
    author["name"] = {}
    author["name"]["family"] = doctor_confirm_obj.family
    author["name"]["name"] = doctor_confirm_obj.name
    author["name"]["patronymic"] = doctor_confirm_obj.patronymic
    return author

def legal_auth_get(legal_auth_doc, is_recursion=False):
    legal_auth = {"id": "", "snils": "", "positionCode": "", "positionName": "", "name": {"family": "", "name": "", "patronymic": ""}}
    if legal_auth_doc and legal_auth_doc["id"]:
        id_doc = legal_auth_doc["id"]
        legal_doctor = DoctorProfile.objects.get(pk=id_doc)
        legal_auth["id"] = legal_doctor.pk
        legal_auth["snils"] = legal_doctor.snils
        legal_auth["positionCode"] = legal_doctor.position.n3_id
        legal_auth["positionName"] = legal_doctor.position.title
        legal_auth["name"]["family"] = legal_doctor.family
        legal_auth["name"]["name"] = legal_doctor.name
        legal_auth["name"]["patronymic"] = legal_doctor.patronymic
    if (legal_auth["positionCode"] not in [7] or "" in [legal_auth["positionCode"], legal_auth["positionName"], legal_auth["snils"]]) and DEF_LABORATORY_LEGAL_AUTH_PK and not is_recursion:
        return legal_auth_get({"id": DEF_LABORATORY_LEGAL_AUTH_PK}, True)
    return legal_auth

def start_pathological_process(date_death, time_data, type_period):
    dt = datetime.datetime.strptime(date_death, '%Y-%m-%d %H:%M')
    period = {
        "часов": relativedelta(hours=-time_data),
        "минут": relativedelta(minutes=-time_data),
        "дней": relativedelta(days=-time_data),
        "суток": relativedelta(days=-time_data),
        "месяцев": relativedelta(months=-time_data),
        "лет": relativedelta(years=-time_data),
    }
    delta = dt + period[type_period]
    delta.strftime("%Y%m%d%H:%M:%S")
    return f"{delta.strftime('%Y%m%d%H%M')}+0800"

def check_type_file(file_path):
    type_file = magic.from_buffer(open(file_path).read(2048)).lower()
    if "pdf" in type_file or "jpeg" in type_file:
        return True
    return False
<|endoftext|>