<commit_before>#
# Description:
#   This module implements the functions needed to keep the
#   required number of idle glideins
#   plus other miscelaneous functions
#
# Author:
#   Igor Sfiligoi (Sept 19th 2006)
#

import os.path
import sets,string
import condorMonitor,condorExe
import logSupport

class LogFiles:
    def __init__(self,log_dir,max_days,min_days,max_mbs):
        self.log_dir=log_dir
        self.activity_log=logSupport.DayLogFile(os.path.join(log_dir,"frontend"),"info.log")
        self.warning_log=logSupport.DayLogFile(os.path.join(log_dir,"frontend"),"err.log")
        self.debug_log=logSupport.DayLogFile(os.path.join(log_dir,"frontend"),"debug.log")
        self.cleanupObj=logSupport.DirCleanupWSpace(log_dir,"(frontend\.[0-9]*\.info\.log)|(frontend\.[0-9]*\.err\.log)|(frontend\.[0-9]*\.debug\.log)",
                                                    int(max_days*24*3600),int(min_days*24*3600),
                                                    long(max_mbs*(1024.0*1024.0)),
                                                    self.activity_log,self.warning_log)

    def logActivity(self,str):
        try:
            self.activity_log.write(str)
        except:
            # logging must never throw an exception!
            self.logWarning("logActivity failed, was logging: %s"%str,False)

    def logWarning(self,str, log_in_activity=True):
        try:
            self.warning_log.write(str)
        except:
            # logging must throw an exception!
            # silently ignore
            pass
        if log_in_activity:
            self.logActivity("WARNING: %s"%str)

    def logDebug(self,str):
        try:
            self.debug_log.write(str)
        except:
            # logging must never throw an exception!
            # silently ignore
            pass

    def cleanup(self):
        try:
            self.cleanupObj.cleanup()
        except:
            # logging must never throw an exception!
            self.logWarning("log cleanup failed.")

# someone needs to initialize this
# type LogFiles
log_files=None

#############################################################################################

#
# Return a dictionary of schedds containing interesting jobs
# Each element is a condorQ
#
# If not all the jobs of the schedd has to be considered,
# specify the appropriate constraint
#
def getCondorQ(schedd_names,constraint=None,format_list=None):
    if format_list!=None:
        format_list=condorMonitor.complete_format_list(format_list,[('JobStatus','i'),('EnteredCurrentStatus','i'),('ServerTime','i')])
    return getCondorQConstrained(schedd_names,"(JobStatus=?=1)||(JobStatus=?=2)",constraint,format_list)

#
# Return a dictionary of schedds containing idle jobs
# Each element is a condorQ
#
# Use the output of getCondorQ
#
def getIdleCondorQ(condorq_dict):
    out={}
    for schedd_name in condorq_dict.keys():
        sq=condorMonitor.SubQuery(condorq_dict[schedd_name],lambda el:(el.has_key('JobStatus') and (el['JobStatus']==1)))
        sq.load()
        out[schedd_name]=sq
    return out

#
# Return a dictionary of schedds containing running jobs
# Each element is a condorQ
#
# Use the output of getCondorQ
#
def getRunningCondorQ(condorq_dict):
    out={}
    for schedd_name in condorq_dict.keys():
        sq=condorMonitor.SubQuery(condorq_dict[schedd_name],lambda el:(el.has_key('JobStatus') and (el['JobStatus']==2)))
        sq.load()
        out[schedd_name]=sq
    return out

#
# Return a dictionary of schedds containing old jobs
# Each element is a condorQ
#
# Use the output of getCondorQ
#
def getOldCondorQ(condorq_dict,min_age):
    out={}
    for schedd_name in condorq_dict.keys():
        sq=condorMonitor.SubQuery(condorq_dict[schedd_name],lambda el:(el.has_key('ServerTime') and el.has_key('EnteredCurrentStatus') and ((el['ServerTime']-el['EnteredCurrentStatus'])>=min_age)))
        sq.load()
        out[schedd_name]=sq
    return out

#
# Return the number of jobs in the dictionary
# Use the output of getCondorQ
#
def countCondorQ(condorq_dict):
    count=0
    for schedd_name in condorq_dict.keys():
        count+=len(condorq_dict[schedd_name].fetchStored())
    return count

#
# Return a set of users present in the dictionary
# Needs "User" attribute
#

def getCondorQUsers(condorq_dict):
    users_set=sets.Set()
    for schedd_name in condorq_dict.keys():
        condorq_data=condorq_dict[schedd_name].fetchStored()
        for jid in condorq_data.keys():
            job=condorq_data[jid]
            users_set.add(job['User'])

    return users_set

#
# Get the number of jobs that match each glidein
#
# match_obj = compile('(job["MIN_NAME"]<glidein["MIN_NAME"]) && (job["ARCH"]==glidein["ARCH"])',"<string>","eval")
# condorq_dict = output of getidlqCondorQ
# glidein_dict = output of interface.findGlideins
#
# Returns:
#  dictionary of glidein name
#   where elements are number of idle jobs matching
def countMatch(match_obj,condorq_dict,glidein_dict):
    out_glidein_counts={}
    for glidename in glidein_dict:
        glidein=glidein_dict[glidename]
        glidein_count=0
        for schedd in condorq_dict.keys():
            condorq=condorq_dict[schedd]
            condorq_data=condorq.fetchStored()
            schedd_count=0
            for jid in condorq_data.keys():
                job=condorq_data[jid]
                if eval(match_obj):
                    schedd_count+=1
                pass
            glidein_count+=schedd_count
            pass
        out_glidein_counts[glidename]=glidein_count
        pass
    return out_glidein_counts

#
# Convert frontend param expression in a value
#
# expr_obj = compile('glidein["MaxTimeout"]+frontend["MaxTimeout"]+600',"<string>","eval")
# frontend = the frontend const parameters
# glidein  = glidein factory parameters
#
# Returns:
#  The evaluated value
def evalParamExpr(expr_obj,frontend,glidein):
    return eval(expr_obj)

#
# Return a dictionary of collectors containing interesting classads
# Each element is a condorStatus
#
# If not all the jobs of the schedd has to be considered,
# specify the appropriate constraint
#
def getCondorStatus(collector_names,constraint=None,format_list=None):
    if format_list!=None:
        format_list=condorMonitor.complete_format_list(format_list,[('State','s'),('Activity','s'),('EnteredCurrentState','i'),('EnteredCurrentActivity','i'),('LastHeardFrom','i'),('GLIDEIN_Factory','s'),('GLIDEIN_Name','s'),('GLIDEIN_Entry_Name','s'),('GLIDECLIENT_Name','s')])
    return getCondorStatusConstrained(collector_names,'IS_MONITOR_VM=!=True',constraint,format_list)

#
# Return a dictionary of collectors containing idle(unclaimed) vms
# Each element is a condorStatus
#
# Use the output of getCondorStatus
#
def getIdleCondorStatus(status_dict):
    out={}
    for collector_name in status_dict.keys():
        sq=condorMonitor.SubQuery(status_dict[collector_name],lambda el:(el.has_key('State') and el.has_key('Activity') and (el['State']=="Unclaimed") and (el['Activity']=="Idle")))
        sq.load()
        out[collector_name]=sq
    return out

#
# Return a dictionary of collectors containing running(claimed) vms
# Each element is a condorStatus
#
# Use the output of getCondorStatus
#
def getRunningCondorStatus(status_dict):
    out={}
    for collector_name in status_dict.keys():
        sq=condorMonitor.SubQuery(status_dict[collector_name],lambda el:(el.has_key('State') and el.has_key('Activity') and (el['State']=="Claimed") and (el['Activity'] in ("Busy","Retiring"))))
        sq.load()
        out[collector_name]=sq
    return out

#
# Return a dictionary of collectors containing idle(unclaimed) vms
# Each element is a condorStatus
#
# Use the output of getCondorStatus
#
def getClientCondorStatus(status_dict,frontend_name,group_name,request_name):
    client_name="%s@%s.%s"%(request_name,frontend_name,group_name)
    out={}
    for collector_name in status_dict.keys():
        sq=condorMonitor.SubQuery(status_dict[collector_name],lambda el:(el.has_key('GLIDECLIENT_Name') and (el['GLIDECLIENT_Name']==client_name)))
        sq.load()
        out[collector_name]=sq
    return out

#
# Return the number of vms in the dictionary
# Use the output of getCondorStatus
#
def countCondorStatus(status_dict):
    count=0
    for collector_name in status_dict.keys():
        count+=len(status_dict[collector_name].fetchStored())
    return count

############################################################
#
# I N T E R N A L - Do not use
#
############################################################

#
# Return a dictionary of schedds containing jobs of a certain type
# Each element is a condorQ
#
# If not all the jobs of the schedd has to be considered,
# specify the appropriate additional constraint
#
def getCondorQConstrained(schedd_names,type_constraint,constraint=None,format_list=None):
    out_condorq_dict={}
    for schedd in schedd_names:
        if schedd=='':
            log_files.logWarning("Skipping empty schedd name")
            continue
        condorq=condorMonitor.CondorQ(schedd)
        full_constraint=type_constraint[0:] #make copy
        if constraint!=None:
            full_constraint="(%s) && (%s)"%(full_constraint,constraint)

        try:
            condorq.load(full_constraint,format_list)
        except condorExe.ExeError, e:
            if schedd!=None:
                log_files.logWarning("Failed to talk to schedd %s. See debug log for more details."%schedd)
                log_files.logDebug("Failed to talk to schedd %s: %s"%(schedd, e))
            else:
                log_files.logWarning("Failed to talk to schedd. See debug log for more details.")
                log_files.logDebug("Failed to talk to schedd: %s"%e)
            continue # if schedd not found it is equivalent to no jobs in the queue
        if len(condorq.fetchStored())>0:
            out_condorq_dict[schedd]=condorq
    return out_condorq_dict

#
# Return a dictionary of collectors containing classads of a certain kind
# Each element is a condorStatus
#
# If not all the jobs of the schedd has to be considered,
# specify the appropriate additional constraint
#
def getCondorStatusConstrained(collector_names,type_constraint,constraint=None,format_list=None):
    out_status_dict={}
    for collector in collector_names:
        status=condorMonitor.CondorStatus(pool_name=collector)
        full_constraint=type_constraint[0:] #make copy
        if constraint!=None:
            full_constraint="(%s) && (%s)"%(full_constraint,constraint)

        try:
            status.load(full_constraint,format_list)
        except condorExe.ExeError, e:
            if collector!=None:
                log_files.logWarning("Failed to talk to collector %s. See debug log for more details."%collector)
                log_files.logDebug("Failed to talk to collector %s: %s"%(collector, e))
            else:
                log_files.logWarning("Failed to talk to collector. See debug log for more details.")
                log_files.logDebug("Failed to talk to collector: %s"%e)
            continue # if collector not found it is equivalent to no classads
        if len(status.fetchStored())>0:
            out_status_dict[collector]=status
    return out_status_dict

<commit_msg>Since the gfactory changed the semantics of GLIDECLIENT_Name, the frontend needs to be changed, too - merge from v2_4plus<commit_after>#
# Description:
#   This module implements the functions needed to keep the
#   required number of idle glideins
#   plus other miscelaneous functions
#
# Author:
#   Igor Sfiligoi (Sept 19th 2006)
#

import os.path
import sets,string
import condorMonitor,condorExe
import logSupport

class LogFiles:
    def __init__(self,log_dir,max_days,min_days,max_mbs):
        self.log_dir=log_dir
        self.activity_log=logSupport.DayLogFile(os.path.join(log_dir,"frontend"),"info.log")
        self.warning_log=logSupport.DayLogFile(os.path.join(log_dir,"frontend"),"err.log")
        self.debug_log=logSupport.DayLogFile(os.path.join(log_dir,"frontend"),"debug.log")
        self.cleanupObj=logSupport.DirCleanupWSpace(log_dir,"(frontend\.[0-9]*\.info\.log)|(frontend\.[0-9]*\.err\.log)|(frontend\.[0-9]*\.debug\.log)",
                                                    int(max_days*24*3600),int(min_days*24*3600),
                                                    long(max_mbs*(1024.0*1024.0)),
                                                    self.activity_log,self.warning_log)

    def logActivity(self,str):
        try:
            self.activity_log.write(str)
        except:
            # logging must never throw an exception!
            self.logWarning("logActivity failed, was logging: %s"%str,False)

    def logWarning(self,str, log_in_activity=True):
        try:
            self.warning_log.write(str)
        except:
            # logging must throw an exception!
            # silently ignore
            pass
        if log_in_activity:
            self.logActivity("WARNING: %s"%str)

    def logDebug(self,str):
        try:
            self.debug_log.write(str)
        except:
            # logging must never throw an exception!
            # silently ignore
            pass

    def cleanup(self):
        try:
            self.cleanupObj.cleanup()
        except:
            # logging must never throw an exception!
            self.logWarning("log cleanup failed.")

# someone needs to initialize this
# type LogFiles
log_files=None

#############################################################################################

#
# Return a dictionary of schedds containing interesting jobs
# Each element is a condorQ
#
# If not all the jobs of the schedd has to be considered,
# specify the appropriate constraint
#
def getCondorQ(schedd_names,constraint=None,format_list=None):
    if format_list!=None:
        format_list=condorMonitor.complete_format_list(format_list,[('JobStatus','i'),('EnteredCurrentStatus','i'),('ServerTime','i')])
    return getCondorQConstrained(schedd_names,"(JobStatus=?=1)||(JobStatus=?=2)",constraint,format_list)

#
# Return a dictionary of schedds containing idle jobs
# Each element is a condorQ
#
# Use the output of getCondorQ
#
def getIdleCondorQ(condorq_dict):
    out={}
    for schedd_name in condorq_dict.keys():
        sq=condorMonitor.SubQuery(condorq_dict[schedd_name],lambda el:(el.has_key('JobStatus') and (el['JobStatus']==1)))
        sq.load()
        out[schedd_name]=sq
    return out

#
# Return a dictionary of schedds containing running jobs
# Each element is a condorQ
#
# Use the output of getCondorQ
#
def getRunningCondorQ(condorq_dict):
    out={}
    for schedd_name in condorq_dict.keys():
        sq=condorMonitor.SubQuery(condorq_dict[schedd_name],lambda el:(el.has_key('JobStatus') and (el['JobStatus']==2)))
        sq.load()
        out[schedd_name]=sq
    return out

#
# Return a dictionary of schedds containing old jobs
# Each element is a condorQ
#
# Use the output of getCondorQ
#
def getOldCondorQ(condorq_dict,min_age):
    out={}
    for schedd_name in condorq_dict.keys():
        sq=condorMonitor.SubQuery(condorq_dict[schedd_name],lambda el:(el.has_key('ServerTime') and el.has_key('EnteredCurrentStatus') and ((el['ServerTime']-el['EnteredCurrentStatus'])>=min_age)))
        sq.load()
        out[schedd_name]=sq
    return out

#
# Return the number of jobs in the dictionary
# Use the output of getCondorQ
#
def countCondorQ(condorq_dict):
    count=0
    for schedd_name in condorq_dict.keys():
        count+=len(condorq_dict[schedd_name].fetchStored())
    return count

#
# Return a set of users present in the dictionary
# Needs "User" attribute
#

def getCondorQUsers(condorq_dict):
    users_set=sets.Set()
    for schedd_name in condorq_dict.keys():
        condorq_data=condorq_dict[schedd_name].fetchStored()
        for jid in condorq_data.keys():
            job=condorq_data[jid]
            users_set.add(job['User'])

    return users_set

#
# Get the number of jobs that match each glidein
#
# match_obj = compile('(job["MIN_NAME"]<glidein["MIN_NAME"]) && (job["ARCH"]==glidein["ARCH"])',"<string>","eval")
# condorq_dict = output of getidlqCondorQ
# glidein_dict = output of interface.findGlideins
#
# Returns:
#  dictionary of glidein name
#   where elements are number of idle jobs matching
def countMatch(match_obj,condorq_dict,glidein_dict):
    out_glidein_counts={}
    for glidename in glidein_dict:
        glidein=glidein_dict[glidename]
        glidein_count=0
        for schedd in condorq_dict.keys():
            condorq=condorq_dict[schedd]
            condorq_data=condorq.fetchStored()
            schedd_count=0
            for jid in condorq_data.keys():
                job=condorq_data[jid]
                if eval(match_obj):
                    schedd_count+=1
                pass
            glidein_count+=schedd_count
            pass
        out_glidein_counts[glidename]=glidein_count
        pass
    return out_glidein_counts

#
# Convert frontend param expression in a value
#
# expr_obj = compile('glidein["MaxTimeout"]+frontend["MaxTimeout"]+600',"<string>","eval")
# frontend = the frontend const parameters
# glidein  = glidein factory parameters
#
# Returns:
#  The evaluated value
def evalParamExpr(expr_obj,frontend,glidein):
    return eval(expr_obj)

#
# Return a dictionary of collectors containing interesting classads
# Each element is a condorStatus
#
# If not all the jobs of the schedd has to be considered,
# specify the appropriate constraint
#
def getCondorStatus(collector_names,constraint=None,format_list=None):
    if format_list!=None:
        format_list=condorMonitor.complete_format_list(format_list,[('State','s'),('Activity','s'),('EnteredCurrentState','i'),('EnteredCurrentActivity','i'),('LastHeardFrom','i'),('GLIDEIN_Factory','s'),('GLIDEIN_Name','s'),('GLIDEIN_Entry_Name','s'),('GLIDECLIENT_Name','s')])
    return getCondorStatusConstrained(collector_names,'(IS_MONITOR_VM=!=True)&&(GLIDEIN_Factory=!=UNDEFINED)&&(GLIDEIN_Name=!=UNDEFINED)&&(GLIDEIN_Entry_Name=!=UNDEFINED)',constraint,format_list)

#
# Return a dictionary of collectors containing idle(unclaimed) vms
# Each element is a condorStatus
#
# Use the output of getCondorStatus
#
def getIdleCondorStatus(status_dict):
    out={}
    for collector_name in status_dict.keys():
        sq=condorMonitor.SubQuery(status_dict[collector_name],lambda el:(el.has_key('State') and el.has_key('Activity') and (el['State']=="Unclaimed") and (el['Activity']=="Idle")))
        sq.load()
        out[collector_name]=sq
    return out

#
# Return a dictionary of collectors containing running(claimed) vms
# Each element is a condorStatus
#
# Use the output of getCondorStatus
#
def getRunningCondorStatus(status_dict):
    out={}
    for collector_name in status_dict.keys():
        sq=condorMonitor.SubQuery(status_dict[collector_name],lambda el:(el.has_key('State') and el.has_key('Activity') and (el['State']=="Claimed") and (el['Activity'] in ("Busy","Retiring"))))
        sq.load()
        out[collector_name]=sq
    return out

#
# Return a dictionary of collectors containing idle(unclaimed) vms
# Each element is a condorStatus
#
# Use the output of getCondorStatus
#
def getClientCondorStatus(status_dict,frontend_name,group_name,request_name):
    client_name_old="%s@%s.%s"%(request_name,frontend_name,group_name)
    client_name_new="%s.%s"%(frontend_name,group_name)
    out={}
    for collector_name in status_dict.keys():
        sq=condorMonitor.SubQuery(status_dict[collector_name],lambda el:(el.has_key('GLIDECLIENT_Name') and ((el['GLIDECLIENT_Name']==client_name_old) or ((el['GLIDECLIENT_Name']==client_name_new) and (("%s@%s@%s"%(el['GLIDEIN_Entry_Name'],el['GLIDEIN_Name'],el['GLIDEIN_Factory']))==request_name)))))
        sq.load()
        out[collector_name]=sq
    return out

#
# Return the number of vms in the dictionary
# Use the output of getCondorStatus
#
def countCondorStatus(status_dict):
    count=0
    for collector_name in status_dict.keys():
        count+=len(status_dict[collector_name].fetchStored())
    return count

############################################################
#
# I N T E R N A L - Do not use
#
############################################################

#
# Return a dictionary of schedds containing jobs of a certain type
# Each element is a condorQ
#
# If not all the jobs of the schedd has to be considered,
# specify the appropriate additional constraint
#
def getCondorQConstrained(schedd_names,type_constraint,constraint=None,format_list=None):
    out_condorq_dict={}
    for schedd in schedd_names:
        if schedd=='':
            log_files.logWarning("Skipping empty schedd name")
            continue
        condorq=condorMonitor.CondorQ(schedd)
        full_constraint=type_constraint[0:] #make copy
        if constraint!=None:
            full_constraint="(%s) && (%s)"%(full_constraint,constraint)

        try:
            condorq.load(full_constraint,format_list)
        except condorExe.ExeError, e:
            if schedd!=None:
                log_files.logWarning("Failed to talk to schedd %s. See debug log for more details."%schedd)
                log_files.logDebug("Failed to talk to schedd %s: %s"%(schedd, e))
            else:
                log_files.logWarning("Failed to talk to schedd. See debug log for more details.")
                log_files.logDebug("Failed to talk to schedd: %s"%e)
            continue # if schedd not found it is equivalent to no jobs in the queue
        if len(condorq.fetchStored())>0:
            out_condorq_dict[schedd]=condorq
    return out_condorq_dict

#
# Return a dictionary of collectors containing classads of a certain kind
# Each element is a condorStatus
#
# If not all the jobs of the schedd has to be considered,
# specify the appropriate additional constraint
#
def getCondorStatusConstrained(collector_names,type_constraint,constraint=None,format_list=None):
    out_status_dict={}
    for collector in collector_names:
        status=condorMonitor.CondorStatus(pool_name=collector)
        full_constraint=type_constraint[0:] #make copy
        if constraint!=None:
            full_constraint="(%s) && (%s)"%(full_constraint,constraint)

        try:
            status.load(full_constraint,format_list)
        except condorExe.ExeError, e:
            if collector!=None:
                log_files.logWarning("Failed to talk to collector %s. See debug log for more details."%collector)
                log_files.logDebug("Failed to talk to collector %s: %s"%(collector, e))
            else:
                log_files.logWarning("Failed to talk to collector. See debug log for more details.")
                log_files.logDebug("Failed to talk to collector: %s"%e)
            continue # if collector not found it is equivalent to no classads
        if len(status.fetchStored())>0:
            out_status_dict[collector]=status
    return out_status_dict

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import logging
import sys

logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
                    level=logging.DEBUG,
                    stream=sys.stdout)
from copy import deepcopy
import pkg_resources
import os
import yaml
from decimal import Decimal, getcontext
getcontext().prec = 100

# hwrt modules
from .segmentation import single_clf
# from ..handwritten_data import HandwrittenData
# from .. import spacial_relationship
from .. import language_model
from ..utils import softmax

__all__ = [
    "Beam"
]

stroke_prob = None

def p_strokes(symbol, count):
    """
    Get the probability of a written `symbol` having `count` strokes.

    Parameters
    ----------
    symbol : str
        LaTeX command
    count : int, >= 1

    Returns
    -------
    float
        In [0.0, 1.0]
    """
    global stroke_prob
    assert count >= 1
    epsilon = 0.00000001
    if stroke_prob is None:
        misc_path = pkg_resources.resource_filename('hwrt', 'misc/')
        stroke_prob_file = os.path.join(misc_path,
                                        'prob_stroke_count_by_symbol.yml')
        with open(stroke_prob_file, 'r') as stream:
            stroke_prob = yaml.load(stream)
    if symbol in stroke_prob:
        if count in stroke_prob[symbol]:
            return stroke_prob[symbol][count]
        else:
            return epsilon
    return epsilon

def _calc_hypothesis_probability(hypothesis):
    """
    Get the probability (or rather a score) of a hypothesis.

    Parameters
    ----------
    hypothesis : dict
        with keys 'segmentation', 'symbols', ...

    Returns
    -------
    float
        in [0.0, 1.0]
    """
    prob = 0.0
    for symbol, seg in zip(hypothesis['symbols'], hypothesis['segmentation']):
        # symbol_latex = symbol['symbol'].split(";")[1]
        # TODO: Does p_strokes really improve the system?
        prob += symbol['probability']  # * p_strokes(symbol_latex, len(seg))

    # Use language model to update probabilities
    pure_symbols = [symbol['symbol'].split(";")[1]
                    for symbol in hypothesis['symbols']]
    pure_symbols = ["<s>"] + pure_symbols + ["</s>"]
    lm_prob = language_model.get_probability(pure_symbols)
    hypothesis['lm_probability'] = 2**lm_prob
    return (prob * float(hypothesis['lm_probability']) *
            (1.0 / len(hypothesis['segmentation'])))

class Beam(object):
    """

    Parameters
    ----------
    m : int, >= 1
        Number of symbols which get considered. This determines how many
        new segmentations come, where `new_stroke` belongs to a new symbol.
    n : int, >= 1
        Number of strokes to which `new_stroke` will be added.
        This determines how many new segmentations come, where `new_stroke`
        belongs to an old symbol.
    k : int, >= 1
        Maximum number of elements in the beam.

    Attributes
    ----------
    hypotheses : list of dicts
        [{'segmentation': [[0, 3], [1, 2]],
          'symbols': [{'symbol': ID, 'probability': 0.12}],
          'geometry': {'symbol': index,
                       'bottom': None or dict,
                       'subscript': None or dict,
                       'right': None or dict,
                       'superscript': None or dict,
                       'top': None or dict},
           'probability': 0.123
         }, ...]
    history: dict
        {'data': [[{'x': 12, 'y': 34, 'time': 56}, ...],
                  ...,
                  ],
         'id': -1}
         }
    """

    def __init__(self, m=10, n=1, k=20):
        self.m = m
        self.n = n
        self.k = k
        self.history = {'data': [], 'id': -1}
        self.hypotheses = []

    def _add_hypotheses_assuming_new_stroke(self,
                                            new_stroke,
                                            stroke_nr,
                                            new_beam):
        """
        Get new guesses by assuming new_stroke is a new symbol.

        Parameters
        ----------
        new_stroke : list of dicts
            A list of dicts [{'x': 12, 'y': 34, 'time': 56}, ...] which
            represent a point.
        stroke_nr : int
            Number of the stroke for segmentation
        new_beam : beam object
        """
        guesses = single_clf.predict({'data': [new_stroke],
                                      'id': None})[:self.m]
        for hyp in self.hypotheses:
            new_geometry = deepcopy(hyp['geometry'])
            most_right = new_geometry
            if len(hyp['symbols']) == 0:
                while 'right' in most_right:
                    most_right = most_right['right']
                most_right['right'] = {'symbol_index': len(hyp['symbols']),
                                       'right': None}
            else:
                most_right = {'symbol_index': len(hyp['symbols']),
                              'right': None}
            for guess in guesses:
                sym = {'symbol': guess['semantics'],
                       'probability': guess['probability']}
                new_seg = deepcopy(hyp['segmentation'])
                new_seg.append([stroke_nr])
                new_sym = deepcopy(hyp['symbols'])
                new_sym.append(sym)
                b = {'segmentation': new_seg,
                     'symbols': new_sym,
                     'geometry': new_geometry,
                     'probability': None
                     }

                # spacial_rels = []  # TODO
                # for s1_indices, s2_indices in zip(b['segmentation'],
                #                                   b['segmentation'][1:]):
                #     tmp = [new_beam.history['data'][el] for el in s1_indices]
                #     s1 = HandwrittenData(json.dumps(tmp))
                #     tmp = [new_beam.history['data'][el] for el in s2_indices]
                #     s2 = HandwrittenData(json.dumps(tmp))
                #     rel = spacial_relationship.estimate(s1, s2)
                #     spacial_rels.append(rel)
                # b['geometry'] = spacial_rels
                new_beam.hypotheses.append(b)

    def add_stroke(self, new_stroke):
        """
        Update the beam so that it considers `new_stroke`.

        When a `new_stroke` comes, it can either belong to a symbol for which
        at least one other stroke was already made or belong to a symbol for
        which `new_stroke` is the first stroke.

        The number of hypotheses after q strokes without pruning is

            f: N_0 -> N_0
            f(0) = 1
            f(1) = m
            f(q) = f(q-1)*(m+n)

        The number of time the single symbol classifier has to be called, when
        already q hypotheses exist:

            f_s: N_0 -> N_0
            f_s(q) = q*n + 1 (upper bound)

        Parameters
        ----------
        new_stroke : list of dicts
            A list of dicts [{'x': 12, 'y': 34, 'time': 56}, ...] which
            represent a point.
        """
        global single_clf
        if len(self.hypotheses) == 0:  # Don't put this in the constructor!
            self.hypotheses = [{'segmentation': [],
                                'symbols': [],
                                'geometry': {},
                                'probability': Decimal(1)
                                }]
        stroke_nr = len(self.history['data'])
        new_history = deepcopy(self.history)
        new_history['data'].append(new_stroke)
        new_beam = Beam()
        new_beam.history = new_history

        evaluated_segmentations = []

        # Get new guesses by assuming new_stroke belongs to an already begun
        # symbol
        had_multisymbol = False
        for hyp in self.hypotheses:
            # Add stroke to last n symbols (seperately)
            for i in range(min(self.n, len(hyp['segmentation']))):
                # Build stroke data
                new_strokes = {'data': [], 'id': -1}
                for stroke_index in hyp['segmentation'][-(i+1)]:
                    curr_stroke = self.history['data'][stroke_index]
                    new_strokes['data'].append(curr_stroke)
                new_strokes['data'].append(new_stroke)

                new_seg = deepcopy(hyp['segmentation'])
                new_seg[-(i+1)].append(stroke_nr)

                if new_seg in evaluated_segmentations:
                    continue
                else:
                    evaluated_segmentations.append(new_seg)

                # Predict this new collection of strokes
                guesses = single_clf.predict(new_strokes)[:self.m]
                for guess in guesses:
                    if guess['semantics'].split(";")[1] == "MULTISYMBOL":
                        # This was a wrong segmentation. Ignore it.
                        had_multisymbol = True
                        continue
                    sym = {'symbol': guess['semantics'],
                           'probability': guess['probability']}
                    new_sym = deepcopy(hyp['symbols'])
                    new_sym[-(i+1)] = sym
                    b = {'segmentation': new_seg,
                         'symbols': new_sym,
                         'geometry': deepcopy(hyp['geometry']),
                         'probability': None
                         }
                    new_beam.hypotheses.append(b)

        if len(self.hypotheses) <= 1 or had_multisymbol:
            self._add_hypotheses_assuming_new_stroke(new_stroke,
                                                     stroke_nr,
                                                     new_beam)

        for hyp in new_beam.hypotheses:
            hyp['probability'] = _calc_hypothesis_probability(hyp)

        # Get probability again

        # Get geometry of each beam entry
        # TODO

        # Update probabilities
        # TODO

        # Normalize to sum=1
        self.hypotheses = new_beam.hypotheses
        self.history = new_beam.history
        self._prune()
        new_probs = softmax([h['probability']
                             for h in self.hypotheses])
        for hyp, prob in zip(self.hypotheses, new_probs):
            hyp['probability'] = prob

    def _prune(self):
        """Shorten hypotheses to the best k ones."""
        self.hypotheses = sorted(self.hypotheses,
                                 key=lambda e: e['probability'],
                                 reverse=True)[:self.k]

    def get_results(self):
        results = []
        for hyp in self.hypotheses:
            results.append({'semantics': build_unicode(hyp),
                            'complete_latex': build_latex(hyp),
                            'probability': float(hyp['probability']),
                            'symbol count': len(hyp['segmentation'])})
        return results

    def get_writemath_results(self):
        """
        Get the result in the format
        [{'probability': 0.987,
          'segmentation': [[0, 1], [2, 4], [3]]}   // index of the stroke
          'symbols': [{'id': 456,  // on write-math.com
                       'probability': 0.123},
                      {'id': 456,
                       'probability': 0.999},  // The sum does not have to be 1
                      {'id': 195,
                       'probability': 0.0001}]
          },
         {...}  // another hypothesis
        ]
        """
        results = []
        for hyp in self.hypotheses:
            symbols = []
            for sym in hyp['symbols']:
                symbols.append({'id': sym['symbol'].split(';')[0],
                                'probability': sym['probability']})
            results.append({'probability': float(hyp['probability']),
                            'segmentation': hyp['segmentation'],
                            'symbols': symbols})
        return results

    def __str__(self):
        s = "Beam(n=%i, m=%i, k=%i)\n" % (self.n, self.m, self.k)
        for hyp in self.hypotheses:
            symbols = [sym['symbol'] for sym in hyp['symbols']]
            symbols = str([sym.split(';')[1] for sym in symbols])
            s += "\t%0.3f%%\t%s\n" % (hyp['probability']*100, symbols)
        return s

def build_unicode(hyp):
    """
    Parameters
    ----------
    hyp : dict
        {'segmentation': [[0, 3], [1, 2]],
         'symbols': [{'symbol': ID, 'probability': 0.12}],
         'geometry': {'symbol': index,
                      'bottom': None or dict,
                      'subscript': None or dict,
                      'right': None or dict,
                      'superscript': None or dict,
                      'top': None or dict},
          'probability': 0.123
        }
    """
    latex = []
    for symbol in hyp['symbols']:
        latex.append(symbol['symbol'])
    return ";;".join(latex)

def build_latex(hyp):
    """
    Parameters
    ----------
    hyp : dict
        {'segmentation': [[0, 3], [1, 2]],
         'symbols': [{'symbol': ID, 'probability': 0.12}],
         'geometry': {'symbol': index,
                      'bottom': None or dict,
                      'subscript': None or dict,
                      'right': None or dict,
                      'superscript': None or dict,
                      'top': None or dict},
          'probability': 0.123
        }
    """
    latex = []
    for symbol in hyp['symbols']:
        latex.append(symbol['symbol'].split(";")[1])
    return " ".join(latex)
<commit_msg>Fix multisymbol special case<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import logging
import sys

logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
                    level=logging.DEBUG,
                    stream=sys.stdout)
from copy import deepcopy
import pkg_resources
import os
import yaml
from decimal import Decimal, getcontext
getcontext().prec = 100

# hwrt modules
from .segmentation import single_clf
# from ..handwritten_data import HandwrittenData
# from .. import spacial_relationship
from .. import language_model
from ..utils import softmax

__all__ = [
    "Beam"
]

stroke_prob = None

def p_strokes(symbol, count):
    """
    Get the probability of a written `symbol` having `count` strokes.

    Parameters
    ----------
    symbol : str
        LaTeX command
    count : int, >= 1

    Returns
    -------
    float
        In [0.0, 1.0]
    """
    global stroke_prob
    assert count >= 1
    epsilon = 0.00000001
    if stroke_prob is None:
        misc_path = pkg_resources.resource_filename('hwrt', 'misc/')
        stroke_prob_file = os.path.join(misc_path,
                                        'prob_stroke_count_by_symbol.yml')
        with open(stroke_prob_file, 'r') as stream:
            stroke_prob = yaml.load(stream)
    if symbol in stroke_prob:
        if count in stroke_prob[symbol]:
            return stroke_prob[symbol][count]
        else:
            return epsilon
    return epsilon

def _calc_hypothesis_probability(hypothesis):
    """
    Get the probability (or rather a score) of a hypothesis.

    Parameters
    ----------
    hypothesis : dict
        with keys 'segmentation', 'symbols', ...

    Returns
    -------
    float
        in [0.0, 1.0]
    """
    prob = 0.0
    for symbol, seg in zip(hypothesis['symbols'], hypothesis['segmentation']):
        # symbol_latex = symbol['symbol'].split(";")[1]
        # TODO: Does p_strokes really improve the system?
        prob += symbol['probability']  # * p_strokes(symbol_latex, len(seg))

    # Use language model to update probabilities
    pure_symbols = [symbol['symbol'].split(";")[1]
                    for symbol in hypothesis['symbols']]
    pure_symbols = ["<s>"] + pure_symbols + ["</s>"]
    lm_prob = language_model.get_probability(pure_symbols)
    hypothesis['lm_probability'] = 2**lm_prob
    return (prob * float(hypothesis['lm_probability']) *
            (1.0 / len(hypothesis['segmentation'])))

class Beam(object):
    """

    Parameters
    ----------
    m : int, >= 1
        Number of symbols which get considered. This determines how many
        new segmentations come, where `new_stroke` belongs to a new symbol.
    n : int, >= 1
        Number of strokes to which `new_stroke` will be added.
        This determines how many new segmentations come, where `new_stroke`
        belongs to an old symbol.
    k : int, >= 1
        Maximum number of elements in the beam.

    Attributes
    ----------
    hypotheses : list of dicts
        [{'segmentation': [[0, 3], [1, 2]],
          'symbols': [{'symbol': ID, 'probability': 0.12}],
          'geometry': {'symbol': index,
                       'bottom': None or dict,
                       'subscript': None or dict,
                       'right': None or dict,
                       'superscript': None or dict,
                       'top': None or dict},
           'probability': 0.123
         }, ...]
    history: dict
        {'data': [[{'x': 12, 'y': 34, 'time': 56}, ...],
                  ...,
                  ],
         'id': -1}
         }
    """

    def __init__(self, m=10, n=1, k=20):
        self.m = m
        self.n = n
        self.k = k
        self.history = {'data': [], 'id': -1}
        self.hypotheses = []

    def _add_hypotheses_assuming_new_stroke(self,
                                            new_stroke,
                                            stroke_nr,
                                            new_beam):
        """
        Get new guesses by assuming new_stroke is a new symbol.

        Parameters
        ----------
        new_stroke : list of dicts
            A list of dicts [{'x': 12, 'y': 34, 'time': 56}, ...] which
            represent a point.
        stroke_nr : int
            Number of the stroke for segmentation
        new_beam : beam object
        """
        guesses = single_clf.predict({'data': [new_stroke],
                                      'id': None})[:self.m]
        for hyp in self.hypotheses:
            new_geometry = deepcopy(hyp['geometry'])
            most_right = new_geometry
            if len(hyp['symbols']) == 0:
                while 'right' in most_right:
                    most_right = most_right['right']
                most_right['right'] = {'symbol_index': len(hyp['symbols']),
                                       'right': None}
            else:
                most_right = {'symbol_index': len(hyp['symbols']),
                              'right': None}
            for guess in guesses:
                sym = {'symbol': guess['semantics'],
                       'probability': guess['probability']}
                new_seg = deepcopy(hyp['segmentation'])
                new_seg.append([stroke_nr])
                new_sym = deepcopy(hyp['symbols'])
                new_sym.append(sym)
                b = {'segmentation': new_seg,
                     'symbols': new_sym,
                     'geometry': new_geometry,
                     'probability': None
                     }

                # spacial_rels = []  # TODO
                # for s1_indices, s2_indices in zip(b['segmentation'],
                #                                   b['segmentation'][1:]):
                #     tmp = [new_beam.history['data'][el] for el in s1_indices]
                #     s1 = HandwrittenData(json.dumps(tmp))
                #     tmp = [new_beam.history['data'][el] for el in s2_indices]
                #     s2 = HandwrittenData(json.dumps(tmp))
                #     rel = spacial_relationship.estimate(s1, s2)
                #     spacial_rels.append(rel)
                # b['geometry'] = spacial_rels
                new_beam.hypotheses.append(b)

    def add_stroke(self, new_stroke):
        """
        Update the beam so that it considers `new_stroke`.

        When a `new_stroke` comes, it can either belong to a symbol for which
        at least one other stroke was already made or belong to a symbol for
        which `new_stroke` is the first stroke.

        The number of hypotheses after q strokes without pruning is

            f: N_0 -> N_0
            f(0) = 1
            f(1) = m
            f(q) = f(q-1)*(m+n)

        The number of time the single symbol classifier has to be called, when
        already q hypotheses exist:

            f_s: N_0 -> N_0
            f_s(q) = q*n + 1 (upper bound)

        Parameters
        ----------
        new_stroke : list of dicts
            A list of dicts [{'x': 12, 'y': 34, 'time': 56}, ...] which
            represent a point.
        """
        global single_clf
        if len(self.hypotheses) == 0:  # Don't put this in the constructor!
            self.hypotheses = [{'segmentation': [],
                                'symbols': [],
                                'geometry': {},
                                'probability': Decimal(1)
                                }]
        stroke_nr = len(self.history['data'])
        new_history = deepcopy(self.history)
        new_history['data'].append(new_stroke)
        new_beam = Beam()
        new_beam.history = new_history

        evaluated_segmentations = []

        # Get new guesses by assuming new_stroke belongs to an already begun
        # symbol
        had_multisymbol = False
        for hyp in self.hypotheses:
            # Add stroke to last n symbols (seperately)
            for i in range(min(self.n, len(hyp['segmentation']))):
                # Build stroke data
                new_strokes = {'data': [], 'id': -1}
                for stroke_index in hyp['segmentation'][-(i+1)]:
                    curr_stroke = self.history['data'][stroke_index]
                    new_strokes['data'].append(curr_stroke)
                new_strokes['data'].append(new_stroke)

                new_seg = deepcopy(hyp['segmentation'])
                new_seg[-(i+1)].append(stroke_nr)

                if new_seg in evaluated_segmentations:
                    continue
                else:
                    evaluated_segmentations.append(new_seg)

                # Predict this new collection of strokes
                guesses = single_clf.predict(new_strokes)[:self.m]
                for guess in guesses:
                    if guess['semantics'].split(";")[1] == "::MULTISYMBOL::":
                        # This was a wrong segmentation. Ignore it.
                        had_multisymbol = True
                        continue
                    sym = {'symbol': guess['semantics'],
                           'probability': guess['probability']}
                    new_sym = deepcopy(hyp['symbols'])
                    new_sym[-(i+1)] = sym
                    b = {'segmentation': new_seg,
                         'symbols': new_sym,
                         'geometry': deepcopy(hyp['geometry']),
                         'probability': None
                         }
                    new_beam.hypotheses.append(b)

        if len(self.hypotheses) <= 1 or had_multisymbol:
            self._add_hypotheses_assuming_new_stroke(new_stroke,
                                                     stroke_nr,
                                                     new_beam)

        for hyp in new_beam.hypotheses:
            hyp['probability'] = _calc_hypothesis_probability(hyp)

        # Get probability again

        # Get geometry of each beam entry
        # TODO

        # Update probabilities
        # TODO

        # Normalize to sum=1
        self.hypotheses = new_beam.hypotheses
        self.history = new_beam.history
        self._prune()
        new_probs = softmax([h['probability']
                             for h in self.hypotheses])
        for hyp, prob in zip(self.hypotheses, new_probs):
            hyp['probability'] = prob

    def _prune(self):
        """Shorten hypotheses to the best k ones."""
        self.hypotheses = sorted(self.hypotheses,
                                 key=lambda e: e['probability'],
                                 reverse=True)[:self.k]

    def get_results(self):
        results = []
        for hyp in self.hypotheses:
            results.append({'semantics': build_unicode(hyp),
                            'complete_latex': build_latex(hyp),
                            'probability': float(hyp['probability']),
                            'symbol count': len(hyp['segmentation'])})
        return results

    def get_writemath_results(self):
        """
        Get the result in the format
        [{'probability': 0.987,
          'segmentation': [[0, 1], [2, 4], [3]]}   // index of the stroke
          'symbols': [{'id': 456,  // on write-math.com
                       'probability': 0.123},
                      {'id': 456,
                       'probability': 0.999},  // The sum does not have to be 1
                      {'id': 195,
                       'probability': 0.0001}]
          },
         {...}  // another hypothesis
        ]
        """
        results = []
        for hyp in self.hypotheses:
            symbols = []
            for sym in hyp['symbols']:
                symbols.append({'id': sym['symbol'].split(';')[0],
                                'probability': sym['probability']})
            results.append({'probability': float(hyp['probability']),
                            'segmentation': hyp['segmentation'],
                            'symbols': symbols})
        return results

    def __str__(self):
        s = "Beam(n=%i, m=%i, k=%i)\n" % (self.n, self.m, self.k)
        for hyp in self.hypotheses:
            symbols = [sym['symbol'] for sym in hyp['symbols']]
            symbols = str([sym.split(';')[1] for sym in symbols])
            s += "\t%0.3f%%\t%s\n" % (hyp['probability']*100, symbols)
        return s

def build_unicode(hyp):
    """
    Parameters
    ----------
    hyp : dict
        {'segmentation': [[0, 3], [1, 2]],
         'symbols': [{'symbol': ID, 'probability': 0.12}],
         'geometry': {'symbol': index,
                      'bottom': None or dict,
                      'subscript': None or dict,
                      'right': None or dict,
                      'superscript': None or dict,
                      'top': None or dict},
          'probability': 0.123
        }
    """
    latex = []
    for symbol in hyp['symbols']:
        latex.append(symbol['symbol'])
    return ";;".join(latex)

def build_latex(hyp):
    """
    Parameters
    ----------
    hyp : dict
        {'segmentation': [[0, 3], [1, 2]],
         'symbols': [{'symbol': ID, 'probability': 0.12}],
         'geometry': {'symbol': index,
                      'bottom': None or dict,
                      'subscript': None or dict,
                      'right': None or dict,
                      'superscript': None or dict,
                      'top': None or dict},
          'probability': 0.123
        }
    """
    latex = []
    for symbol in hyp['symbols']:
        latex.append(symbol['symbol'].split(";")[1])
    return " ".join(latex)
<|endoftext|>


<commit_before>from ontobio.sim.api.interfaces import SimApi
from typing import Iterable, Collection, Union, Optional, List
from ontobio.model.similarity import SimResult, TypedNode
from ontobio.vocabulary.similarity import SimAlgorithm
from ontobio.sim.api.interfaces import FilteredSearchable
from ontobio.golr.golr_associations import get_objects_for_subject
from ontobio.util.scigraph_util import get_id_type_map, typed_node_from_id

class PhenoSimEngine():
    """
    Engine for computing phenotype similarity

    A wrapper for a similarity API whereby non-phenotype identifiers
    are resolved to a list of phenotypes
    """

    def __init__(self, sim_api: SimApi):
        self.sim_api = sim_api

    def search(
            self,
            id_list: Iterable[str],
            negated_ids: Optional[Collection] = None,
            limit: Optional[int] = 100,
            taxon_filter: Optional[str]= None,
            category_filter: Optional[str]= None,
            method: Optional[SimAlgorithm] = SimAlgorithm.PHENODIGM,
            is_feature_set: bool = True
    ) -> SimResult:
        """
        Execute a search using sim_api,

        :param taxon_filter: only
        :param category_filter:
        :param method: similarity metric
        :param is_feature_set: if True, treats all identifiers as phenotypic
                               features, else resolves ids to a list of phenotypes
                               (eg set to false if some identifiers are
                                diseases, genes, etc)

        :raises NotImplementedError:
            - If sim method or filters are not supported
        """

        if negated_ids is None: negated_ids = []

        if method not in self.sim_api.matchers():
            raise NotImplementedError("Sim method not implemented "
                                      "in {}".format(str(self.sim_api)))

        if not is_feature_set:
            # Determine if entity is a phenotype or individual containing
            # a pheno profile (gene, disease, case, etc)
            pheno_list = PhenoSimEngine._resolve_nodes_to_phenotypes(id_list)
        else:
            pheno_list = id_list

        if taxon_filter is not None or category_filter is not None:
            if not isinstance(self.sim_api, FilteredSearchable):
                raise NotImplementedError("filtered search not implemented "
                                          "in {}".format(str(self.sim_api)))
            search_result = self.sim_api.filtered_search(
                pheno_list, negated_ids, limit, taxon_filter, category_filter, method
            )
        else:
            search_result = self.sim_api.search(pheno_list, negated_ids, limit, method)

        return search_result

    def compare(self,
                reference_ids: Collection,
                query_profiles: Collection[Collection],
                method: Optional[SimAlgorithm] = SimAlgorithm.PHENODIGM,
                is_feature_set: bool = True) -> SimResult:
        """
        Execute one or more comparisons using sim_api
        :param reference_ids: a list of phenotypes or ids that comprise
                              one or more phenotypes
        :param query_profiles: a list of lists of phenotypes or ids
                                   that comprise one or more phenotypes
        :param method: similarity metric
        :param is_feature_set: if True, treats all identifiers as phenotypic
                               features, else resolves ids to a list of phenotypes
                               (eg set to false if some identifiers are
                                diseases, genes, etc)
        :return: SimResult object
        :raises NotImplementedError: If sim method or filters are not supported
        """
        if method not in self.sim_api.matchers():
            raise NotImplementedError("Sim method not implemented "
                                      "in {}".format(str(self.sim_api)))

        is_first_result = True
        comparisons = None

        if is_feature_set:
            reference_phenos = reference_ids
        else:
            reference_phenos = PhenoSimEngine._resolve_nodes_to_phenotypes(reference_ids)
        for query_profile in query_profiles:
            if is_feature_set:
                query_phenos = query_profile
            else:
                query_phenos = PhenoSimEngine._resolve_nodes_to_phenotypes(query_profile)

            sim_result = self.sim_api.compare(reference_phenos, query_phenos, method)
            if len(query_profile) > 1:
                id = " + ".join(query_profile)
                sim_result.matches[0].id = id
                sim_result.matches[0].label = id
            else:
                node = typed_node_from_id(query_profile[0])
                if sim_result.matches:
                    sim_result.matches[0].id = node.id
                    sim_result.matches[0].label = node.label
                    sim_result.matches[0].type = node.type
                    sim_result.matches[0].taxon = node.taxon

            if is_first_result:
                comparisons = sim_result
                is_first_result = False
            else:
                comparisons.matches.append(sim_result.matches[0])
                comparisons.query.target_ids.append(sim_result.query.target_ids[0])

        if len(reference_ids) == 1:
            comparisons.query.reference = typed_node_from_id(reference_ids[0])
        else:
            reference_id = " + ".join(reference_ids)
            comparisons.query.reference = TypedNode(
                id=reference_id,
                label=reference_id,
                type='unknown'
            )

        return comparisons

    @staticmethod
    def _resolve_nodes_to_phenotypes(id_list: Iterable[str]) -> Iterable[str]:
        """
        Given a list of ids of unknown type, determine which ids
        are phenotypes, if the id is not a phenotype, check to
        see if it is associated with one or more phenotypes via
        the 'has_phenotype' relation
        :param id_list: list of ids of any type (curies as strings)
        :return: list of phenotypes (curies as strings)
        """
        pheno_list = []
        node_types = get_id_type_map(id_list)

        for node in id_list:
            if 'phenotype' in node_types[node]:
                pheno_list.append(node)
            else:
                phenotypes = get_objects_for_subject(
                    subject=node, object_category='phenotype', relation='RO:0002200'
                )
                pheno_list = pheno_list + phenotypes
        return pheno_list

    @staticmethod
    def _get_node_info(id_list: Iterable[str]) -> Iterable[str]:
        """
        Given a list of ids of unknown type, determine which ids
        are phenotypes, if the id is not a phenotype, check to
        see if it is associated with one or more phenotypes via
        the 'has_phenotype' relation
        :param id_list: list of ids of any type (curies as strings)
        :return: list of phenotypes (curies as strings)
        """
        pheno_list = []
        node_types = get_id_type_map(id_list)

        for node in id_list:
            if 'phenotype' in node_types[node]:
                pheno_list.append(node)
            else:
                phenotypes = get_objects_for_subject(
                    subject=node, object_category='phenotype', relation='RO:0002200'
                )
                pheno_list = pheno_list + phenotypes
        return pheno_list
<commit_msg>only get direct phenotypes for input nodes (diseases, genes)<commit_after>from ontobio.sim.api.interfaces import SimApi
from typing import Iterable, Collection, Union, Optional, List
from ontobio.model.similarity import SimResult, TypedNode
from ontobio.vocabulary.similarity import SimAlgorithm
from ontobio.sim.api.interfaces import FilteredSearchable
from ontobio.golr.golr_associations import get_objects_for_subject
from ontobio.util.scigraph_util import get_id_type_map, typed_node_from_id

class PhenoSimEngine():
    """
    Engine for computing phenotype similarity

    A wrapper for a similarity API whereby non-phenotype identifiers
    are resolved to a list of phenotypes
    """

    def __init__(self, sim_api: SimApi):
        self.sim_api = sim_api

    def search(
            self,
            id_list: Iterable[str],
            negated_ids: Optional[Collection] = None,
            limit: Optional[int] = 100,
            taxon_filter: Optional[str]= None,
            category_filter: Optional[str]= None,
            method: Optional[SimAlgorithm] = SimAlgorithm.PHENODIGM,
            is_feature_set: bool = True
    ) -> SimResult:
        """
        Execute a search using sim_api,

        :param taxon_filter: only
        :param category_filter:
        :param method: similarity metric
        :param is_feature_set: if True, treats all identifiers as phenotypic
                               features, else resolves ids to a list of phenotypes
                               (eg set to false if some identifiers are
                                diseases, genes, etc)

        :raises NotImplementedError:
            - If sim method or filters are not supported
        """

        if negated_ids is None: negated_ids = []

        if method not in self.sim_api.matchers():
            raise NotImplementedError("Sim method not implemented "
                                      "in {}".format(str(self.sim_api)))

        if not is_feature_set:
            # Determine if entity is a phenotype or individual containing
            # a pheno profile (gene, disease, case, etc)
            pheno_list = PhenoSimEngine._resolve_nodes_to_phenotypes(id_list)
        else:
            pheno_list = id_list

        if taxon_filter is not None or category_filter is not None:
            if not isinstance(self.sim_api, FilteredSearchable):
                raise NotImplementedError("filtered search not implemented "
                                          "in {}".format(str(self.sim_api)))
            search_result = self.sim_api.filtered_search(
                pheno_list, negated_ids, limit, taxon_filter, category_filter, method
            )
        else:
            search_result = self.sim_api.search(pheno_list, negated_ids, limit, method)

        return search_result

    def compare(self,
                reference_ids: Collection,
                query_profiles: Collection[Collection],
                method: Optional[SimAlgorithm] = SimAlgorithm.PHENODIGM,
                is_feature_set: bool = True) -> SimResult:
        """
        Execute one or more comparisons using sim_api
        :param reference_ids: a list of phenotypes or ids that comprise
                              one or more phenotypes
        :param query_profiles: a list of lists of phenotypes or ids
                                   that comprise one or more phenotypes
        :param method: similarity metric
        :param is_feature_set: if True, treats all identifiers as phenotypic
                               features, else resolves ids to a list of phenotypes
                               (eg set to false if some identifiers are
                                diseases, genes, etc)
        :return: SimResult object
        :raises NotImplementedError: If sim method or filters are not supported
        """
        if method not in self.sim_api.matchers():
            raise NotImplementedError("Sim method not implemented "
                                      "in {}".format(str(self.sim_api)))

        is_first_result = True
        comparisons = None

        if is_feature_set:
            reference_phenos = reference_ids
        else:
            reference_phenos = PhenoSimEngine._resolve_nodes_to_phenotypes(reference_ids)
        for query_profile in query_profiles:
            if is_feature_set:
                query_phenos = query_profile
            else:
                query_phenos = PhenoSimEngine._resolve_nodes_to_phenotypes(query_profile)

            sim_result = self.sim_api.compare(reference_phenos, query_phenos, method)
            if len(query_profile) > 1:
                id = " + ".join(query_profile)
                sim_result.matches[0].id = id
                sim_result.matches[0].label = id
            else:
                node = typed_node_from_id(query_profile[0])
                if sim_result.matches:
                    sim_result.matches[0].id = node.id
                    sim_result.matches[0].label = node.label
                    sim_result.matches[0].type = node.type
                    sim_result.matches[0].taxon = node.taxon

            if is_first_result:
                comparisons = sim_result
                is_first_result = False
            else:
                comparisons.matches.append(sim_result.matches[0])
                comparisons.query.target_ids.append(sim_result.query.target_ids[0])

        if len(reference_ids) == 1:
            comparisons.query.reference = typed_node_from_id(reference_ids[0])
        else:
            reference_id = " + ".join(reference_ids)
            comparisons.query.reference = TypedNode(
                id=reference_id,
                label=reference_id,
                type='unknown'
            )

        return comparisons

    @staticmethod
    def _resolve_nodes_to_phenotypes(id_list: Iterable[str]) -> Iterable[str]:
        """
        Given a list of ids of unknown type, determine which ids
        are phenotypes, if the id is not a phenotype, check to
        see if it is associated with one or more phenotypes via
        the 'has_phenotype' relation
        :param id_list: list of ids of any type (curies as strings)
        :return: list of phenotypes (curies as strings)
        """
        pheno_list = []
        node_types = get_id_type_map(id_list)

        for node in id_list:
            if 'phenotype' in node_types[node]:
                pheno_list.append(node)
            else:
                phenotypes = get_objects_for_subject(
                    subject=node, object_category='phenotype', relation='RO:0002200', subject_direct=True
                )
                pheno_list = pheno_list + phenotypes
        return pheno_list

    @staticmethod
    def _get_node_info(id_list: Iterable[str]) -> Iterable[str]:
        """
        Given a list of ids of unknown type, determine which ids
        are phenotypes, if the id is not a phenotype, check to
        see if it is associated with one or more phenotypes via
        the 'has_phenotype' relation
        :param id_list: list of ids of any type (curies as strings)
        :return: list of phenotypes (curies as strings)
        """
        pheno_list = []
        node_types = get_id_type_map(id_list)

        for node in id_list:
            if 'phenotype' in node_types[node]:
                pheno_list.append(node)
            else:
                phenotypes = get_objects_for_subject(
                    subject=node, object_category='phenotype', relation='RO:0002200', subject_direct=True
                )
                pheno_list = pheno_list + phenotypes
        return pheno_list
<|endoftext|>


<commit_before>"""

uritemplate.template
====================

This module contains the essential inner workings of uritemplate.

What treasures await you:

- URITemplate class

You see a treasure chest of knowledge in front of you.
What do you do?
>

"""

import re
from uritemplate.variable import URIVariable

template_re = re.compile('{([^\}]+)}')

def _merge(var_dict, overrides):
    if var_dict:
        for k, v in var_dict.items():
            if k in overrides:
                continue
            overrides[k] = v
    return overrides

class URITemplate(object):

    """This parses the template and will be used to expand it.

    This is the most important object as the center of the API.

    Example::

        from uritemplate import URITemplate
        import requests

        t = URITemplate(
            'https://api.github.com/users/sigmavirus24/gists{/gist_id}'
        )
        uri = t.expand(gist_id=123456)
        resp = requests.get(uri)
        for gist in resp.json():
            print(gist['html_url'])

    Please note::

        str(t)
        # 'https://api.github.com/users/sigmavirus24/gists{/gistid}'
        repr(t)  # is equivalent to
        # URITemplate(str(t))
        # Where str(t) is interpreted as the URI string.

    Also, ``URITemplates`` are hashable so they can be used as keys in
    dictionaries.

    """

    def __init__(self, uri):
        #: The original URI to be parsed.
        self.uri = uri
        #: A list of the variables in the URI. They are stored as
        #: :class:`URIVariable`\ s
        self.variables = [
            URIVariable(m.groups()[0]) for m in template_re.finditer(self.uri)
        ]
        #: A set of variable names in the URI.
        self.variable_names = set()
        for variable in self.variables:
            self.variable_names.update(variable.variable_names)

    def __repr__(self):
        return 'URITemplate("%s")' % self

    def __str__(self):
        return self.uri

    def __eq__(self, other):
        return self.uri == other.uri

    def __hash__(self):
        return hash(self.uri)

    def _expand(self, var_dict, replace):
        if not self.variables:
            return self.uri

        expansion = var_dict
        expanded = {}
        for v in self.variables:
            expanded.update(v.expand(expansion))

        def replace_all(match):
            return expanded.get(match.groups()[0], '')

        def replace_partial(match):
            match = match.groups()[0]
            var = '{%s}' % match
            return expanded.get(match) or var

        replace = replace_partial if replace else replace_all

        return template_re.sub(replace, self.uri)

    def expand(self, var_dict=None, **kwargs):
        """Expand the template with the given parameters.

        :param dict var_dict: Optional dictionary with variables and values
        :param kwargs: Alternative way to pass arguments
        :returns: str

        Example::

            t = URITemplate('https://api.github.com{/end}')
            t.expand({'end': 'users'})
            t.expand(end='gists')

        .. note:: Passing values by both parts, may override values in
                  ``var_dict``. For example::

                      expand('https://{var}', {'var': 'val1'}, var='val2')

                  ``val2`` will be used instead of ``val1``.

        """
        return self._expand(_merge(var_dict, kwargs), False)

    def partial(self, var_dict=None, **kwargs):
        """Partially expand the template with the given parameters.

        If all of the parameters for the template are not given, return a
        partially expanded template.

        :param dict var_dict: Optional dictionary with variables and values
        :param kwargs: Alternative way to pass arguments
        :returns: :class:`URITemplate`

        Example::

            t = URITemplate('https://api.github.com{/end}')
            t.partial()  # => URITemplate('https://api.github.com{/end}')

        """
        return URITemplate(self._expand(_merge(var_dict, kwargs), True))
<commit_msg>avoid premature optimisations<commit_after>"""

uritemplate.template
====================

This module contains the essential inner workings of uritemplate.

What treasures await you:

- URITemplate class

You see a treasure chest of knowledge in front of you.
What do you do?
>

"""

import re
from uritemplate.variable import URIVariable

template_re = re.compile('{([^\}]+)}')

def _merge(var_dict, overrides):
    if var_dict:
        opts = var_dict.copy()
        opts.update(overrides)
        return opts
    return overrides

class URITemplate(object):

    """This parses the template and will be used to expand it.

    This is the most important object as the center of the API.

    Example::

        from uritemplate import URITemplate
        import requests

        t = URITemplate(
            'https://api.github.com/users/sigmavirus24/gists{/gist_id}'
        )
        uri = t.expand(gist_id=123456)
        resp = requests.get(uri)
        for gist in resp.json():
            print(gist['html_url'])

    Please note::

        str(t)
        # 'https://api.github.com/users/sigmavirus24/gists{/gistid}'
        repr(t)  # is equivalent to
        # URITemplate(str(t))
        # Where str(t) is interpreted as the URI string.

    Also, ``URITemplates`` are hashable so they can be used as keys in
    dictionaries.

    """

    def __init__(self, uri):
        #: The original URI to be parsed.
        self.uri = uri
        #: A list of the variables in the URI. They are stored as
        #: :class:`URIVariable`\ s
        self.variables = [
            URIVariable(m.groups()[0]) for m in template_re.finditer(self.uri)
        ]
        #: A set of variable names in the URI.
        self.variable_names = set()
        for variable in self.variables:
            self.variable_names.update(variable.variable_names)

    def __repr__(self):
        return 'URITemplate("%s")' % self

    def __str__(self):
        return self.uri

    def __eq__(self, other):
        return self.uri == other.uri

    def __hash__(self):
        return hash(self.uri)

    def _expand(self, var_dict, replace):
        if not self.variables:
            return self.uri

        expansion = var_dict
        expanded = {}
        for v in self.variables:
            expanded.update(v.expand(expansion))

        def replace_all(match):
            return expanded.get(match.groups()[0], '')

        def replace_partial(match):
            match = match.groups()[0]
            var = '{%s}' % match
            return expanded.get(match) or var

        replace = replace_partial if replace else replace_all

        return template_re.sub(replace, self.uri)

    def expand(self, var_dict=None, **kwargs):
        """Expand the template with the given parameters.

        :param dict var_dict: Optional dictionary with variables and values
        :param kwargs: Alternative way to pass arguments
        :returns: str

        Example::

            t = URITemplate('https://api.github.com{/end}')
            t.expand({'end': 'users'})
            t.expand(end='gists')

        .. note:: Passing values by both parts, may override values in
                  ``var_dict``. For example::

                      expand('https://{var}', {'var': 'val1'}, var='val2')

                  ``val2`` will be used instead of ``val1``.

        """
        return self._expand(_merge(var_dict, kwargs), False)

    def partial(self, var_dict=None, **kwargs):
        """Partially expand the template with the given parameters.

        If all of the parameters for the template are not given, return a
        partially expanded template.

        :param dict var_dict: Optional dictionary with variables and values
        :param kwargs: Alternative way to pass arguments
        :returns: :class:`URITemplate`

        Example::

            t = URITemplate('https://api.github.com{/end}')
            t.partial()  # => URITemplate('https://api.github.com{/end}')

        """
        return URITemplate(self._expand(_merge(var_dict, kwargs), True))
<|endoftext|>


<commit_before>import unittest

from hdl_toolkit.hdlObjects.typeShortcuts import hInt, hBool, hBit, vec, vecT
from hdl_toolkit.hdlObjects.types.defs import INT, STR
from hdl_toolkit.synthesizer.rtlLevel.netlist import RtlNetlist
from hdl_toolkit.synthesizer.rtlLevel.signalUtils.walkers import walkAllOriginSignals
from hdl_toolkit.bitmask import Bitmask

andTable = [ (None, None, None),
             (None, 0, 0),
             (None, 1, None),
             (0, None, 0),
             (0, 0, 0),
             (0, 1, 0),
             (1, 1, 1)]
orTable = [ (None, None, None),
            (None, 0, None),
            (None, 1, 1),
            (0, None, None),
            (0, 0, 0),
            (0, 1, 1),
            (1, 1, 1)]

class OperatorTC(unittest.TestCase):
    def setUp(self):
        unittest.TestCase.setUp(self)
        self.n = RtlNetlist()

    def testNoBool(self):
        for v in [True, False]:
            res = ~hBool(v)
            self.assertEqual(res.val, not v)
            self.assertEqual(res.vldMask, 1)
            self.assertEqual(res.updateTime, -1)

    def testNotBit(self):
        for v in [False, True]:
            res = ~hBit(v)

            self.assertEqual(res.val, int(not v))
            self.assertEqual(res.vldMask, 1)
            self.assertEqual(res.updateTime, -1)

    def testBitAnd(self):
        vals = {
            1: hBit(1),
            0: hBit(0),
            None: hBit(None)
        }

        for a, b, expected in andTable:
            res = vals[a] & vals[b]
            expectedRes = vals[expected]

            self.assertEqual(expectedRes.val, res.val,
                             "%s & %s  val=%s (should be %s)" % (repr(a), repr(b), repr(res.val), repr(expectedRes.val)))
            self.assertEqual(expectedRes.vldMask, res.vldMask,
                             "%s & %s  vldMask=%s (should be %s)" % (repr(a), repr(b), repr(res.vldMask), repr(expectedRes.vldMask)))
    def testBitOr(self):
        vals = {
            1: hBit(1),
            0: hBit(0),
            None: hBit(None)
        }

        for a, b, expected in orTable:
            res = vals[a] | vals[b]
            expectedRes = vals[expected]

            self.assertEqual(expectedRes.val, res.val,
                             "%s & %s  val=%s (should be %s)" % (repr(a), repr(b), repr(res.val), repr(expectedRes.val)))
            self.assertEqual(expectedRes.vldMask, res.vldMask,
                             "%s & %s  vldMask=%s (should be %s)" % (repr(a), repr(b), repr(res.vldMask), repr(expectedRes.vldMask)))

    def testBoolAnd(self):
        vals = {
            1: hBool(1),
            0: hBool(0),
            None: hBool(None)
        }

        for a, b, expected in andTable:
            res = vals[a] & vals[b]
            expectedRes = vals[expected]

            self.assertEqual(expectedRes.val, res.val,
                             "%s & %s  val=%s (should be %s)" % (repr(a), repr(b), repr(res.val), repr(expectedRes.val)))
            self.assertEqual(expectedRes.vldMask, res.vldMask,
                             "%s & %s  vldMask=%s (should be %s)" % (repr(a), repr(b), repr(res.vldMask), repr(expectedRes.vldMask)))
    def testBoolOr(self):
        vals = {
            1: hBool(1),
            0: hBool(0),
            None: hBool(None)
        }

        for a, b, expected in orTable:
            res = vals[a] | vals[b]
            expectedRes = vals[expected]

            self.assertEqual(expectedRes.val, res.val,
                             "%s & %s  val=%s (should be %s)" % (repr(a), repr(b), repr(res.val), repr(expectedRes.val)))
            self.assertEqual(expectedRes.vldMask, res.vldMask,
                             "%s & %s  vldMask=%s (should be %s)" % (repr(a), repr(b), repr(res.vldMask), repr(expectedRes.vldMask)))

    def testNotNotisOrigSig(self):
        a = self.n.sig("a")
        self.assertIs(a, ~ ~a)

    def testDownto(self):
        a = self.n.sig('a', typ=INT)
        a.defaultVal = hInt(10)
        b = hInt(0)
        r = a._downto(b)
        res = r.staticEval()
        self.assertEqual(res.val[0].val, 10)
        self.assertEqual(res.val[1].val, 0)

    def testwalkAllOriginSignalsDownto(self):
        a = self.n.sig('a', typ=INT)
        a.defaultVal = hInt(10)
        b = hInt(0)
        r = a._downto(b)
        origins = set(walkAllOriginSignals(r))
        self.assertSetEqual(origins, set([a]))

    def testwalkAllOriginSignalsDowntoAndPlus(self):
        a = self.n.sig('a', typ=INT)
        a.defaultVal = hInt(10)
        b = hInt(0)
        am = a + hInt(5)
        r = am._downto(b)
        origins = set(walkAllOriginSignals(r))
        self.assertSetEqual(origins, set([a]))

    def testADD_InvalidOperands(self):
        a = self.n.sig('a', typ=STR)
        b = self.n.sig('b')
        self.assertRaises(NotImplementedError, lambda : a + b)

    def testADD_IntBits(self):
        a = vec(7, 8)
        b = hInt(1)
        c = a + b
        self.assertEqual(c.val, 8)
        self.assertEqual(c.vldMask, Bitmask.mask(8))

        a = vec(255, 8)
        b = hInt(1)
        c = a + b
        self.assertEqual(c.val, 0)
        self.assertEqual(c.vldMask, Bitmask.mask(8))

        a = vec(7, 8, False)
        b = hInt(1)
        c = a + b
        self.assertEqual(c.val, 8)
        self.assertEqual(c.vldMask, Bitmask.mask(8))

        a = vec(255, 8, False)
        b = hInt(1)
        c = a + b
        self.assertEqual(c.val, 0)
        self.assertEqual(c.vldMask, Bitmask.mask(8))

    def testAND_eval(self):
        for a_in, b_in, out in [(0, 0, 0),
                                (0, 1, 0),
                                (1, 0, 0),
                                (1, 1, 1)]:
            res = hBit(a_in) & hBit(b_in)
            self.assertEqual(res.vldMask, 1)
            self.assertEqual(res.val, out, "a_in %d, b_in %d, out %d" % (a_in, b_in, out))

    def testADD_eval(self):
        for a_in, b_in, out in [(0, 0, 0),
                                (0, 1, 1),
                                (1, 0, 1),
                                (1, 1, 2)]:
            res = hInt(a_in) + hInt(b_in)

            b_w = 2

            self.assertTrue(res.vldMask)
            self.assertEqual(res.val, out, "a_in %d, b_in %d, out %d" % (a_in, b_in, out))

            resBit = vec(a_in, b_w) + vec(b_in, b_w)
            self.assertEqual(resBit.vldMask, 3)
            self.assertEqual(resBit.val, out, "a_in %d, b_in %d, out %d" % (a_in, b_in, out))

if __name__ == '__main__':
    suite = unittest.TestSuite()
    # suite.addTest(OperatorTC('testBitAnd'))
    suite.addTest(unittest.makeSuite(OperatorTC))
    runner = unittest.TextTestRunner(verbosity=3)
    runner.run(suite)
<commit_msg>comments<commit_after>import unittest

from hdl_toolkit.hdlObjects.typeShortcuts import hInt, hBool, hBit, vec, vecT
from hdl_toolkit.hdlObjects.types.defs import INT, STR
from hdl_toolkit.synthesizer.rtlLevel.netlist import RtlNetlist
from hdl_toolkit.synthesizer.rtlLevel.signalUtils.walkers import walkAllOriginSignals
from hdl_toolkit.bitmask import Bitmask
from hdl_toolkit.hdlObjects.types.sliceVal import SliceVal

andTable = [ (None, None, None),
             (None, 0, 0),
             (None, 1, None),
             (0, None, 0),
             (0, 0, 0),
             (0, 1, 0),
             (1, 1, 1)]
orTable = [ (None, None, None),
            (None, 0, None),
            (None, 1, 1),
            (0, None, None),
            (0, 0, 0),
            (0, 1, 1),
            (1, 1, 1)]

class OperatorTC(unittest.TestCase):
    def setUp(self):
        unittest.TestCase.setUp(self)
        self.n = RtlNetlist()

    def testNoBool(self):
        for v in [True, False]:
            res = ~hBool(v)
            self.assertEqual(res.val, not v)
            self.assertEqual(res.vldMask, 1)
            self.assertEqual(res.updateTime, -1)

    def testNotBit(self):
        for v in [False, True]:
            res = ~hBit(v)

            self.assertEqual(res.val, int(not v))
            self.assertEqual(res.vldMask, 1)
            self.assertEqual(res.updateTime, -1)

    def testBitAnd(self):
        vals = {
            1: hBit(1),
            0: hBit(0),
            None: hBit(None)
        }

        for a, b, expected in andTable:
            res = vals[a] & vals[b]
            expectedRes = vals[expected]

            self.assertEqual(expectedRes.val, res.val,
                             "%s & %s  val=%s (should be %s)" % (repr(a), repr(b), repr(res.val), repr(expectedRes.val)))
            self.assertEqual(expectedRes.vldMask, res.vldMask,
                             "%s & %s  vldMask=%s (should be %s)" % (repr(a), repr(b), repr(res.vldMask), repr(expectedRes.vldMask)))
    def testBitOr(self):
        vals = {
            1: hBit(1),
            0: hBit(0),
            None: hBit(None)
        }

        for a, b, expected in orTable:
            res = vals[a] | vals[b]
            expectedRes = vals[expected]

            self.assertEqual(expectedRes.val, res.val,
                             "%s & %s  val=%s (should be %s)" % (repr(a), repr(b), repr(res.val), repr(expectedRes.val)))
            self.assertEqual(expectedRes.vldMask, res.vldMask,
                             "%s & %s  vldMask=%s (should be %s)" % (repr(a), repr(b), repr(res.vldMask), repr(expectedRes.vldMask)))

    def testBoolAnd(self):
        vals = {
            1: hBool(1),
            0: hBool(0),
            None: hBool(None)
        }

        for a, b, expected in andTable:
            res = vals[a] & vals[b]
            expectedRes = vals[expected]

            self.assertEqual(expectedRes.val, res.val,
                             "%s & %s  val=%s (should be %s)" % (repr(a), repr(b), repr(res.val), repr(expectedRes.val)))
            self.assertEqual(expectedRes.vldMask, res.vldMask,
                             "%s & %s  vldMask=%s (should be %s)" % (repr(a), repr(b), repr(res.vldMask), repr(expectedRes.vldMask)))
    def testBoolOr(self):
        vals = {
            1: hBool(1),
            0: hBool(0),
            None: hBool(None)
        }

        for a, b, expected in orTable:
            res = vals[a] | vals[b]
            expectedRes = vals[expected]

            self.assertEqual(expectedRes.val, res.val,
                             "%s & %s  val=%s (should be %s)" % (repr(a), repr(b), repr(res.val), repr(expectedRes.val)))
            self.assertEqual(expectedRes.vldMask, res.vldMask,
                             "%s & %s  vldMask=%s (should be %s)" % (repr(a), repr(b), repr(res.vldMask), repr(expectedRes.vldMask)))

    def testNotNotisOrigSig(self):
        a = self.n.sig("a")
        self.assertIs(a, ~ ~a)

    def testDownto(self):
        a = self.n.sig('a', typ=INT)
        a.defaultVal = hInt(10)
        b = hInt(0)
        r = a._downto(b)
        res = r.staticEval()
        self.assertEqual(res.val[0].val, 10)
        self.assertEqual(res.val[1].val, 0)

    def testwalkAllOriginSignalsDownto(self):
        a = self.n.sig('a', typ=INT)
        a.defaultVal = hInt(10)
        b = hInt(0)
        r = a._downto(b)
        origins = set(walkAllOriginSignals(r))
        self.assertSetEqual(origins, set([a]))

    def testwalkAllOriginSignalsDowntoAndPlus(self):
        a = self.n.sig('a', typ=INT)
        a.defaultVal = hInt(10)
        b = hInt(0)
        am = a + hInt(5)
        r = am._downto(b)
        origins = set(walkAllOriginSignals(r))
        self.assertSetEqual(origins, set([a]))

    def testADD_InvalidOperands(self):
        a = self.n.sig('a', typ=STR)
        b = self.n.sig('b')
        self.assertRaises(NotImplementedError, lambda : a + b)

    def testADD_IntBits(self):
        a = vec(7, 8)
        b = hInt(1)
        c = a + b
        self.assertEqual(c.val, 8)
        self.assertEqual(c.vldMask, Bitmask.mask(8))

        a = vec(255, 8)
        b = hInt(1)
        c = a + b
        self.assertEqual(c.val, 0)
        self.assertEqual(c.vldMask, Bitmask.mask(8))

        a = vec(7, 8, False)
        b = hInt(1)
        c = a + b
        self.assertEqual(c.val, 8)
        self.assertEqual(c.vldMask, Bitmask.mask(8))

        a = vec(255, 8, False)
        b = hInt(1)
        c = a + b
        self.assertEqual(c.val, 0)
        self.assertEqual(c.vldMask, Bitmask.mask(8))

    def testAND_eval(self):
        for a_in, b_in, out in [(0, 0, 0),
                                (0, 1, 0),
                                (1, 0, 0),
                                (1, 1, 1)]:
            res = hBit(a_in) & hBit(b_in)
            self.assertEqual(res.vldMask, 1)
            self.assertEqual(res.val, out, "a_in %d, b_in %d, out %d" % (a_in, b_in, out))

    def testADD_eval(self):
        for a_in, b_in, out in [(0, 0, 0),
                                (0, 1, 1),
                                (1, 0, 1),
                                (1, 1, 2)]:
            res = hInt(a_in) + hInt(b_in)

            b_w = 2

            self.assertTrue(res.vldMask)
            self.assertEqual(res.val, out, "a_in %d, b_in %d, out %d" % (a_in, b_in, out))

            resBit = vec(a_in, b_w) + vec(b_in, b_w)
            self.assertEqual(resBit.vldMask, 3)
            self.assertEqual(resBit.val, out, "a_in %d, b_in %d, out %d" % (a_in, b_in, out))
    #[TODO]
    #def testSlice(self):
    #    a = vec(11, 8)
    #    b = SliceVal(())

if __name__ == '__main__':
    suite = unittest.TestSuite()
    # suite.addTest(OperatorTC('testBitAnd'))
    suite.addTest(unittest.makeSuite(OperatorTC))
    runner = unittest.TextTestRunner(verbosity=3)
    runner.run(suite)
<|endoftext|>


<commit_before>import click
import yaml
import os
import glob

from dataclasses import dataclass

import yamldown

from typing import Dict, Set

"""
This module is for interacting with the different types of metadata that live
in the geneontology/go-site/metadata directory
"""

###############################################

class BiDiMultiMap(dict):

    def __init__(self, *args):
        super(BiDiMultiMap, self).__init__(*args)
        self.__reverse = dict()

    def __setitem__(self, key, val):

        # Turn everything into a set
        inval = None
        if isinstance(val, set):
            inval = val
        elif isinstance(val, iter):
            inval = set(val)
        elif val == None:
            inval = set()
        else:
            inval = set([val])

        olds = self.get(key, [])
        super(BiDiMultiMap, self).__setitem__(key, inval)
        # Remove old value mappings from the key
        for old in olds:
            if old in self.__reverse:
                del self.__reverse[old]

        # Add the items
        for v in inval:
            self.__reverse[v] = key

    def __delitem__(self, key):
        vs = self[key]
        super(BiDiMultiMap, self).__delitem__(key)
        for v in vs:
            del self.__reverse[v]

    def reverse(self, element):
        """
        Returns the key that has element in the set for the mapping of `key`, or
        None, of there is no such element.

        Example:
        Given "FB" --> set("fb", "FlyBase")

        reverse("fb") --> "FB"
        reverse("FlyBase") --> "FB"
        reverse("Foo") --> None
        """
        return self.__reverse.get(element, None)

# d = BiDiMultiMap()
#
# d["FB"] = set(["fb", "FlyBase"])
# d.reverse("fb") # returns "FB"
# d.reverse("FlyBase") # returns "FB"
# d.reverse("mgi:mgi") # returns None

###############################################
#
# @dataclass
# class YamldownMetadata:
#     id: str
#     layout: str
#     _meta: dict
#     _text: str

def yamldown_lookup(yamldown_dir):
    d = {
        metayaml_id(meta_path): get_yamldown_metadata(yamldown_dir, metayaml_id(meta_path))
        for meta_path in glob.glob("{}/*.md".format(os.path.join(yamldown_dir))) if metayaml_id(meta_path) not in ["ABOUT", "README", "SOP", "README-editors"]
    }

    return d

def dataset_metadata_file(metadata, group, empty_ok=False) -> Dict:
    metadata_yaml = os.path.join(metadata, "datasets", "{}.yaml".format(group))
    try:
        with open(metadata_yaml, "r") as group_data:
            click.echo("Found {group} metadata at {path}".format(group=group, path=metadata_yaml))
            return yaml.load(group_data, Loader=yaml.FullLoader)
    except Exception as e:
        if not empty_ok:
            raise click.ClickException("Could not find or read {}: {}".format(metadata_yaml, str(e)))
        else:
            return None

def extensions_constraints_file(metadata):
    constraints_path = os.path.join(metadata, "extensions-constraints.yaml")
    try:
        with open(constraints_path, "r") as constraints_file:
            click.echo("Found groups at {path}".format(path=constraints_path))
            constraints_list = yaml.load(constraints_file, Loader=yaml.FullLoader)
    except Exception as e:
        raise click.ClickException("Could not find or read {}: {}".format(constraints_path, str(e)))

    return constraints_list

def gorule_title(metadata, rule_id) -> str:
    return gorule_metadata(metadata, rule_id)["title"]

def gorule_metadata(metadata, rule_id) -> dict:
    gorule_yamldown = os.path.join(metadata, "rules")
    return get_yamldown_metadata(gorule_yamldown, rule_id)

def parse_goref_metadata(metadata, goref_id) -> dict:
    goref_yamldown = os.path.join(metadata, "gorefs")
    return get_yamldown_metadata(goref_yamldown, goref_id)

def get_yamldown_metadata(yamldown_dir, meta_id) -> dict:
    yamldown_md_path = os.path.join(yamldown_dir, "{}.md".format(meta_id))
    try:
        with open(yamldown_md_path, "r") as gorule_data:
            return yamldown.load(gorule_data)[0]
    except Exception as e:
        raise click.ClickException("Could not find or read {}: {}".format(yamldown_md_path))

def metayaml_id(rule_path) -> str:
    return os.path.splitext(os.path.basename(rule_path))[0]

def get_dataset_by_type(metadata, type):
    for dataset in metadata["datasets"]:
        if dataset["type"] == type:
            return dataset

def source_path(dataset_metadata, target_dir, group):
    extension = dataset_metadata["type"]
    if dataset_metadata["compression"]:
        extension = "{ext}.gz".format(ext=extension)

    path = os.path.join(target_dir, "groups", group, "{name}-src.{ext}".format(name=dataset_metadata["dataset"], ext=extension))
    return path

def target_path_by_type(metadata, target_dir, group, type):
    dataset_metadata = get_dataset_by_type(metadata, type)
    if dataset_metadata["compression"]:
        extension = "{ext}.gz".format(ext=type)

    path = os.path.join(target_dir, "groups", group, "{name}.{ext}".format(name=dataset_metadata["dataset"], ext=extension))
    if not os.path.exists(path):
        raise click.ClickException("Could not find or read {}".format(path))
    return path

def database_entities(metadata):
    dbxrefs_path = os.path.join(os.path.abspath(metadata), "db-xrefs.yaml")
    try:
        with open(dbxrefs_path, "r") as db_xrefs_file:
            click.echo("Found db-xrefs at {path}".format(path=dbxrefs_path))
            dbxrefs = yaml.load(db_xrefs_file, Loader=yaml.FullLoader)
    except Exception as e:
        raise click.ClickException("Could not find or read {}: {}".format(dbxrefs_path, str(e)))

    d = BiDiMultiMap()
    for entity in dbxrefs:
        d[entity["database"]] = set(entity.get("synonyms", []))

    return d

def groups(metadata) -> Set[str]:
    groups_path = os.path.join(os.path.abspath(metadata), "groups.yaml")
    try:
        with open(groups_path, "r") as groups_file:
            click.echo("Found groups at {path}".format(path=groups_path))
            groups_list = yaml.load(groups_file, Loader=yaml.FullLoader)
    except Exception as e:
        raise click.ClickException("Could not find or read {}: {}".format(groups_path, str(e)))

    return set([group["shorthand"] for group in groups_list])
<commit_msg>Remove changes to metadata.py<commit_after>import click
import yaml
import os
import glob

from dataclasses import dataclass

import yamldown

from typing import Dict, Set

"""
This module is for interacting with the different types of metadata that live
in the geneontology/go-site/metadata directory
"""

###############################################

class BiDiMultiMap(dict):

    def __init__(self, *args):
        super(BiDiMultiMap, self).__init__(*args)
        self.__reverse = dict()

    def __setitem__(self, key, val):

        # Turn everything into a set
        inval = None
        if isinstance(val, set):
            inval = val
        elif isinstance(val, iter):
            inval = set(val)
        elif val == None:
            inval = set()
        else:
            inval = set([val])

        olds = self.get(key, [])
        super(BiDiMultiMap, self).__setitem__(key, inval)
        # Remove old value mappings from the key
        for old in olds:
            if old in self.__reverse:
                del self.__reverse[old]

        # Add the items
        for v in inval:
            self.__reverse[v] = key

    def __delitem__(self, key):
        vs = self[key]
        super(BiDiMultiMap, self).__delitem__(key)
        for v in vs:
            del self.__reverse[v]

    def reverse(self, element):
        """
        Returns the key that has element in the set for the mapping of `key`, or
        None, of there is no such element.

        Example:
        Given "FB" --> set("fb", "FlyBase")

        reverse("fb") --> "FB"
        reverse("FlyBase") --> "FB"
        reverse("Foo") --> None
        """
        return self.__reverse.get(element, None)

# d = BiDiMultiMap()
#
# d["FB"] = set(["fb", "FlyBase"])
# d.reverse("fb") # returns "FB"
# d.reverse("FlyBase") # returns "FB"
# d.reverse("mgi:mgi") # returns None

###############################################
#
# @dataclass
# class YamldownMetadata:
#     id: str
#     layout: str
#     _meta: dict
#     _text: str

def yamldown_lookup(yamldown_dir):
    d = {
        metayaml_id(meta_path): get_yamldown_metadata(yamldown_dir, metayaml_id(meta_path))
        for meta_path in glob.glob("{}/*.md".format(os.path.join(yamldown_dir))) if metayaml_id(meta_path) not in ["ABOUT", "README", "SOP", "README-editors"]
    }

    return d

def dataset_metadata_file(metadata, group, empty_ok=False) -> Dict:
    metadata_yaml = os.path.join(metadata, "datasets", "{}.yaml".format(group))
    try:
        with open(metadata_yaml, "r") as group_data:
            click.echo("Found {group} metadata at {path}".format(group=group, path=metadata_yaml))
            return yaml.load(group_data, Loader=yaml.FullLoader)
    except Exception as e:
        if not empty_ok:
            raise click.ClickException("Could not find or read {}: {}".format(metadata_yaml, str(e)))
        else:
            return None

def extensions_constraints_file(metadata):
    constraints_path = os.path.join(metadata, "extensions-constraints.yaml")
    try:
        with open(constraints_path, "r") as constraints_file:
            click.echo("Found groups at {path}".format(path=constraints_path))
            constraints_list = yaml.load(constraints_file, Loader=yaml.FullLoader)
    except Exception as e:
        raise click.ClickException("Could not find or read {}: {}".format(constraints_path, str(e)))

    return constraints_list

def gorule_title(metadata, rule_id) -> str:
    return gorule_metadata(metadata, rule_id)["title"]

def gorule_metadata(metadata, rule_id) -> dict:
    gorule_yamldown = os.path.join(metadata, "rules")
    return get_yamldown_metadata(gorule_yamldown, rule_id)

def parse_goref_metadata(metadata, goref_id) -> dict:
    goref_yamldown = os.path.join(metadata, "gorefs")
    return get_yamldown_metadata(goref_yamldown, goref_id)

def get_yamldown_metadata(yamldown_dir, meta_id) -> dict:
    yamldown_md_path = os.path.join(yamldown_dir, "{}.md".format(meta_id))
    try:
        with open(yamldown_md_path, "r") as gorule_data:
            return yamldown.load(gorule_data)[0]
    except Exception as e:
        raise click.ClickException("Could not find or read {}: {}".format(yamldown_md_path))

def metayaml_id(rule_path) -> str:
    return os.path.splitext(os.path.basename(rule_path))[0]

def source_path(dataset_metadata, target_dir, group):
    extension = dataset_metadata["type"]
    if dataset_metadata["compression"]:
        extension = "{ext}.gz".format(ext=extension)

    path = os.path.join(target_dir, "groups", group, "{name}-src.{ext}".format(name=dataset_metadata["dataset"], ext=extension))
    return path

def database_entities(metadata):
    dbxrefs_path = os.path.join(os.path.abspath(metadata), "db-xrefs.yaml")
    try:
        with open(dbxrefs_path, "r") as db_xrefs_file:
            click.echo("Found db-xrefs at {path}".format(path=dbxrefs_path))
            dbxrefs = yaml.load(db_xrefs_file, Loader=yaml.FullLoader)
    except Exception as e:
        raise click.ClickException("Could not find or read {}: {}".format(dbxrefs_path, str(e)))

    d = BiDiMultiMap()
    for entity in dbxrefs:
        d[entity["database"]] = set(entity.get("synonyms", []))

    return d

def groups(metadata) -> Set[str]:
    groups_path = os.path.join(os.path.abspath(metadata), "groups.yaml")
    try:
        with open(groups_path, "r") as groups_file:
            click.echo("Found groups at {path}".format(path=groups_path))
            groups_list = yaml.load(groups_file, Loader=yaml.FullLoader)
    except Exception as e:
        raise click.ClickException("Could not find or read {}: {}".format(groups_path, str(e)))

    return set([group["shorthand"] for group in groups_list])
<|endoftext|>