<commit_before>
from basic import Basic, Atom, S, C, sympify
from methods import RelMeths, ArithMeths
#from cache import cache_it_nondummy

# from function import Function, WildFunction   /cyclic/

class Symbol(Atom, RelMeths, ArithMeths):
    """
    Assumptions::
       real = True
       commutative = True

    You can override the default assumptions in the constructor::
       >>> A,B = symbols('AB', commutative = False)
       >>> bool(A*B != B*A)
       True
       >>> bool(A*B*2 == 2*A*B) == True # multiplication by scalars is commutative
       True
    """

    is_comparable = False
    dummycount = 0

    #@cache_it_nondummy
    def __new__(cls, name, commutative=True, dummy=False,
                **assumptions):
        """if dummy == True, then this Symbol is totally unique, i.e.::

        >>> bool(Symbol("x") == Symbol("x")) == True
        True

        but with the dummy variable ::

        >>> bool(Symbol("x", dummy = True) == Symbol("x", dummy = True)) == True
        False

        """
        obj = Basic.__new__(cls,
                            commutative=commutative,
                            dummy=dummy,
                            **assumptions)
        if dummy:
            Symbol.dummycount += 1
            obj.dummy_index = Symbol.dummycount
        assert isinstance(name, str),`type(name)`
        obj.name = name
        return obj

    def _hashable_content(self):
        if self.is_dummy:
            return (self.name, self.dummy_index)
        return (self.name,)

    def tostr(self, level=0):
        if self.is_dummy:
            return '_' + self.name
        return self.name

    def torepr(self):
        if self.is_dummy:
            return '%s(%r, dummy=True)' % (self.__class__.__name__, self.name)
        return '%s(%r)' % (self.__class__.__name__, self.name)

    def as_dummy(self):
        assumptions = self._assumptions.copy()
        assumptions['dummy'] = True
        return self.__class__(self.name, **assumptions)

    def __call__(self, *args):
        assumptions = self._assumptions
        return Function(self.name, nargs=len(args))(*args, **assumptions)

    def _eval_integral(self, s):
        if self==s:
            return self**2/2
        return self*s

    def _eval_defined_integral(self, s, a, b):
        if self==s:
            return (b**2-a**2)/2
        return self*(b-a)

    def _eval_expand_complex(self, *args):
        return C.re(self) + C.im(self)*S.ImaginaryUnit

    def _sage_(self):
        import sage.all as sage
        return sage.var(self.name)

class Wild(Symbol):
    """
    Wild() matches any expression but another Wild().
    """

    def __new__(cls, name=None, exclude=None, **assumptions):
        assumptions['dummy'] = True
        if name is None:
            name = 'W%s' % (Symbol.dummycount+1)
        obj = Symbol.__new__(cls, name, **assumptions)
        if exclude is None:
            obj.exclude = None
        else:
            obj.exclude = [sympify(x) for x in exclude]
        return obj

    def matches(pattern, expr, repl_dict={}, evaluate=False):
        for p,v in repl_dict.items():
            if p==pattern:
                if v==expr: return repl_dict
                return None
        if pattern.exclude:
            for x in pattern.exclude:
                if x in expr:
                    return None
                #else:
                #    print x, expr, pattern, expr, pattern.exclude
        repl_dict = repl_dict.copy()
        repl_dict[pattern] = expr
        return repl_dict

    def __call__(self, *args, **assumptions):
        return WildFunction(self.name, nargs=len(args))(*args, **assumptions)

    def tostr(self, level=0):
        return self.name + '_'

class Temporary(Symbol):
    """
    Indexed dummy symbol.
    """
    def __new__(cls, **assumptions):
        assumptions['dummy'] = True
        name = 'T%s' % (Symbol.dummycount+1)
        return Symbol.__new__(cls, name, **assumptions)

def symbols(*names, **kwargs):
    """Returns a list of symbols with names taken from 'names'
       argument, which can be a string, then each character
       forms a separate symbol, or a sequence of strings.

       All newly created symbols have assumptions set acordingly
       to 'kwargs'. Main intention behind this function is to
       simplify and shorten examples code in doc-strings.

       >>> x, y, z = symbols('xyz')

       >>> a = symbols('a', integer=True)

       >>> a.is_integer
       True

       >>> xx, yy, zz = symbols('xx', 'yy', 'zz', real=True)

       >>> xx.is_real and yy.is_real and zz.is_real
       True

    """
    if len(names) == 1:
        result = [ Symbol(name, **kwargs) for name in names[0] ]
    else:
        result = [ Symbol(name, **kwargs) for name in names ]

    if len(result) == 1:
        return result[0]
    else:
        return result

def var(s):
    """
        Create a symbolic variable with the name *s*.

        INPUT:
            s -- a string, either a single variable name, or
                 a space separated list of variable names, or
                 a list of variable names.

        NOTE: The new variable is both returned and automatically injected into
        the parent's *global* namespace.  It's recommended not to use "var" in
        library code, it is better to use symbols() instead.

        EXAMPLES:
        We define some symbolic variables:
            >>> var('m')
            m
            >>> var('n xx yy zz')
            (n, xx, yy, zz)
            >>> n
            n

    """
    import inspect
    frame = inspect.currentframe().f_back

    try:
        if not isinstance(s, list):
            s = s.split(" ")

        res = []

        for t in s:
            # skip empty strings
            if not t:
                continue
            sym = Symbol(t)
            frame.f_globals[t] = sym
            res.append(sym)

        res = tuple(res)
        if len(res) == 0:   # var('')
            res = None
        elif len(res) == 1: # var('x')
            res = res[0]
                            # otherwise var('a b ...')
        return res

    finally:
        # we should explicitly break cyclic dependencies as stated in inspect
        # doc
        del frame

# /cyclic/
import basic as _
_.Symbol    = Symbol
_.Wild      = Wild
_.Temporary = Temporary
del _
<commit_msg>Symbol: split into Symbol & Dummy for efficiency<commit_after>
from basic import Basic, Atom, S, C, sympify
from methods import RelMeths, ArithMeths
#from cache import cache_it_nondummy

# from function import Function, WildFunction   /cyclic/

class Symbol(Atom, RelMeths, ArithMeths):
    """
    Assumptions::
       real = True
       commutative = True

    You can override the default assumptions in the constructor::
       >>> A,B = symbols('AB', commutative = False)
       >>> bool(A*B != B*A)
       True
       >>> bool(A*B*2 == 2*A*B) == True # multiplication by scalars is commutative
       True
    """

    is_comparable = False
    is_dummy = False

    #@cache_it_nondummy
    def __new__(cls, name, commutative=True, dummy=False,
                **assumptions):
        """if dummy == True, then this Symbol is totally unique, i.e.::

        >>> bool(Symbol("x") == Symbol("x")) == True
        True

        but with the dummy variable ::

        >>> bool(Symbol("x", dummy = True) == Symbol("x", dummy = True)) == True
        False

        """

        # XXX compatibility stuff
        if dummy==True:
            return Dummy(name, commutative=commutative, **assumptions)

        obj = Basic.__new__(cls,
                            commutative=commutative,
                            **assumptions)
        assert isinstance(name, str),`type(name)`
        obj.name = name
        return obj

    def _hashable_content(self):
        return (self.name,)

    def tostr(self, level=0):
        return self.name

    def torepr(self):
        return '%s(%r)' % (self.__class__.__name__, self.name)

    def as_dummy(self):
        assumptions = self._assumptions.copy()
        return Dummy(self.name, **assumptions)

    def __call__(self, *args):
        assumptions = self._assumptions
        return Function(self.name, nargs=len(args))(*args, **assumptions)

    def _eval_integral(self, s):
        if self==s:
            return self**2/2
        return self*s

    def _eval_defined_integral(self, s, a, b):
        if self==s:
            return (b**2-a**2)/2
        return self*(b-a)

    def _eval_expand_complex(self, *args):
        return C.re(self) + C.im(self)*S.ImaginaryUnit

    def _sage_(self):
        import sage.all as sage
        return sage.var(self.name)

class Dummy(Symbol):
    """Dummy Symbol

       use this through Symol:

       >>> x1 = Symbol('x', dummy=True)
       >>> x2 = Symbol('x', dummy=True)
       >>> bool(x1 == x2)
       False
    """

    is_dummy = True
    dummycount = 0

    def __new__(cls, name, commutative=True, **assumptions):
        obj = Symbol.__new__(cls, name, commutative=commutative, **assumptions)

        Dummy.dummycount += 1
        obj.dummy_index = Dummy.dummycount
        return obj

    def _hashable_content(self):
        return (self.name, self.dummy_index)

    def tostr(self, level=0):
        return '_' + self.name

class Temporary(Dummy):
    """
    Indexed dummy symbol.
    """
    def __new__(cls, **assumptions):
        obj = Dummy.__new__(cls, 'T%i' % Dummy.dummycount, **assumptions)
        return obj

class Wild(Dummy):
    """
    Wild() matches any expression but another Wild().
    """

    def __new__(cls, name=None, exclude=None, **assumptions):
        if name is None:
            name = 'W%s' % (Dummy.dummycount+1)
        obj = Dummy.__new__(cls, name, **assumptions)
        if exclude is None:
            obj.exclude = None
        else:
            obj.exclude = [sympify(x) for x in exclude]
        return obj

    def matches(pattern, expr, repl_dict={}, evaluate=False):
        for p,v in repl_dict.items():
            if p==pattern:
                if v==expr: return repl_dict
                return None
        if pattern.exclude:
            for x in pattern.exclude:
                if x in expr:
                    return None
                #else:
                #    print x, expr, pattern, expr, pattern.exclude
        repl_dict = repl_dict.copy()
        repl_dict[pattern] = expr
        return repl_dict

    def __call__(self, *args, **assumptions):
        return WildFunction(self.name, nargs=len(args))(*args, **assumptions)

    def tostr(self, level=0):
        return self.name + '_'

def symbols(*names, **kwargs):
    """Returns a list of symbols with names taken from 'names'
       argument, which can be a string, then each character
       forms a separate symbol, or a sequence of strings.

       All newly created symbols have assumptions set acordingly
       to 'kwargs'. Main intention behind this function is to
       simplify and shorten examples code in doc-strings.

       >>> x, y, z = symbols('xyz')

       >>> a = symbols('a', integer=True)

       >>> a.is_integer
       True

       >>> xx, yy, zz = symbols('xx', 'yy', 'zz', real=True)

       >>> xx.is_real and yy.is_real and zz.is_real
       True

    """
    if len(names) == 1:
        result = [ Symbol(name, **kwargs) for name in names[0] ]
    else:
        result = [ Symbol(name, **kwargs) for name in names ]

    if len(result) == 1:
        return result[0]
    else:
        return result

def var(s):
    """
        Create a symbolic variable with the name *s*.

        INPUT:
            s -- a string, either a single variable name, or
                 a space separated list of variable names, or
                 a list of variable names.

        NOTE: The new variable is both returned and automatically injected into
        the parent's *global* namespace.  It's recommended not to use "var" in
        library code, it is better to use symbols() instead.

        EXAMPLES:
        We define some symbolic variables:
            >>> var('m')
            m
            >>> var('n xx yy zz')
            (n, xx, yy, zz)
            >>> n
            n

    """
    import inspect
    frame = inspect.currentframe().f_back

    try:
        if not isinstance(s, list):
            s = s.split(" ")

        res = []

        for t in s:
            # skip empty strings
            if not t:
                continue
            sym = Symbol(t)
            frame.f_globals[t] = sym
            res.append(sym)

        res = tuple(res)
        if len(res) == 0:   # var('')
            res = None
        elif len(res) == 1: # var('x')
            res = res[0]
                            # otherwise var('a b ...')
        return res

    finally:
        # we should explicitly break cyclic dependencies as stated in inspect
        # doc
        del frame

# /cyclic/
import basic as _
_.Symbol    = Symbol
_.Wild      = Wild
_.Temporary = Temporary
del _
<|endoftext|>


<commit_before>import os
import time
import json
import requests
from oti_search import OTISearch
from ConfigParser import SafeConfigParser
import urllib2
import sys
import traceback
import api_utils

@request.restful()
def v1():
    "The OpenTree API v1"
    response.view = 'generic.json'

    oti_base_url = api_utils.get_oti_base_url(request)  # WAS conf.get("apis", "oti_base_url")
    oti = OTISearch(oti_base_url)

    def GET(kind, property_name, search_term,jsoncallback=None,callback=None,_=None,**kwargs):
        """"OpenTree API methods relating to searching
Example:

    http://localhost:8000/api/search/v1/tree/ot-ottTaxonName/Carex
    http://localhost:8000/api/search/v1/node/ot-ottId/1000455

When searching for a property name ot:foo, ot-foo must be used
because web2py does not recognize URLs that contain a colon
other than specifying a port, even if URL encoded.

"""
        opentree_docstore_url = _read_from_local_config(request, "apis", "opentree_docstore_url")

        # support JSONP request from another domain
        if jsoncallback or callback:
            response.view = 'generic.jsonp'

        # colons don't play nicely with GET requests
        property_name = property_name.replace("-",":")

        valid_kinds = ["study", "tree", "node"]
        if kind in valid_kinds:
            return oti.do_search(kind, property_name, search_term)
        else:
            raise HTTP(400,json.dumps({"error":1, "description":"not a valid property name"}))

    return locals()

def nudgeStudyIndexOnUpdates():
    """"Support method to update oti index in response to GitHub webhooks

This examines the JSON payload of a GitHub webhook to see which studies have
been added, modified, or removed. Then it calls oti's index service to
(re)index the NexSON for those studies, or to delete a study's information if
it was deleted from the docstore.

Finally, we clear the cached study list (response to find_studies with no args).

N.B. This depends on a GitHub webhook on the chosen docstore.
"""
    opentree_docstore_url = _read_from_local_config(request, "apis", "opentree_docstore_url")

    payload = request.vars
    msg = ''

    # EXAMPLE of a working curl call to nudge index:
    # curl -X POST -d '{"urls": ["https://raw.github.com/OpenTreeOfLife/phylesystem/master/study/10/10.json", "https://raw.github.com/OpenTreeOfLife/phylesystem/master/study/9/9.json"]}' -H "Content-type: application/json" http://ec2-54-203-194-13.us-west-2.compute.amazonaws.com/oti/ext/IndexServices/graphdb/indexNexsons

    # Pull needed values from config file (typical values shown)
    #   opentree_docstore_url = "https://github.com/OpenTreeOfLife/phylesystem"        # munge this to grab raw NexSON)
    #   oti_base_url='http://ec2-54-203-194-13.us-west-2.compute.amazonaws.com/oti'    # confirm we're pushing to the right OTI service(s)!
    try:
        # how we nudge the index depends on which studies are new, changed, or deleted
        added_study_ids = [ ]
        modified_study_ids = [ ]
        removed_study_ids = [ ]
        # TODO: Should any of these lists override another? maybe use commit timestamps to "trump" based on later operations?
        for commit in payload['commits']:
            _harvest_study_ids_from_paths( commit['added'], added_study_ids )
            _harvest_study_ids_from_paths( commit['modified'], modified_study_ids )
            _harvest_study_ids_from_paths( commit['removed'], removed_study_ids )

        # "flatten" each list to remove duplicates
        added_study_ids = list(set(added_study_ids))
        modified_study_ids = list(set(modified_study_ids))
        removed_study_ids = list(set(removed_study_ids))

    except:
        raise HTTP(400,json.dumps({"error":1, "description":"malformed GitHub payload"}))

    if payload['repository']['url'] != opentree_docstore_url:
        raise HTTP(400,json.dumps({"error":1, "description":"wrong repo for this API instance"}))

    #nexson_url_template = opentree_docstore_url.replace("github.com", "raw.github.com") + "/master/study/%s/%s.json"
    nexson_url_template = URL(r=request,
                              c="default",
                              f="v1",
                              args=["study", "%s"],
                              vars={'output_nexml2json': '0.0.0'},
                              scheme=True,
                              host=True,
                              url_encode=False)

    # for now, let's just add/update new and modified studies using indexNexsons
    add_or_update_ids = added_study_ids + modified_study_ids
    # NOTE that passing deleted_study_ids (any non-existent file paths) will
    # fail on oti, with a FileNotFoundException!
    add_or_update_ids = list(set(add_or_update_ids))  # remove any duplicates

    oti_base_url = api_utils.get_oti_base_url(request)  # WAS conf.get("apis", "oti_base_url")
    if len(add_or_update_ids) > 0:
        nudge_url = "%s/oti/ext/IndexServices/graphdb/indexNexsons" % (oti_base_url,)
        nexson_urls = [ (nexson_url_template % (study_id,)) for study_id in add_or_update_ids ]

        # N.B. that gluon.tools.fetch() can't be used here, since it won't send
        # "raw" JSON data as treemachine expects
        req = urllib2.Request(
            url=nudge_url,
            data=json.dumps({
                "urls": nexson_urls
            }),
            headers={"Content-Type": "application/json"}
        )
        try:
            nudge_response = urllib2.urlopen(req).read()
            updated_study_ids = json.loads( nudge_response )
        except Exception, e:
            # TODO: log oti exceptions into my response
            exc_type, exc_value, exc_traceback = sys.exc_info()
            msg += """indexNexsons failed!'
nudge_url: %s
nexson_url_template: %s
nexson_urls: %s
%s""" % (nudge_url, nexson_url_template, nexson_urls, traceback.format_exception(exc_type, exc_value, exc_traceback),)

        # TODO: check returned IDs against our original lists... what if something failed?

    if len(removed_study_ids) > 0:
        # Un-index the studies that were removed from docstore
        remove_url = "%s/oti/ext/IndexServices/graphdb/unindexNexsons" % (oti_base_url,)
        req = urllib2.Request(
            url=remove_url,
            data=json.dumps({
                "ids": removed_study_ids
            }),
            headers={"Content-Type": "application/json"}
        )
        try:
            remove_response = urllib2.urlopen(req).read()
            unindexed_study_ids = json.loads( remove_response )
        except Exception, e:
            # TODO: log oti exceptions into my response
            exc_type, exc_value, exc_traceback = sys.exc_info()
            msg += """unindexNexsons failed!'
remove_url: %s
removed_study_ids: %s
%s""" % (remove_url, removed_study_ids, traceback.format_exception(exc_type, exc_value, exc_traceback),)

        # TODO: check returned IDs against our original list... what if something failed?

    # Clear any cached study lists (both verbose and non-verbose)
    api_utils.clear_matching_cache_keys(".*find_studies.*")

    github_webhook_url = "%s/settings/hooks" % opentree_docstore_url
    full_msg = """This URL should be called by a webhook set in the docstore repo:

%s

%s

""" % (github_webhook_url, github_webhook_url, msg,)
    if msg == '':
        return full_msg
    else:
        raise HTTP(500, full_msg)

def nudgeTaxonIndexOnUpdates():
    """"Support method to update taxon index (taxomachine) in response to GitHub webhooks

This examines the JSON payload of a GitHub webhook to see which taxa have
been added, modified, or removed. Then it calls the appropriate index service to
(re)index these taxa, or to delete a taxon's information if it was deleted in
an amendment.

TODO: Clear any cached taxon list.

N.B. This depends on a GitHub webhook on the taxonomic-amendments docstore!
"""
    amendments_repo_url = _read_from_local_config(request, "apis", "amendments_repo_url")
    payload = request.vars
    if payload['repository']['url'] != amendments_repo_url:
        raise HTTP(400,json.dumps({"error":1, "description":"wrong repo for this API instance"}))

    msg = ''
    try:
        # how we nudge the index depends on which taxa are new, changed, or deleted
        added_ott_ids = [ ]
        modified_ott_ids = [ ]
        removed_ott_ids = [ ]
        # TODO: Should any of these lists override another? maybe use commit timestamps to "trump" based on later operations?
        for commit in payload['commits']:
            _harvest_ott_ids_from_paths( commit['added'], added_ott_ids )
            _harvest_ott_ids_from_paths( commit['modified'], modified_ott_ids )
            _harvest_ott_ids_from_paths( commit['removed'], removed_ott_ids )
        # "flatten" each list to remove duplicates
        added_ott_ids = list(set(added_ott_ids))
        modified_ott_ids = list(set(modified_ott_ids))
        removed_ott_ids = list(set(removed_ott_ids))
    except:
        raise HTTP(400,json.dumps({"error":1, "description":"malformed GitHub payload"}))

    # build a working URL, gather amendment body, and nudge the index!
    amendments_api_base_url = api_utils.get_amendments_api_base_url(request)

    if len(added_ott_ids) > 0:
        nudge_url = "{b}v3/taxonomy/process_additions".format(b=amendments_api_base_url)

        for ott_id in added_ott_ids:
            # fetch the JSON body of each new amendment and submit it for indexing
            fetch_url = "{b}v3/amendment/{i}".format(b=amendments_api_base_url, i=ott_id)
            req = urllib2.Request(
                url=nudge_url,
            )
            try:
                nudge_response = urllib2.urlopen(req).read()
                # strip away metadata (version history, etc.)
                amendment_blob = json.loads( nudge_response ).get('data')
            except Exception, e:
                # bail and report the error in webhook response
                exc_type, exc_value, exc_traceback = sys.exc_info()
                msg += """fetch of amendment failed!'
    fetch_url: %s
    ott_id: %s
    %s""" % (fetch_url, ott_id, traceback.format_exception(exc_type, exc_value, exc_traceback),)
                break

            # N.B. that gluon.tools.fetch() can't be used here, since it won't send
            # "raw" JSON data as taxomachine expects
            POST_data = json.dumps({"addition_document": amendment_blob})
            req = urllib2.Request(
                url=nudge_url,
                data=POST_data,
                headers={"Content-Type": "application/json"}
            )
            try:
                nudge_response = urllib2.urlopen(req).read()
                msg = nudge_response   # TODO: REMOVE THIS!
            except Exception, e:
                # report the error in webhook response
                exc_type, exc_value, exc_traceback = sys.exc_info()
                msg += """index amendments failed!'
    nudge_url: %s
    POST_data: %s
    nudge_response: %s
    added_ott_ids: %s
    %s""" % (nudge_url, POST_data, nudge_response, added_ott_ids, traceback.format_exception(exc_type, exc_value, exc_traceback),)

    # LATER: add handlers for modified and removed taxa?
    if len(modified_ott_ids) > 0:
        raise HTTP(400,json.dumps({
            "error":1,
            "description":"We don't currently re-index modified taxa!"}))
    if len(removed_ott_ids) > 0:
        raise HTTP(400,json.dumps({
            "error":1,
            "description":"We don't currently re-index removed taxa!"}))

    # N.B. If we had any cached amendment results, we'd clear them now
    #api_utils.clear_matching_cache_keys(...)

    github_webhook_url = "%s/settings/hooks" % amendments_repo_url
    full_msg = """This URL should be called by a webhook set in the amendments repo:

%s

%s

""" % (github_webhook_url, github_webhook_url, msg,)
    if msg == '':
        return full_msg
    else:
        raise HTTP(500, full_msg)

def _read_from_local_config(request, section_name, key_name):
    app_name = request.application
    conf = SafeConfigParser(allow_no_value=True)
    if os.path.isfile("%s/applications/%s/private/localconfig" % (os.path.abspath('.'), app_name,)):
        conf.read("%s/applications/%s/private/localconfig" % (os.path.abspath('.'), app_name,))
    else:
        conf.read("%s/applications/%s/private/config" % (os.path.abspath('.'), app_name,))
    return conf.get(section_name, key_name)

def _harvest_study_ids_from_paths( path_list, target_array ):
    for path in path_list:
        path_parts = path.split('/')
        if path_parts[0] == "study":
            # skip any intermediate directories in docstore repo
            study_id = path_parts[ len(path_parts) - 2 ]
            target_array.append(study_id)

def _harvest_ott_ids_from_paths( path_list, target_array ):
    for path in path_list:
        path_parts = path.split('/')
        # ignore changes to counter file, other directories, etc.
        if path_parts[0] == "amendments":
            # skip intermediate directories in docstore repo
            amendment_file_name = path_parts.pop()
            ott_id = amendment_file_name[:-5]
            target_array.append(ott_id)
<commit_msg>Yet more diagnostic junk<commit_after>import os
import time
import json
import requests
from oti_search import OTISearch
from ConfigParser import SafeConfigParser
import urllib2
import sys
import traceback
import api_utils

@request.restful()
def v1():
    "The OpenTree API v1"
    response.view = 'generic.json'

    oti_base_url = api_utils.get_oti_base_url(request)  # WAS conf.get("apis", "oti_base_url")
    oti = OTISearch(oti_base_url)

    def GET(kind, property_name, search_term,jsoncallback=None,callback=None,_=None,**kwargs):
        """"OpenTree API methods relating to searching
Example:

    http://localhost:8000/api/search/v1/tree/ot-ottTaxonName/Carex
    http://localhost:8000/api/search/v1/node/ot-ottId/1000455

When searching for a property name ot:foo, ot-foo must be used
because web2py does not recognize URLs that contain a colon
other than specifying a port, even if URL encoded.

"""
        opentree_docstore_url = _read_from_local_config(request, "apis", "opentree_docstore_url")

        # support JSONP request from another domain
        if jsoncallback or callback:
            response.view = 'generic.jsonp'

        # colons don't play nicely with GET requests
        property_name = property_name.replace("-",":")

        valid_kinds = ["study", "tree", "node"]
        if kind in valid_kinds:
            return oti.do_search(kind, property_name, search_term)
        else:
            raise HTTP(400,json.dumps({"error":1, "description":"not a valid property name"}))

    return locals()

def nudgeStudyIndexOnUpdates():
    """"Support method to update oti index in response to GitHub webhooks

This examines the JSON payload of a GitHub webhook to see which studies have
been added, modified, or removed. Then it calls oti's index service to
(re)index the NexSON for those studies, or to delete a study's information if
it was deleted from the docstore.

Finally, we clear the cached study list (response to find_studies with no args).

N.B. This depends on a GitHub webhook on the chosen docstore.
"""
    opentree_docstore_url = _read_from_local_config(request, "apis", "opentree_docstore_url")

    payload = request.vars
    msg = ''

    # EXAMPLE of a working curl call to nudge index:
    # curl -X POST -d '{"urls": ["https://raw.github.com/OpenTreeOfLife/phylesystem/master/study/10/10.json", "https://raw.github.com/OpenTreeOfLife/phylesystem/master/study/9/9.json"]}' -H "Content-type: application/json" http://ec2-54-203-194-13.us-west-2.compute.amazonaws.com/oti/ext/IndexServices/graphdb/indexNexsons

    # Pull needed values from config file (typical values shown)
    #   opentree_docstore_url = "https://github.com/OpenTreeOfLife/phylesystem"        # munge this to grab raw NexSON)
    #   oti_base_url='http://ec2-54-203-194-13.us-west-2.compute.amazonaws.com/oti'    # confirm we're pushing to the right OTI service(s)!
    try:
        # how we nudge the index depends on which studies are new, changed, or deleted
        added_study_ids = [ ]
        modified_study_ids = [ ]
        removed_study_ids = [ ]
        # TODO: Should any of these lists override another? maybe use commit timestamps to "trump" based on later operations?
        for commit in payload['commits']:
            _harvest_study_ids_from_paths( commit['added'], added_study_ids )
            _harvest_study_ids_from_paths( commit['modified'], modified_study_ids )
            _harvest_study_ids_from_paths( commit['removed'], removed_study_ids )

        # "flatten" each list to remove duplicates
        added_study_ids = list(set(added_study_ids))
        modified_study_ids = list(set(modified_study_ids))
        removed_study_ids = list(set(removed_study_ids))

    except:
        raise HTTP(400,json.dumps({"error":1, "description":"malformed GitHub payload"}))

    if payload['repository']['url'] != opentree_docstore_url:
        raise HTTP(400,json.dumps({"error":1, "description":"wrong repo for this API instance"}))

    #nexson_url_template = opentree_docstore_url.replace("github.com", "raw.github.com") + "/master/study/%s/%s.json"
    nexson_url_template = URL(r=request,
                              c="default",
                              f="v1",
                              args=["study", "%s"],
                              vars={'output_nexml2json': '0.0.0'},
                              scheme=True,
                              host=True,
                              url_encode=False)

    # for now, let's just add/update new and modified studies using indexNexsons
    add_or_update_ids = added_study_ids + modified_study_ids
    # NOTE that passing deleted_study_ids (any non-existent file paths) will
    # fail on oti, with a FileNotFoundException!
    add_or_update_ids = list(set(add_or_update_ids))  # remove any duplicates

    oti_base_url = api_utils.get_oti_base_url(request)  # WAS conf.get("apis", "oti_base_url")
    if len(add_or_update_ids) > 0:
        nudge_url = "%s/oti/ext/IndexServices/graphdb/indexNexsons" % (oti_base_url,)
        nexson_urls = [ (nexson_url_template % (study_id,)) for study_id in add_or_update_ids ]

        # N.B. that gluon.tools.fetch() can't be used here, since it won't send
        # "raw" JSON data as treemachine expects
        req = urllib2.Request(
            url=nudge_url,
            data=json.dumps({
                "urls": nexson_urls
            }),
            headers={"Content-Type": "application/json"}
        )
        try:
            nudge_response = urllib2.urlopen(req).read()
            updated_study_ids = json.loads( nudge_response )
        except Exception, e:
            # TODO: log oti exceptions into my response
            exc_type, exc_value, exc_traceback = sys.exc_info()
            msg += """indexNexsons failed!'
nudge_url: %s
nexson_url_template: %s
nexson_urls: %s
%s""" % (nudge_url, nexson_url_template, nexson_urls, traceback.format_exception(exc_type, exc_value, exc_traceback),)

        # TODO: check returned IDs against our original lists... what if something failed?

    if len(removed_study_ids) > 0:
        # Un-index the studies that were removed from docstore
        remove_url = "%s/oti/ext/IndexServices/graphdb/unindexNexsons" % (oti_base_url,)
        req = urllib2.Request(
            url=remove_url,
            data=json.dumps({
                "ids": removed_study_ids
            }),
            headers={"Content-Type": "application/json"}
        )
        try:
            remove_response = urllib2.urlopen(req).read()
            unindexed_study_ids = json.loads( remove_response )
        except Exception, e:
            # TODO: log oti exceptions into my response
            exc_type, exc_value, exc_traceback = sys.exc_info()
            msg += """unindexNexsons failed!'
remove_url: %s
removed_study_ids: %s
%s""" % (remove_url, removed_study_ids, traceback.format_exception(exc_type, exc_value, exc_traceback),)

        # TODO: check returned IDs against our original list... what if something failed?

    # Clear any cached study lists (both verbose and non-verbose)
    api_utils.clear_matching_cache_keys(".*find_studies.*")

    github_webhook_url = "%s/settings/hooks" % opentree_docstore_url
    full_msg = """This URL should be called by a webhook set in the docstore repo:

%s

%s

""" % (github_webhook_url, github_webhook_url, msg,)
    if msg == '':
        return full_msg
    else:
        raise HTTP(500, full_msg)

def nudgeTaxonIndexOnUpdates():
    """"Support method to update taxon index (taxomachine) in response to GitHub webhooks

This examines the JSON payload of a GitHub webhook to see which taxa have
been added, modified, or removed. Then it calls the appropriate index service to
(re)index these taxa, or to delete a taxon's information if it was deleted in
an amendment.

TODO: Clear any cached taxon list.

N.B. This depends on a GitHub webhook on the taxonomic-amendments docstore!
"""
    amendments_repo_url = _read_from_local_config(request, "apis", "amendments_repo_url")
    payload = request.vars
    if payload['repository']['url'] != amendments_repo_url:
        raise HTTP(400,json.dumps({"error":1, "description":"wrong repo for this API instance"}))

    msg = ''
    try:
        # how we nudge the index depends on which taxa are new, changed, or deleted
        added_ott_ids = [ ]
        modified_ott_ids = [ ]
        removed_ott_ids = [ ]
        # TODO: Should any of these lists override another? maybe use commit timestamps to "trump" based on later operations?
        for commit in payload['commits']:
            _harvest_ott_ids_from_paths( commit['added'], added_ott_ids )
            _harvest_ott_ids_from_paths( commit['modified'], modified_ott_ids )
            _harvest_ott_ids_from_paths( commit['removed'], removed_ott_ids )
        # "flatten" each list to remove duplicates
        added_ott_ids = list(set(added_ott_ids))
        modified_ott_ids = list(set(modified_ott_ids))
        removed_ott_ids = list(set(removed_ott_ids))
    except:
        raise HTTP(400,json.dumps({"error":1, "description":"malformed GitHub payload"}))

    # build a working URL, gather amendment body, and nudge the index!
    amendments_api_base_url = api_utils.get_amendments_api_base_url(request)

    if len(added_ott_ids) > 0:
        nudge_url = "{b}v3/taxonomy/process_additions".format(b=amendments_api_base_url)

        for ott_id in added_ott_ids:
            # fetch the JSON body of each new amendment and submit it for indexing
            fetch_url = "{b}v3/amendment/{i}".format(b=amendments_api_base_url, i=ott_id)
            req = urllib2.Request(
                url=nudge_url,
                data={}
            )
            try:
                fetch_response = urllib2.urlopen(req).read()
                # strip away metadata (version history, etc.)
                amendment_blob = json.loads( fetch_response ).get('data')
            except Exception, e:
                # bail and report the error in webhook response
                exc_type, exc_value, exc_traceback = sys.exc_info()
                msg += """fetch of amendment failed!'
    fetch_url: %s
    ott_id: %s
    %s""" % (fetch_url, ott_id, traceback.format_exception(exc_type, exc_value, exc_traceback),)
                break

            # N.B. that gluon.tools.fetch() can't be used here, since it won't send
            # "raw" JSON data as taxomachine expects
            POST_data = json.dumps({"addition_document": amendment_blob})
            req = urllib2.Request(
                url=nudge_url,
                data=POST_data,
                headers={"Content-Type": "application/json"}
            )
            try:
                nudge_response = urllib2.urlopen(req).read()
                msg = nudge_response   # TODO: REMOVE THIS!
            except Exception, e:
                # report the error in webhook response
                exc_type, exc_value, exc_traceback = sys.exc_info()
                msg += """index amendments failed!'
    nudge_url: %s
    POST_data: %s
    fetch_response: %s
    nudge_response: %s
    added_ott_ids: %s
    %s""" % (nudge_url, POST_data, fetch_response, nudge_response, added_ott_ids, traceback.format_exception(exc_type, exc_value, exc_traceback),)

    # LATER: add handlers for modified and removed taxa?
    if len(modified_ott_ids) > 0:
        raise HTTP(400,json.dumps({
            "error":1,
            "description":"We don't currently re-index modified taxa!"}))
    if len(removed_ott_ids) > 0:
        raise HTTP(400,json.dumps({
            "error":1,
            "description":"We don't currently re-index removed taxa!"}))

    # N.B. If we had any cached amendment results, we'd clear them now
    #api_utils.clear_matching_cache_keys(...)

    github_webhook_url = "%s/settings/hooks" % amendments_repo_url
    full_msg = """This URL should be called by a webhook set in the amendments repo:

%s

%s

""" % (github_webhook_url, github_webhook_url, msg,)
    if msg == '':
        return full_msg
    else:
        raise HTTP(500, full_msg)

def _read_from_local_config(request, section_name, key_name):
    app_name = request.application
    conf = SafeConfigParser(allow_no_value=True)
    if os.path.isfile("%s/applications/%s/private/localconfig" % (os.path.abspath('.'), app_name,)):
        conf.read("%s/applications/%s/private/localconfig" % (os.path.abspath('.'), app_name,))
    else:
        conf.read("%s/applications/%s/private/config" % (os.path.abspath('.'), app_name,))
    return conf.get(section_name, key_name)

def _harvest_study_ids_from_paths( path_list, target_array ):
    for path in path_list:
        path_parts = path.split('/')
        if path_parts[0] == "study":
            # skip any intermediate directories in docstore repo
            study_id = path_parts[ len(path_parts) - 2 ]
            target_array.append(study_id)

def _harvest_ott_ids_from_paths( path_list, target_array ):
    for path in path_list:
        path_parts = path.split('/')
        # ignore changes to counter file, other directories, etc.
        if path_parts[0] == "amendments":
            # skip intermediate directories in docstore repo
            amendment_file_name = path_parts.pop()
            ott_id = amendment_file_name[:-5]
            target_array.append(ott_id)
<|endoftext|>


<commit_before>VERSION = (1, 0, 12)
<commit_msg>Bump version<commit_after>VERSION = (1, 0, 13)
<|endoftext|>


<commit_before># some logic (NOT all) has been moved to views to be more 'pyramid-y'
from pyramid.response import Response
from pyramid.view import view_config
from pyramid.renderers import render_to_response
from pyramid.httpexceptions import HTTPFound, HTTPNotFound, HTTPInternalServerError, HTTPMovedPermanently
from src.sgd.frontend import config
from src.sgd.frontend.yeastgenome.views.cms_helpers import BLOG_BASE_URL, BLOG_PAGE_SIZE, add_simple_date_to_post, get_archive_years, get_meetings, get_recent_blog_posts, wp_categories
import urllib
import datetime
import json
import requests

SEARCH_URL = config.backend_url + '/get_search_results'
TEMPLATE_ROOT = 'src:sgd/frontend/yeastgenome/static/templates/'

@view_config(route_name='healthcheck')
def healthcheck(request):
    return Response(body='sgd ok', content_type='text/plain', charset='utf-8')

@view_config(route_name='redirect_no_overview')
@view_config(route_name='redirect_no_overview_long')
def redirect_no_overview(request):
    new_url = request.path.replace('/overview', '')
    return HTTPMovedPermanently(get_https_url(new_url, request))

@view_config(context=HTTPNotFound)
def not_found(request):
    request.response.status = 404
    return render_to_response(TEMPLATE_ROOT + 'lost.jinja2', {}, request=request)

@view_config(context=HTTPInternalServerError)
def error(self, request):
    request.response.status = 500
    return render_to_response(TEMPLATE_ROOT + 'error.jinja2', {}, request=request)

@view_config(route_name='blast_fungal')
def blast_fungal(request):
    return render_to_response(TEMPLATE_ROOT + 'blast_fungal.jinja2', {}, request=request)

@view_config(route_name='blog_archive')
@view_config(route_name='blog_category')
@view_config(route_name='blog_tag')
@view_config(route_name='blog_index')
def blog_list(request):
    url_params = request.matchdict
    page = request.params.get('page')
    if page is None:
        page = 1
    else:
        page = int(page)
    next_url = request.path + '?page=' + str(page + 1)
    offset = str((page - 1) * BLOG_PAGE_SIZE)
    offset_expression = 'offset=' + offset + 'number=' + str(BLOG_PAGE_SIZE)
    if url_params.has_key('category'):
        url_suffix = '?category=' + url_params['category'] + '&' + offset_expression
    elif url_params.has_key('tag'):
        url_suffix = '?tag=' + url_params['tag'] + '&' + offset_expression
    elif url_params.has_key('year'):
        year = url_params['year']
        start_date = year + '-01-01'
        end_date = year + '-12-31'
        url_suffix = '?before=' + end_date + '&after=' + start_date
    else:
        url_suffix = '?' + offset_expression
    wp_url = BLOG_BASE_URL + url_suffix
    response = requests.get(wp_url)
    posts = json.loads(response.text)['posts']
    for post in posts:
        post = add_simple_date_to_post(post)
    return render_to_response(TEMPLATE_ROOT + 'blog_list.jinja2', { 'posts': posts, 'categories': wp_categories, 'next_url': next_url, 'years': get_archive_years() }, request=request)

@view_config(route_name='blog_post')
def blog_post(self, request):
    slug = request.matchdict['slug']
    wp_url = BLOG_BASE_URL + '/slug:' + slug
    response = requests.get(wp_url)
    if response.status_code == 404:
        return not_found(request)
    post = add_simple_date_to_post(json.loads(response.text))
    return render_to_response(TEMPLATE_ROOT + 'blog_post.jinja2', { 'post': post, 'categories': wp_categories, 'years': get_archive_years() }, request=request)

@view_config(route_name='blast_sgd')
def blast_sgd(request):
    return render_to_response(TEMPLATE_ROOT + 'blast_sgd.jinja2', {}, request=request)

@view_config(route_name=patmatch')
def patmatch(request):
    return render_to_response(TEMPLATE_ROOT + 'patmatch.jinja2', {}, request=request)

@view_config(route_name='colleague_show')
def colleague_show(request):
    return render_to_response(TEMPLATE_ROOT + 'misc.jinja2', {}, request=request)

@view_config(route_name='downloads')
def downloads(request):
    return render_to_response(TEMPLATE_ROOT + 'downloads.jinja2', {}, request=request)

@view_config(route_name='new_colleague')
def new_colleague(request):
    return render_to_response(TEMPLATE_ROOT + 'new_colleague.jinja2', {}, request=request)

@view_config(route_name='interaction_search')
def interaction_search(request):
    return render_to_response(TEMPLATE_ROOT + 'interaction_search.jinja2', {}, request=request)

@view_config(route_name='download_list')
def download_list(request):
    date = datetime.datetime.now().strftime("%m/%d/%Y")
    query = request.params.get('query')
    url = request.params.get('url')
    description = '!Search results for "' + query + '"\n!Date: ' + date + '\n' + "!From: Saccharomyces Genome Database (SGD) \n!URL: http://yeastgenome.org" + url +  "\n!Contact Email: sgd-helpdesk@lists.stanford.edu \n!Funding: NHGRI at US NIH, grant number 5-U41-HG001315 \n!"
    response_text = description + '\n\n'
    loci_list = json.loads(request.params.get('bioent_ids'))
    for locus_name in loci_list:
        response_text += (locus_name + '\n')
    return Response(body=response_text, content_type='text/plain', charset='utf-8', content_disposition='attachment; filename=search_results.txt')

@view_config(route_name='references_this_week')
def references_this_week(request):
    page = {}
    return render_to_response(TEMPLATE_ROOT + 'references_this_week.jinja2', page, request=request)

@view_config(route_name='reference')
def reference(request):
    ref_id = request.matchdict['identifier']
    ref_obj = get_obj(ref_id, 'reference')
    if ref_obj is None:
        return not_found(request)
    return render_to_response(TEMPLATE_ROOT + 'reference.jinja2', ref_obj, request=request)

@view_config(route_name='phenotype')
def phenotype(request):
    pheno_id = request.matchdict['identifier']
    pheno_obj = get_obj(pheno_id, 'phenotype')
    if pheno_obj is None:
        return not_found(request)
    return render_to_response(TEMPLATE_ROOT + 'phenotype.jinja2', pheno_obj, request=request)

# If is_quick, try to redirect to gene page.  If not, or no suitable response, then just show results in script tag and let client js do the rest.
@view_config(route_name='search')
def search(request):
    # get limit, default to 25
    limit = '25' if request.params.get('page_size') is None else request.params.get('page_size')
    # get search results
    search_url = SEARCH_URL + '?' + request.query_string + '&limit=' + limit
    json_results = requests.get(search_url).text
    # if param is_quick = true, try to do some redirecting
    if request.params.get('is_quick') == 'true':
        query = request.params.get('q', '').lower()
        parsed_results = json.loads(json_results)['results']
        # if query ends in "p", try to search for protein page by executing a second search without the "p"
        if query.endswith('p'):
            temp_query = query[:-1]
            temp_query_url = SEARCH_URL + '?q=' + temp_query
            temp_json_results = requests.get(temp_query_url).text
            temp_parsed_results = json.loads(temp_json_results)['results']
            redirect_url = get_redirect_url_from_results(temp_parsed_results)
            if redirect_url:
                protein_url = redirect_url = redirect_url + '/protein'
                return HTTPFound(get_https_url(protein_url, request))
        # no protein search or no protein page redirect applicable
        redirect_url  = get_redirect_url_from_results(parsed_results)
        if redirect_url:
           return HTTPFound(get_https_url(redirect_url, request))
    # if wrapped, or page > 0, just make bootstrapped results None to avoid pagination logic in python and fetch on client
    page = 0 if request.params.get('page') is None else int(request.params.get('page'))
    if request.params.get('wrapResults') == 'true' or page > 0:
        json_results = 'false'
    # otherwise, render results page and put results in script tag
    return render_to_response(TEMPLATE_ROOT + 'search.jinja2', { 'bootstrapped_search_results_json': json_results }, request=request)

@view_config(route_name='snapshot')
def snapshot(request):
    return render_to_response(TEMPLATE_ROOT + 'snapshot.jinja2', {}, request=request)

@view_config(route_name='style_guide')
def style_guide(request):
    return render_to_response(TEMPLATE_ROOT + 'style_guide.jinja2', {}, request=request)

@view_config(route_name='suggestion')
def suggestion(request):
    return render_to_response(TEMPLATE_ROOT + 'suggestion.jinja2', {}, request=request)

@view_config(route_name='variant_viewer')
def variant_viewer(request):
    return render_to_response(TEMPLATE_ROOT + 'variant_viewer.jinja2', {}, request=request)

@view_config(route_name='home')
def home(request):
    blog_posts = get_recent_blog_posts()
    meetings = get_meetings()
    return render_to_response(TEMPLATE_ROOT + 'homepage.jinja2', { 'meetings': meetings, 'blog_posts': blog_posts }, request=request)

# # example
# @view_config(route_name='example')
# def example(request):
#     return render_to_response(TEMPLATE_ROOT + 'example.jinja2', {}, request=request)

def get_obj(identifier, obj_type):
    backend_url = config.backend_url + '/' + obj_type + '/' + identifier
    backend_response = requests.get(backend_url)
    if backend_response.status_code != 200:
        return None
    obj = json.loads(backend_response.text)
    return {
        obj_type: obj,
        obj_type + '_js': json.dumps(obj)
    }

# helper method that goes through responses, and returns a redirect URL if is_quick is true for just 1, otherwise returns false
def get_redirect_url_from_results(results):
    quick_results = [x for x in results if x.get('is_quick')]
    if len(quick_results) == 1:
        return quick_results[0]['href']
    return False

# needed to force https redirects on reverse proxy LB
def get_https_url(url, request):
    host = request.host_url.replace('http', 'https').replace(':8080', '')
    return host + url
<commit_msg>fixed the typo<commit_after># some logic (NOT all) has been moved to views to be more 'pyramid-y'
from pyramid.response import Response
from pyramid.view import view_config
from pyramid.renderers import render_to_response
from pyramid.httpexceptions import HTTPFound, HTTPNotFound, HTTPInternalServerError, HTTPMovedPermanently
from src.sgd.frontend import config
from src.sgd.frontend.yeastgenome.views.cms_helpers import BLOG_BASE_URL, BLOG_PAGE_SIZE, add_simple_date_to_post, get_archive_years, get_meetings, get_recent_blog_posts, wp_categories
import urllib
import datetime
import json
import requests

SEARCH_URL = config.backend_url + '/get_search_results'
TEMPLATE_ROOT = 'src:sgd/frontend/yeastgenome/static/templates/'

@view_config(route_name='healthcheck')
def healthcheck(request):
    return Response(body='sgd ok', content_type='text/plain', charset='utf-8')

@view_config(route_name='redirect_no_overview')
@view_config(route_name='redirect_no_overview_long')
def redirect_no_overview(request):
    new_url = request.path.replace('/overview', '')
    return HTTPMovedPermanently(get_https_url(new_url, request))

@view_config(context=HTTPNotFound)
def not_found(request):
    request.response.status = 404
    return render_to_response(TEMPLATE_ROOT + 'lost.jinja2', {}, request=request)

@view_config(context=HTTPInternalServerError)
def error(self, request):
    request.response.status = 500
    return render_to_response(TEMPLATE_ROOT + 'error.jinja2', {}, request=request)

@view_config(route_name='blast_fungal')
def blast_fungal(request):
    return render_to_response(TEMPLATE_ROOT + 'blast_fungal.jinja2', {}, request=request)

@view_config(route_name='blog_archive')
@view_config(route_name='blog_category')
@view_config(route_name='blog_tag')
@view_config(route_name='blog_index')
def blog_list(request):
    url_params = request.matchdict
    page = request.params.get('page')
    if page is None:
        page = 1
    else:
        page = int(page)
    next_url = request.path + '?page=' + str(page + 1)
    offset = str((page - 1) * BLOG_PAGE_SIZE)
    offset_expression = 'offset=' + offset + 'number=' + str(BLOG_PAGE_SIZE)
    if url_params.has_key('category'):
        url_suffix = '?category=' + url_params['category'] + '&' + offset_expression
    elif url_params.has_key('tag'):
        url_suffix = '?tag=' + url_params['tag'] + '&' + offset_expression
    elif url_params.has_key('year'):
        year = url_params['year']
        start_date = year + '-01-01'
        end_date = year + '-12-31'
        url_suffix = '?before=' + end_date + '&after=' + start_date
    else:
        url_suffix = '?' + offset_expression
    wp_url = BLOG_BASE_URL + url_suffix
    response = requests.get(wp_url)
    posts = json.loads(response.text)['posts']
    for post in posts:
        post = add_simple_date_to_post(post)
    return render_to_response(TEMPLATE_ROOT + 'blog_list.jinja2', { 'posts': posts, 'categories': wp_categories, 'next_url': next_url, 'years': get_archive_years() }, request=request)

@view_config(route_name='blog_post')
def blog_post(self, request):
    slug = request.matchdict['slug']
    wp_url = BLOG_BASE_URL + '/slug:' + slug
    response = requests.get(wp_url)
    if response.status_code == 404:
        return not_found(request)
    post = add_simple_date_to_post(json.loads(response.text))
    return render_to_response(TEMPLATE_ROOT + 'blog_post.jinja2', { 'post': post, 'categories': wp_categories, 'years': get_archive_years() }, request=request)

@view_config(route_name='blast_sgd')
def blast_sgd(request):
    return render_to_response(TEMPLATE_ROOT + 'blast_sgd.jinja2', {}, request=request)

@view_config(route_name='patmatch')
def patmatch(request):
    return render_to_response(TEMPLATE_ROOT + 'patmatch.jinja2', {}, request=request)

@view_config(route_name='colleague_show')
def colleague_show(request):
    return render_to_response(TEMPLATE_ROOT + 'misc.jinja2', {}, request=request)

@view_config(route_name='downloads')
def downloads(request):
    return render_to_response(TEMPLATE_ROOT + 'downloads.jinja2', {}, request=request)

@view_config(route_name='new_colleague')
def new_colleague(request):
    return render_to_response(TEMPLATE_ROOT + 'new_colleague.jinja2', {}, request=request)

@view_config(route_name='interaction_search')
def interaction_search(request):
    return render_to_response(TEMPLATE_ROOT + 'interaction_search.jinja2', {}, request=request)

@view_config(route_name='download_list')
def download_list(request):
    date = datetime.datetime.now().strftime("%m/%d/%Y")
    query = request.params.get('query')
    url = request.params.get('url')
    description = '!Search results for "' + query + '"\n!Date: ' + date + '\n' + "!From: Saccharomyces Genome Database (SGD) \n!URL: http://yeastgenome.org" + url +  "\n!Contact Email: sgd-helpdesk@lists.stanford.edu \n!Funding: NHGRI at US NIH, grant number 5-U41-HG001315 \n!"
    response_text = description + '\n\n'
    loci_list = json.loads(request.params.get('bioent_ids'))
    for locus_name in loci_list:
        response_text += (locus_name + '\n')
    return Response(body=response_text, content_type='text/plain', charset='utf-8', content_disposition='attachment; filename=search_results.txt')

@view_config(route_name='references_this_week')
def references_this_week(request):
    page = {}
    return render_to_response(TEMPLATE_ROOT + 'references_this_week.jinja2', page, request=request)

@view_config(route_name='reference')
def reference(request):
    ref_id = request.matchdict['identifier']
    ref_obj = get_obj(ref_id, 'reference')
    if ref_obj is None:
        return not_found(request)
    return render_to_response(TEMPLATE_ROOT + 'reference.jinja2', ref_obj, request=request)

@view_config(route_name='phenotype')
def phenotype(request):
    pheno_id = request.matchdict['identifier']
    pheno_obj = get_obj(pheno_id, 'phenotype')
    if pheno_obj is None:
        return not_found(request)
    return render_to_response(TEMPLATE_ROOT + 'phenotype.jinja2', pheno_obj, request=request)

# If is_quick, try to redirect to gene page.  If not, or no suitable response, then just show results in script tag and let client js do the rest.
@view_config(route_name='search')
def search(request):
    # get limit, default to 25
    limit = '25' if request.params.get('page_size') is None else request.params.get('page_size')
    # get search results
    search_url = SEARCH_URL + '?' + request.query_string + '&limit=' + limit
    json_results = requests.get(search_url).text
    # if param is_quick = true, try to do some redirecting
    if request.params.get('is_quick') == 'true':
        query = request.params.get('q', '').lower()
        parsed_results = json.loads(json_results)['results']
        # if query ends in "p", try to search for protein page by executing a second search without the "p"
        if query.endswith('p'):
            temp_query = query[:-1]
            temp_query_url = SEARCH_URL + '?q=' + temp_query
            temp_json_results = requests.get(temp_query_url).text
            temp_parsed_results = json.loads(temp_json_results)['results']
            redirect_url = get_redirect_url_from_results(temp_parsed_results)
            if redirect_url:
                protein_url = redirect_url = redirect_url + '/protein'
                return HTTPFound(get_https_url(protein_url, request))
        # no protein search or no protein page redirect applicable
        redirect_url  = get_redirect_url_from_results(parsed_results)
        if redirect_url:
           return HTTPFound(get_https_url(redirect_url, request))
    # if wrapped, or page > 0, just make bootstrapped results None to avoid pagination logic in python and fetch on client
    page = 0 if request.params.get('page') is None else int(request.params.get('page'))
    if request.params.get('wrapResults') == 'true' or page > 0:
        json_results = 'false'
    # otherwise, render results page and put results in script tag
    return render_to_response(TEMPLATE_ROOT + 'search.jinja2', { 'bootstrapped_search_results_json': json_results }, request=request)

@view_config(route_name='snapshot')
def snapshot(request):
    return render_to_response(TEMPLATE_ROOT + 'snapshot.jinja2', {}, request=request)

@view_config(route_name='style_guide')
def style_guide(request):
    return render_to_response(TEMPLATE_ROOT + 'style_guide.jinja2', {}, request=request)

@view_config(route_name='suggestion')
def suggestion(request):
    return render_to_response(TEMPLATE_ROOT + 'suggestion.jinja2', {}, request=request)

@view_config(route_name='variant_viewer')
def variant_viewer(request):
    return render_to_response(TEMPLATE_ROOT + 'variant_viewer.jinja2', {}, request=request)

@view_config(route_name='home')
def home(request):
    blog_posts = get_recent_blog_posts()
    meetings = get_meetings()
    return render_to_response(TEMPLATE_ROOT + 'homepage.jinja2', { 'meetings': meetings, 'blog_posts': blog_posts }, request=request)

# # example
# @view_config(route_name='example')
# def example(request):
#     return render_to_response(TEMPLATE_ROOT + 'example.jinja2', {}, request=request)

def get_obj(identifier, obj_type):
    backend_url = config.backend_url + '/' + obj_type + '/' + identifier
    backend_response = requests.get(backend_url)
    if backend_response.status_code != 200:
        return None
    obj = json.loads(backend_response.text)
    return {
        obj_type: obj,
        obj_type + '_js': json.dumps(obj)
    }

# helper method that goes through responses, and returns a redirect URL if is_quick is true for just 1, otherwise returns false
def get_redirect_url_from_results(results):
    quick_results = [x for x in results if x.get('is_quick')]
    if len(quick_results) == 1:
        return quick_results[0]['href']
    return False

# needed to force https redirects on reverse proxy LB
def get_https_url(url, request):
    host = request.host_url.replace('http', 'https').replace(':8080', '')
    return host + url
<|endoftext|>


<commit_before>#!/usr/bin/python
import random
import numpy as np
from sklearn import svm

#OVERALL NOTES--created during code development so some may be irrelevant#

###I used one-class support vector machines to get cutoffs for CNVs based on the size (in bins) and integer distance of a segment
	##building two separate models
		#'Small' segments, using the immune loci recombination events
		#'Large' segments, using the sex chromosome and euploid segments
	##to compensate for the limited sample size of the immune recombination and potential skewing by outliers I am using a bootstrapping approach
		#run a large number of iterations
		#each time randomly sample with replacement from appropriate data source to build a dataset half as large as the original one
		#build a one class SVM for the large and small test groups
		#save the information about which size/intdist values pass and fail
	##combine the pass/fail results from all iterations
		#This lets me find a sort of confidence interval -- which values pass in >/= 95% of iterations

	##The overall border had issues after doing this
		#SVM seems to require a sort of symmetry where some samples are excluded in every (2D) direction
		#this makes for biologically inaccurate cases such as a size/int dist of 10 and 0.1 passes but 10 and 0.01 do not
		#so I needed to draw my own boundary

	##drawing boundaries strategy
		#minimum sizes (small or large) are the smallest number in the relevant 95% CI
			#therefore the max size of small segments is one less than the minimum of the large
				#at 25k this is only slightly larger than the largest observed immune recombination
		#find maximal int dists at each size in the relevant 95% CI
			#for small segments these are filtered down to only the pairs where the intdist is >= that at smaller sizes
			#for large segments these are filtered to start at the size with the maximal intdist and then the first size/intdist pair where no larger size has a higher intdist
				#ignores that the distribution briefly increases at the smaller end sizes
				#removes local dips like 0.2->0.19->0.2
				#taking the 'underside' of the curve, slightly more conserverative
		#the maximal size for large segments is the largest observed segment (so the size of chromosome 1)
		#the int dist at the max size is calculated by extending the slope of the nearest datapoints
		#the int dist at the small-large crossing point is the maximum of either
			#the cutoff for large if extended to the switching point using the slope of the nearest datapoints
			#the maximum passing intdist of small segments
	##cutoffs are saved as txt files for consistant future usage

###THIS CODE DOES PROVIDE SLIGHTLY DIFFERENT OUTPUTS UPON EACH USAGE DUE TO THE RANDOMIZATION COMPONENT
  ##10,000 iterations gave the most minimal variability for processing time required
  ##the variability that does occur does not appear to alter biological analysis

#perform one iteration of SVM on the input (bootstrapped) dataset#
def runIteration(iterationDict, parameterDict, xxx, yyy):
  Zdict = {}
  for i in iterationDict:
    thisModel = svm.OneClassSVM(kernel=parameterDict['kernel'], gamma=parameterDict['gamma'], nu=parameterDict['nu'])
    thisModel.fit(iterationDict[i])

    thisZ = thisModel.predict( np.c_[xxx.ravel(), yyy.ravel()] )
    Zdict[i] = thisZ.reshape(xxx.shape)

  return Zdict

#used when determining which combinations of size and integer distance are at the boundary between good and bad#
def CheckIfEdge(i1, i2, Zdata):
  edgeOptions =	[
      (i1, i2-1),
      (i1, i2+1),
      (i1-1, i2-1),
      (i1-1, i2),
      (i1-1, i2+1),
      (i1+1, i2-1),
      (i1+1, i2),
      (i1+1, i2+1)
      ]
  edges = [x for x in edgeOptions if x[0] >= 0 and x[1] >= 0]

  thisValue = Zdata[i1, i2]
  surroundingData = [Zdata[x[0], x[1]] for x in edges]
  if any(x for x in surroundingData if x < thisValue):
    return True
  else:
    return False

#the master code that performs SVM and calculates the cutoff boundary#
  #dataDict is a dictionary of structured numpy arrays
    #one array for each category of segments
  #rawOutputFilename is a path and name of a file
    #include if you would like to save the SVM Frequency output to
  #same concept for smallCutoffFile and largeCutoffFFile
    #need paths if you want to save the information to a text file

def getCNVcutoffs(dataDict, rawOutputFilename=False, smallCutoffFile=False, largeCutoffFile=False):

	###variable setup

	variables = ['bins', 'intdist']
	parameterDict = {'kernel': 'rbf', 'gamma': 10, 'nu': 0.125}
  iterations = 10000
 # numBins = 25

	maxDict = {x: max([max(dataDict[y][x]) for y in dataDict]) for x in variables}
	intdistVals = np.arange(0.00, 0.51, 0.01)
	sizeVals = np.array([0.9] + list(np.arange(1, 10001, 1)))
	xx, yy = np.meshgrid( np.log10(sizeVals), intdistVals )
	xxx = xx/np.log10(maxDict['bins'])
	yyy = yy/maxDict['intdist']

	testDataDict =	{
			'Immune': [[np.log10(x[y])/np.log10(maxDict[y]) if y == 'bins' else x[y]/maxDict[y] for y in variables] for x in dataDict['Immune']],
			'Sex Chrom': [[np.log10(x[y])/np.log10(maxDict[y]) if y == 'bins' else x[y]/maxDict[y] for y in variables] for x in dataDict['Sex Chrom']],
			'Euploid': [[np.log10(x[y])/np.log10(maxDict[y]) if y == 'bins' else x[y]/maxDict[y] for y in variables] for x in dataDict['Euploid']],
			'Putative CNV': [[np.log10(x[y])/np.log10(maxDict[y]) if y == 'bins' else x[y]/maxDict[y] for y in variables] for x in dataDict['Putative CNV']]
			}

	downsampleSizeDict = { x: len(testDataDict[x]) / 2 for x in testDataDict }

	maxSizeEup = np.log10(max(dataDict['Euploid']['bins']))

	###Run bootstrapping of fitting a one-class SVM model to large and small alterations
	allZdict = {'small': np.zeros(xxx.shape), 'large': np.zeros(xxx.shape)}

  #note: doing this in multiple iterations to avoid running out of memory
	for i in range(iterations/250):

		argList = []
		for j in range(250):
			iterationData = { x: [random.choice(testDataDict[x]) for y in range(downsampleSizeDict[x])] for x in testDataDict if x != 'Putative CNV'}

			thisDict =	{
					'small': np.array(iterationData['Immune']),
					'large': np.array( iterationData['Sex Chrom'] + iterationData['Euploid'] )
					}
			argList.append( (thisDict, parameterDict, xxx, yyy, ) )

		startArg = i * 250
		endArg = min([iterations, (i+1)*250])
		print i+1, 'round of iteration multiprocessing, will go from', startArg, 'to', endArg-1
#!#!#!#!#!#!#!#!#!#!#!#!#!#!# I NEED TO FIX THIS DAEMON THING
		results = daemon( runIteration, argList, 0.5, True, 'SVM bootstrapping round ' + str(i+1) )

		for j in results:
			allZdict = { x: allZdict[x] + j[x] for x in allZdict }
		print '\n'

	allZdictNorm = { x: (allZdict[x] / float(iterations)) for x in allZdict }

	thisShape = allZdictNorm[allZdictNorm.keys()[0]].shape
	overallZ = np.zeros(thisShape)
	for i in range(thisShape[0]):
		for j in range(thisShape[1]):
			thisMax = max([ allZdictNorm[x][i,j] for x in allZdictNorm ])
			overallZ[i,j] = thisMax

  if rawOutputFilename:
	  np.savetxt(rawOutputFilename, overallZ, delimiter='\t')

	###Interpret bootstrapping results
	cutoffDtypes = {'names': ('bins', 'intdist'), 'formats': ('int', 'float')}

	Zdict95 = {x: [index for index,value in np.ndenumerate(allZdictNorm[x]) if value >= 0.95] for x in allZdictNorm}
	Zdict95edge = {x: [y for y in Zdict95[x] if CheckIfEdge(y[0], y[1], allZdictNorm[x])] for x in Zdict95}
	Zdict95edge = { x: np.sort( np.array([ (10**xx[y[0], y[1]], yy[y[0], y[1]]) for y in Zdict95edge[x] ], dtype=cutoffDtypes), order='bins' ) for x in Zdict95 }

	maxSizePassingLym = min(Zdict95edge['large']['bins']) - 1

	filteredZedgeDict = {}
	for i in Zdict95edge:
		filterData = []
		thisData = Zdict95edge[i]
		for j in np.unique(thisData['bins']):
			useTest = True

			sizeData = thisData[thisData['bins']==j]
			sizeData = np.sort(sizeData, order='intdist')

			if i == 'small':
				if len(filterData) != 0 and sizeData['intdist'][-1] <= filterData[-1][1]:
					useTest = False
			else:
				if (len(filterData) == 0 and max(sizeData['intdist']) < max(thisData['intdist'])):
					useTest = False
				elif len(filterData) != 0 and np.round(max(sizeData['intdist']), 3) == np.round(filterData[-1][1], 3):
					useTest = False
				elif j < maxSizePassingLym:
					useTest = False

				elif len(filterData) > 0 and max(sizeData['intdist']) > filterData[-1][1]:
					useTest = False
					while max(sizeData['intdist']) > filterData[-1][1]:
						filterData.pop()

				if len(sizeData) > 0 and max(sizeData['intdist']) != sizeData['intdist'][-1]:
					print 'ERROR'

			if useTest:
				filterData.append( (j, sizeData['intdist'][-1]) )

		if i == 'small':
			filterData.insert(0, (filterData[0][0], 0.) )
			filterData.append( (maxSizePassingLym, filterData[-1][1]) )
			filterData.append( (maxSizePassingLym, 0.) )

		else:
			smallDiff = filterData[0][0] - maxSizePassingLym
			diffDown = filterData[0][0] + smallDiff
			diffEdge = [(x, abs(y[0] - diffDown)) for x,y in enumerate(filterData)]
			diffMin = min([x[1] for x in diffEdge])
			diffEdge = [filterData[x[0]] for x in diffEdge if x[1] == diffMin]
			nextIntDist = max([x[1] for x in diffEdge])
			diffIntDist = filterData[0][1] - nextIntDist
			filterDiff = filterData[0][1] + diffIntDist
			if filterDiff < filteredZedgeDict['small'][-2][1]:
				filterDiff = filteredZedgeDict['small'][-2][1]
			filterData.insert( 0, (filteredZedgeDict['small'][-2][0], filterDiff) )

			bigDiff = 10**maxSizeEup - filterData[-1][0]
			diffUp = filterData[-1][0] - bigDiff
			diffEdge = [(x, abs(y[0] - diffUp)) for x,y in enumerate(filterData)]
			diffMin = min([x[1] for x in diffEdge])
			diffEdge = [filterData[x[0]] for x in diffEdge if x[1] == diffMin]

			prevIntDist = max([x[1] for x in diffEdge])
			diffIntDist = prevIntDist - filterData[-1][1]
			filterDiff = filterData[-1][1] - diffIntDist

			filterData.append( ((10 ** maxSizeEup), filterDiff) )
			filterData.append( ((10 ** maxSizeEup), 0.) )

		filterData = np.array(filterData, dtype=cutoffDtypes)
		filteredZedgeDict[i] = filterData

	if max(filteredZedgeDict['large']['intdist']) > max(filteredZedgeDict['small']['intdist']):
		filteredZedgeDict['small']['intdist'][-2] = max(filteredZedgeDict['large']['intdist'])

	if smallCutoffFile:
    OUT = open(smallCutoffFile, 'w')
    OUT.write('Size\tIntegerDistance\n')

    smallSizeOptions = np.arange(filteredZedgeDict['small']['bins'][0], filteredZedgeDict['small']['bins'][-1]+1)
    smallThresholdDict = {}
    smallData = filteredZedgeDict['small'][1:-1]
    for i in smallSizeOptions:
      if i in smallData['bins']:
        cutoff = [x['intdist'] for x in smallData if x['bins'] == i]
        if len(cutoff) > 1:
          print 'ERROR: why are there multiple cutoffs? Fix edge assignment code', cutoff
        cutoff = cutoff[0]

      else:
        distances = [(x, i-y) for x,y in enumerate(smallData['bins'])]
        bigger = [x for x in distances if x[1] < 0][0][0]
        smaller = [x for x in distances if x[1] > 0][-1][0]

        big = smallData[bigger]
        small = smallData[smaller]

        slope = (big['intdist'] - small['intdist']) / float(big['bins'] - small['bins'])
        cutoff = ( slope * (i - small['bins']) ) + small['intdist']

      smallThresholdDict[int(i)] = np.round(cutoff, 2)
      OUT.write(str(int(i)))
      OUT.write('\t')
      OUT.write(str(np.round(cutoff, 2)))
      OUT.write('\n')

    OUT.close()

	if largeCutoffFile:
    OUT = open(largeCutoffFile, 'w')
    OUT.write('Size\tIntegerDistance\n')

    largeSizeOptions = np.arange(smallSizeOptions[-1]+1, filteredZedgeDict['large']['bins'][-1]+1)
    largeThresholdDict = {}
    largeData = filteredZedgeDict['large'][:-1]
    for i in largeSizeOptions:
      if i in largeData['bins']:
        cutoff = [x['intdist'] for x in largeData if x['bins'] == i]
        if len(cutoff) > 1:
          print 'ERROR:', cutoff
        cutoff = cutoff[0]

      else:
        distances = [(x, i-y) for x,y in enumerate(largeData['bins'])]
        bigger = [x for x in distances if x[1] < 0][0][0]
        smaller = [x for x in distances if x[1] > 0][-1][0]
        big = largeData[bigger]
        small = largeData[smaller]

        slope = (big['intdist'] - small['intdist']) / float(big['bins'] - small['bins'])
        cutoff = ( slope * (i - small['bins']) ) + small['intdist']

      largeThresholdDict[int(i)] = np.round(cutoff, 2)
      OUT.write(str(int(i)))
      OUT.write('\t')
      OUT.write(str(np.round(cutoff, 2)))
      OUT.write('\n')

    OUT.close()

<commit_msg>Fixed 'daemon' reference<commit_after>#!/usr/bin/python
import random
import numpy as np
from sklearn import svm
import multiprocessing as mp

#OVERALL NOTES--created during code development so some may be irrelevant#

###I used one-class support vector machines to get cutoffs for CNVs based on the size (in bins) and integer distance of a segment
	##building two separate models
		#'Small' segments, using the immune loci recombination events
		#'Large' segments, using the sex chromosome and euploid segments
	##to compensate for the limited sample size of the immune recombination and potential skewing by outliers I am using a bootstrapping approach
		#run a large number of iterations
		#each time randomly sample with replacement from appropriate data source to build a dataset half as large as the original one
		#build a one class SVM for the large and small test groups
		#save the information about which size/intdist values pass and fail
	##combine the pass/fail results from all iterations
		#This lets me find a sort of confidence interval -- which values pass in >/= 95% of iterations

	##The overall border had issues after doing this
		#SVM seems to require a sort of symmetry where some samples are excluded in every (2D) direction
		#this makes for biologically inaccurate cases such as a size/int dist of 10 and 0.1 passes but 10 and 0.01 do not
		#so I needed to draw my own boundary

	##drawing boundaries strategy
		#minimum sizes (small or large) are the smallest number in the relevant 95% CI
			#therefore the max size of small segments is one less than the minimum of the large
				#at 25k this is only slightly larger than the largest observed immune recombination
		#find maximal int dists at each size in the relevant 95% CI
			#for small segments these are filtered down to only the pairs where the intdist is >= that at smaller sizes
			#for large segments these are filtered to start at the size with the maximal intdist and then the first size/intdist pair where no larger size has a higher intdist
				#ignores that the distribution briefly increases at the smaller end sizes
				#removes local dips like 0.2->0.19->0.2
				#taking the 'underside' of the curve, slightly more conserverative
		#the maximal size for large segments is the largest observed segment (so the size of chromosome 1)
		#the int dist at the max size is calculated by extending the slope of the nearest datapoints
		#the int dist at the small-large crossing point is the maximum of either
			#the cutoff for large if extended to the switching point using the slope of the nearest datapoints
			#the maximum passing intdist of small segments
	##cutoffs are saved as txt files for consistant future usage

###THIS CODE DOES PROVIDE SLIGHTLY DIFFERENT OUTPUTS UPON EACH USAGE DUE TO THE RANDOMIZATION COMPONENT
  ##10,000 iterations gave the most minimal variability for processing time required
  ##the variability that does occur does not appear to alter biological analysis

#perform one iteration of SVM on the input (bootstrapped) dataset#
def runIteration(iterationDict, parameterDict, xxx, yyy):
  Zdict = {}
  for i in iterationDict:
    thisModel = svm.OneClassSVM(kernel=parameterDict['kernel'], gamma=parameterDict['gamma'], nu=parameterDict['nu'])
    thisModel.fit(iterationDict[i])

    thisZ = thisModel.predict( np.c_[xxx.ravel(), yyy.ravel()] )
    Zdict[i] = thisZ.reshape(xxx.shape)

  return Zdict

#used when determining which combinations of size and integer distance are at the boundary between good and bad#
def CheckIfEdge(i1, i2, Zdata):
  edgeOptions =	[
      (i1, i2-1),
      (i1, i2+1),
      (i1-1, i2-1),
      (i1-1, i2),
      (i1-1, i2+1),
      (i1+1, i2-1),
      (i1+1, i2),
      (i1+1, i2+1)
      ]
  edges = [x for x in edgeOptions if x[0] >= 0 and x[1] >= 0]

  thisValue = Zdata[i1, i2]
  surroundingData = [Zdata[x[0], x[1]] for x in edges]
  if any(x for x in surroundingData if x < thisValue):
    return True
  else:
    return False

#the master code that performs SVM and calculates the cutoff boundary#
  #dataDict is a dictionary of structured numpy arrays
    #one array for each category of segments
  #rawOutputFilename is a path and name of a file
    #include if you would like to save the SVM Frequency output to
  #same concept for smallCutoffFile and largeCutoffFFile
    #need paths if you want to save the information to a text file

def getCNVcutoffs(dataDict, rawOutputFilename=False, smallCutoffFile=False, largeCutoffFile=False):

	###variable setup
	variables = ['bins', 'intdist']
	parameterDict = {'kernel': 'rbf', 'gamma': 10, 'nu': 0.125}
 	iterations = 10000
	numCPU = mp.cpu_count()

	maxDict = {x: max([max(dataDict[y][x]) for y in dataDict]) for x in variables}
	intdistVals = np.arange(0.00, 0.51, 0.01)
	sizeVals = np.array([0.9] + list(np.arange(1, 10001, 1)))
	xx, yy = np.meshgrid( np.log10(sizeVals), intdistVals )
	xxx = xx/np.log10(maxDict['bins'])
	yyy = yy/maxDict['intdist']

	testDataDict =	{
			'Immune': [[np.log10(x[y])/np.log10(maxDict[y]) if y == 'bins' else x[y]/maxDict[y] for y in variables] for x in dataDict['Immune']],
			'Sex Chrom': [[np.log10(x[y])/np.log10(maxDict[y]) if y == 'bins' else x[y]/maxDict[y] for y in variables] for x in dataDict['Sex Chrom']],
			'Euploid': [[np.log10(x[y])/np.log10(maxDict[y]) if y == 'bins' else x[y]/maxDict[y] for y in variables] for x in dataDict['Euploid']],
			'Putative CNV': [[np.log10(x[y])/np.log10(maxDict[y]) if y == 'bins' else x[y]/maxDict[y] for y in variables] for x in dataDict['Putative CNV']]
			}

	downsampleSizeDict = { x: len(testDataDict[x]) / 2 for x in testDataDict }

	maxSizeEup = np.log10(max(dataDict['Euploid']['bins']))

	###Run bootstrapping of fitting a one-class SVM model to large and small alterations
	allZdict = {'small': np.zeros(xxx.shape), 'large': np.zeros(xxx.shape)}

  	#note: doing this in multiple iterations to avoid running out of memory
	for i in range(iterations/250):

		argList = []
		for j in range(250):
			iterationData = { x: [random.choice(testDataDict[x]) for y in range(downsampleSizeDict[x])] for x in testDataDict if x != 'Putative CNV'}

			thisDict =	{
					'small': np.array(iterationData['Immune']),
					'large': np.array( iterationData['Sex Chrom'] + iterationData['Euploid'] )
					}
			argList.append( (thisDict, parameterDict, xxx, yyy, ) )

		startArg = i * 250
		endArg = min([iterations, (i+1)*250])

		print i+1, 'round of iteration multiprocessing, will go from', startArg, 'to', endArg-1

		pool = mp.Pool(numCPU)
		processes = [ pool.apply_async(runIteration, args=x) for x in argList ]
		pool.close()

		results = len(argList) * [ [] ]

		for i,j in enumerate(processes):
			j.wait()
			results[i] = j.get()

		for j in results:
			allZdict = { x: allZdict[x] + j[x] for x in allZdict }
		print '\n'

	allZdictNorm = { x: (allZdict[x] / float(iterations)) for x in allZdict }

	thisShape = allZdictNorm[allZdictNorm.keys()[0]].shape
	overallZ = np.zeros(thisShape)
	for i in range(thisShape[0]):
		for j in range(thisShape[1]):
			thisMax = max([ allZdictNorm[x][i,j] for x in allZdictNorm ])
			overallZ[i,j] = thisMax

  if rawOutputFilename:
	  np.savetxt(rawOutputFilename, overallZ, delimiter='\t')

	###Interpret bootstrapping results
	cutoffDtypes = {'names': ('bins', 'intdist'), 'formats': ('int', 'float')}

	Zdict95 = {x: [index for index,value in np.ndenumerate(allZdictNorm[x]) if value >= 0.95] for x in allZdictNorm}
	Zdict95edge = {x: [y for y in Zdict95[x] if CheckIfEdge(y[0], y[1], allZdictNorm[x])] for x in Zdict95}
	Zdict95edge = { x: np.sort( np.array([ (10**xx[y[0], y[1]], yy[y[0], y[1]]) for y in Zdict95edge[x] ], dtype=cutoffDtypes), order='bins' ) for x in Zdict95 }

	maxSizePassingLym = min(Zdict95edge['large']['bins']) - 1

	filteredZedgeDict = {}
	for i in Zdict95edge:
		filterData = []
		thisData = Zdict95edge[i]
		for j in np.unique(thisData['bins']):
			useTest = True

			sizeData = thisData[thisData['bins']==j]
			sizeData = np.sort(sizeData, order='intdist')

			if i == 'small':
				if len(filterData) != 0 and sizeData['intdist'][-1] <= filterData[-1][1]:
					useTest = False
			else:
				if (len(filterData) == 0 and max(sizeData['intdist']) < max(thisData['intdist'])):
					useTest = False
				elif len(filterData) != 0 and np.round(max(sizeData['intdist']), 3) == np.round(filterData[-1][1], 3):
					useTest = False
				elif j < maxSizePassingLym:
					useTest = False

				elif len(filterData) > 0 and max(sizeData['intdist']) > filterData[-1][1]:
					useTest = False
					while max(sizeData['intdist']) > filterData[-1][1]:
						filterData.pop()

				if len(sizeData) > 0 and max(sizeData['intdist']) != sizeData['intdist'][-1]:
					print 'ERROR'

			if useTest:
				filterData.append( (j, sizeData['intdist'][-1]) )

		if i == 'small':
			filterData.insert(0, (filterData[0][0], 0.) )
			filterData.append( (maxSizePassingLym, filterData[-1][1]) )
			filterData.append( (maxSizePassingLym, 0.) )

		else:
			smallDiff = filterData[0][0] - maxSizePassingLym
			diffDown = filterData[0][0] + smallDiff
			diffEdge = [(x, abs(y[0] - diffDown)) for x,y in enumerate(filterData)]
			diffMin = min([x[1] for x in diffEdge])
			diffEdge = [filterData[x[0]] for x in diffEdge if x[1] == diffMin]
			nextIntDist = max([x[1] for x in diffEdge])
			diffIntDist = filterData[0][1] - nextIntDist
			filterDiff = filterData[0][1] + diffIntDist
			if filterDiff < filteredZedgeDict['small'][-2][1]:
				filterDiff = filteredZedgeDict['small'][-2][1]
			filterData.insert( 0, (filteredZedgeDict['small'][-2][0], filterDiff) )

			bigDiff = 10**maxSizeEup - filterData[-1][0]
			diffUp = filterData[-1][0] - bigDiff
			diffEdge = [(x, abs(y[0] - diffUp)) for x,y in enumerate(filterData)]
			diffMin = min([x[1] for x in diffEdge])
			diffEdge = [filterData[x[0]] for x in diffEdge if x[1] == diffMin]

			prevIntDist = max([x[1] for x in diffEdge])
			diffIntDist = prevIntDist - filterData[-1][1]
			filterDiff = filterData[-1][1] - diffIntDist

			filterData.append( ((10 ** maxSizeEup), filterDiff) )
			filterData.append( ((10 ** maxSizeEup), 0.) )

		filterData = np.array(filterData, dtype=cutoffDtypes)
		filteredZedgeDict[i] = filterData

	if max(filteredZedgeDict['large']['intdist']) > max(filteredZedgeDict['small']['intdist']):
		filteredZedgeDict['small']['intdist'][-2] = max(filteredZedgeDict['large']['intdist'])

	if smallCutoffFile:
    OUT = open(smallCutoffFile, 'w')
    OUT.write('Size\tIntegerDistance\n')

    smallSizeOptions = np.arange(filteredZedgeDict['small']['bins'][0], filteredZedgeDict['small']['bins'][-1]+1)
    smallThresholdDict = {}
    smallData = filteredZedgeDict['small'][1:-1]
    for i in smallSizeOptions:
      if i in smallData['bins']:
        cutoff = [x['intdist'] for x in smallData if x['bins'] == i]
        if len(cutoff) > 1:
          print 'ERROR: why are there multiple cutoffs? Fix edge assignment code', cutoff
        cutoff = cutoff[0]

      else:
        distances = [(x, i-y) for x,y in enumerate(smallData['bins'])]
        bigger = [x for x in distances if x[1] < 0][0][0]
        smaller = [x for x in distances if x[1] > 0][-1][0]

        big = smallData[bigger]
        small = smallData[smaller]

        slope = (big['intdist'] - small['intdist']) / float(big['bins'] - small['bins'])
        cutoff = ( slope * (i - small['bins']) ) + small['intdist']

      smallThresholdDict[int(i)] = np.round(cutoff, 2)
      OUT.write(str(int(i)))
      OUT.write('\t')
      OUT.write(str(np.round(cutoff, 2)))
      OUT.write('\n')

    OUT.close()

	if largeCutoffFile:
    OUT = open(largeCutoffFile, 'w')
    OUT.write('Size\tIntegerDistance\n')

    largeSizeOptions = np.arange(smallSizeOptions[-1]+1, filteredZedgeDict['large']['bins'][-1]+1)
    largeThresholdDict = {}
    largeData = filteredZedgeDict['large'][:-1]
    for i in largeSizeOptions:
      if i in largeData['bins']:
        cutoff = [x['intdist'] for x in largeData if x['bins'] == i]
        if len(cutoff) > 1:
          print 'ERROR:', cutoff
        cutoff = cutoff[0]

      else:
        distances = [(x, i-y) for x,y in enumerate(largeData['bins'])]
        bigger = [x for x in distances if x[1] < 0][0][0]
        smaller = [x for x in distances if x[1] > 0][-1][0]
        big = largeData[bigger]
        small = largeData[smaller]

        slope = (big['intdist'] - small['intdist']) / float(big['bins'] - small['bins'])
        cutoff = ( slope * (i - small['bins']) ) + small['intdist']

      largeThresholdDict[int(i)] = np.round(cutoff, 2)
      OUT.write(str(int(i)))
      OUT.write('\t')
      OUT.write(str(np.round(cutoff, 2)))
      OUT.write('\n')

    OUT.close()

<|endoftext|>