<commit_before>__author__ = 'nicole'

import os
from datetime import datetime
from stat import *
from zipfile import ZipFile
import re

from rdflib.namespace import FOAF, DC, RDFS

from dipper import curie_map
from dipper.sources.Source import Source
from dipper.models.InteractionAssoc import InteractionAssoc
from dipper.models.Dataset import Dataset
from dipper.utils.CurieUtil import CurieUtil
from dipper.utils.GraphUtils import GraphUtils

core_bindings = {'dc': DC, 'foaf': FOAF, 'rdfs': RDFS}

class BioGrid(Source):
    '''
    Biogrid interaction data
    '''

    files = {
        'interactions' : {'file': 'interactions.mitab.zip',
                           'url' : 'http://thebiogrid.org/downloads/archives/Latest%20Release/BIOGRID-ALL-LATEST.mitab.zip'
        },
        'identifiers' : {'file' : 'identifiers.tab.zip',
                          'url' : 'http://thebiogrid.org/downloads/archives/Latest%20Release/BIOGRID-IDENTIFIERS-LATEST.tab.zip'
        }
    }

    def __init__(self, tax_ids=None):
        super().__init__('biogrid')

        self.tax_ids = tax_ids
        self.load_bindings()

        self.dataset = Dataset('biogrid', 'The BioGrid', 'http://thebiogrid.org/')

        # Defaults
        if self.tax_ids is None:
            self.tax_ids = [9606, 10090]

        #data-source specific warnings (will be removed when issues are cleared)
        print("WARN: several MI experimental codes do not exactly map to ECO; using approximations.")
        return

    def fetch(self, is_dl_forced):
        '''
        :return: None
        '''

        for f in self.files.keys():
            file = self.files.get(f)
            self.fetch_from_url(file['url'],
                                ('/').join((self.rawdir,file['file'])),
                                is_dl_forced)
            self.dataset.setFileAccessUrl(file['url'])

            st = os.stat(('/').join((self.rawdir,file['file'])))

        filedate=datetime.utcfromtimestamp(st[ST_CTIME]).strftime("%Y-%m-%d")

        #the version number is encoded in the filename in the zip.
        #for example, the interactions file may unzip to BIOGRID-ALL-3.2.119.mitab.txt,
        #where the version number is 3.2.119
        f=('/').join((self.rawdir,self.files['interactions']['file']))
        with ZipFile(f, 'r') as myzip:
            flist=myzip.namelist()
            #assume that the first entry is the item
            fname=flist[0]
            #get the version from the filename
            version = re.match('BIOGRID-ALL-(\d+\.\d+\.\d+)\.mitab.txt',fname)
        myzip.close()

        self.dataset.setVersion(filedate,str(version.groups()[0]))

        return

    def parse(self,limit=None):
        '''
        abstract method to parse all data from an external resource, that was fetched in
        fetch()
        this should be overridden by subclasses
        :return: None
        '''

        #TODO make each of these items an option... we may want to process them separately
        self._get_interactions(limit)

        self._get_identifiers(limit)
        self.load_bindings()

        print("Loaded", len(self.graph), "nodes")

        return

    def _get_interactions(self,limit):
        print("INFO: getting interactions")
        line_counter = 0
        f=('/').join((self.rawdir,self.files['interactions']['file']))
        myzip = ZipFile(f,'r')
        #assume that the first entry is the item
        fname=myzip.namelist()[0]
        matchcounter=0

        with myzip.open(fname,'r') as csvfile:
            for line in csvfile:
                #skip comment lines
                if (re.match('^#',line.decode())):
                    print("Skipping header line")
                    continue
                line_counter += 1
                line=line.decode().strip()
                #print(line)
                (interactor_a, interactor_b, alt_ids_a, alt_ids_b,aliases_a, aliases_b,
                 detection_method, pub_author,pub_id,
                 taxid_a, taxid_b, interaction_type,
                 source_db, interaction_id, confidence_val) = line.split('\t')

                #TODO remove these filters, or parameterize them
                #uncomment the following codeblock if you want to filter based on taxid
                #taxids = [9606,10090,10116,7227,7955,6239,8355]  #our favorite animals
                #taxids = [9606] #human
                if (int(re.sub('taxid:','', taxid_a.rstrip())) not in self.tax_ids
                        or int(re.sub('taxid:','', taxid_b.rstrip())) not in self.tax_ids):
                    continue
                else:
                    matchcounter += 1

                #TODO proper testing/catching of these search/match methods
                #get the actual gene ids, typically formated like: gene/locuslink:351|BIOGRID:106848
                gene_a='NCBIGene:'+re.search('locuslink\:(\d+)\|',interactor_a).groups()[0]
                gene_b='NCBIGene:'+re.search('locuslink\:(\d+)\|',interactor_b).groups()[0]

                #get the interaction type
                #psi-mi:"MI:0407"(direct interaction)
                int_type=re.search('MI:\d+',interaction_type).group()
                rel=self._map_MI_to_RO(int_type)

                #scrub pubmed-->PMID prefix
                pub_id = re.sub('pubmed','PMID',pub_id)
                #remove bogus whitespace
                pub_id = pub_id.strip()

                #get the method, and convert to evidence code
                det_code=re.search('MI:\d+',detection_method).group()
                evidence=self._map_MI_to_ECO(det_code)

                #note that the interaction_id is some kind of internal biogrid identifier that does not
                #map to a public URI.  we will construct a monarch identifier from this
                assoc_id = self.make_id(interaction_id)

                assoc = InteractionAssoc(assoc_id,gene_a,gene_b,pub_id,evidence)
                assoc.setRelationship(rel)
                assoc.loadAllProperties(self.graph)    #FIXME - this seems terribly inefficient
                assoc.addInteractionAssociationToGraph(self.graph)
                if (limit is not None and line_counter > limit):
                    break

        myzip.close()
        #print("INFO: found",str(matchcounter),"matching rows")  #for testing

        return

    def _get_identifiers(self,limit):
        '''
        This will process the id mapping file provided by Biogrid.
        The file has a very large header, which we scan past, then pull the identifiers, and make
        equivalence axioms
        :param limit:
        :return:
        '''
        print("INFO: getting identifier mapping")
        line_counter = 0
        f=('/').join((self.rawdir,self.files['identifiers']['file']))
        myzip = ZipFile(f,'r')
        #assume that the first entry is the item
        fname=myzip.namelist()[0]
        matchcounter=0
        foundheader=False
        with myzip.open(fname,'r') as csvfile:
            for line in csvfile:
                #skip header lines
                if (not foundheader):
                    if (re.match('BIOGRID_ID',line.decode())):
                        foundheader=True
                    continue

                line=line.decode().strip()
                #BIOGRID_ID	IDENTIFIER_VALUE	IDENTIFIER_TYPE	ORGANISM_OFFICIAL_NAME
                #1	814566	ENTREZ_GENE	Arabidopsis thaliana
                (biogrid_num,id_num,id_type,organism_label) = line.split('\t')

                #for each one of these, create the node and add equivalent classes
                cu = CurieUtil(curie_map.get())
                gu = GraphUtils(curie_map.get())
                g = self.graph
                biogrid_id='BIOGRID:'+biogrid_num
                prefix = self._map_idtype_to_prefix(id_type)

                #geneidtypefilters='NCBIGene,OMIM,MGI,FlyBase,ZFIN,MGI,HGNC,WormBase,XenBase,ENSEMBL,miRBase'.split(',')
                #TODO make these filters available as commandline options
                geneidtypefilters='NCBIGene,MGI,ENSEMBL'.split(',')
                #speciesfilters='Homo sapiens,Mus musculus,Drosophila melanogaster,Danio rerio,Caenorhabditis elegans,Xenopus laevis'.split(',')
                speciesfilters='Homo sapiens,Mus musculus'.split(',')
                #proteinidtypefilters='HPRD,Swiss-Prot,NCBIProtein'
                if ((speciesfilters is not None) and (organism_label.strip() in speciesfilters)):
                    line_counter += 1
                    if ((geneidtypefilters is not None) and (prefix in geneidtypefilters)):
                        mapped_id=(':').join((prefix,id_num))
                        gu.addEquivalentClass(g,biogrid_id,mapped_id)
                    elif (id_type == 'OFFICIAL_SYMBOL'):  #this symbol will only get attached to the biogrid class
                        gu.addClassToGraph(g,biogrid_id,id_num)
                    #elif (id_type == 'SYNONYM'):
                    #    gu.addSynonym(g,biogrid_id,id_num)  #FIXME - i am not sure these are synonyms, altids?

                if (limit is not None and line_counter > limit):
                    break

        return

    def _map_MI_to_RO(self,mi_id):
        rel = InteractionAssoc(None,None,None,None,None).get_properties()
        mi_ro_map = {
            'MI:0403' : rel['colocalizes_with'], #colocalization
            'MI:0407' : rel['interacts_with'], #direct interaction
            'MI:0794' : rel['genetically_interacts_with'], #synthetic genetic interaction defined by inequality
            'MI:0796' : rel['genetically_interacts_with'], #suppressive genetic interaction defined by inequality
            'MI:0799' : rel['genetically_interacts_with'], #additive genetic interaction defined by inequality
            'MI:0914' : rel['interacts_with'], #association
            'MI:0915' : rel['interacts_with'] #physical association
        }

        ro_id = rel['interacts_with']  #default
        if (mi_id in mi_ro_map):
            ro_id = mi_ro_map.get(mi_id)

        return ro_id

    def _map_MI_to_ECO(self,mi_id):
        eco_id = 'ECO:0000006' #default to experimental evidence
        mi_to_eco_map = {
            'MI:0018' : 'ECO:0000068', #yeast two-hybrid
            'MI:0004' : 'ECO:0000079', #affinity chromatography
            'MI:0047' : 'ECO:0000076', #far western blotting
            'MI:0055' : 'ECO:0000021', #should be FRET, but using physical_interaction FIXME
            'MI:0090' : 'ECO:0000012', #desired: protein complementation, using: functional complementation
            'MI:0096' : 'ECO:0000085', #desired: pull down, using: immunoprecipitation
            'MI:0114' : 'ECO:0000324', #desired: x-ray crystallography, using: imaging assay
            'MI:0254' : 'ECO:0000011', #desired: genetic interference, using: genetic interaction evidence
            'MI:0401' : 'ECO:0000172', #desired: biochemical, using: biochemical trait evidence
            'MI:0415' : 'ECO:0000005', #desired: enzymatic study, using: enzyme assay evidence
            'MI:0428' : 'ECO:0000324', #imaging
            'MI:0686' : 'ECO:0000006', #desired: unspecified, using: experimental evidence
            'MI:1313' : 'ECO:0000006'  #None?
        }
        if (mi_id in mi_to_eco_map):
            eco_id = mi_to_eco_map.get(mi_id)
        else:
            print("WARN: unmapped code",mi_id, ". Defaulting to experimental_evidence")

        return eco_id

    def _map_idtype_to_prefix(self,idtype):
        prefix = idtype
        idtype_to_prefix_map = {
            'XENBASE' : 'XenBase',
            'TREMBL' : 'TrEMBL',
            'MGI' : 'MGI',
            'REFSEQ_DNA_ACCESSION': 'RefSeqNA',
            'MAIZEGDB' : 'MaizeGDB',
            'BEEBASE' : 'BeeBase',
            'ENSEMBL' : 'ENSEMBL',
            'TAIR' : 'TAIR',
            'GENBANK_DNA_GI' : 'NCBIgi',
            'CGNC' : 'CGNC',
            'RGD' : 'RGD',
            'GENBANK_GENOMIC_DNA_GI' : 'NCBIgi',
            'SWISSPROT' : 'Swiss-Prot',
            'MIM' : 'OMIM',
            'FLYBASE' : 'FlyBase',
            'VEGA' : 'VEGA',
            'ANIMALQTLDB' : 'AQTLDB',
            'ENTREZ_GENE_ETG' : 'ETG',
            'HPRD' : 'HPRD',
            'APHIDBASE' : 'APHIDBASE',
            'GENBANK_PROTEIN_ACCESSION' : 'NCBIProtein',
            'ENTREZ_GENE' : 'NCBIGene',
            'SGD' : 'SGD',
            'GENBANK_GENOMIC_DNA_ACCESSION' : 'NCBIGenome',
            'BGD' : 'BGD',
            'WORMBASE' : 'WormBase',
            'ZFIN' : 'ZFIN',
            'DICTYBASE' : 'dictyBase',
            'ECOGENE' : 'ECOGENE',
            'BIOGRID' : 'BIOGRID',
            'GENBANK_DNA_ACCESSION' : 'NCBILocus',
            'VECTORBASE' : 'VectorBase',
            'MIRBASE' : 'miRBase',
            'IMGT/GENE-DB' : 'IGMT',
            'HGNC' : 'HGNC',
            'SYSTEMATIC_NAME' : None,
            'OFFICIAL_SYMBOL' : None,
            'REFSEQ_GENOMIC_DNA_ACCESSION' : 'NCBILocus',
            'GENBANK_PROTEIN_GI' : 'NCBIgi',
            'REFSEQ_PROTEIN_ACCESSION' : 'RefSeqProt',
            'SYNONYM' : None,
            'GRID_LEGACY' : None
        }
        if (idtype in idtype_to_prefix_map):
            prefix = idtype_to_prefix_map.get(idtype)
        else:
            print("WARN: unmapped prefix",prefix)

        return prefix
<commit_msg>refactor with test set #91<commit_after>__author__ = 'nicole'

import os
from datetime import datetime
from stat import *
from zipfile import ZipFile
import re

from rdflib.namespace import FOAF, DC, RDFS

from dipper import curie_map
from dipper import config
from dipper.sources.Source import Source
from dipper.models.InteractionAssoc import InteractionAssoc
from dipper.models.Dataset import Dataset
from dipper.utils.GraphUtils import GraphUtils

core_bindings = {'dc': DC, 'foaf': FOAF, 'rdfs': RDFS}

class BioGrid(Source):
    '''
    Biogrid interaction data
    '''

    files = {
        'interactions' : {'file': 'interactions.mitab.zip',
                           'url' : 'http://thebiogrid.org/downloads/archives/Latest%20Release/BIOGRID-ALL-LATEST.mitab.zip'
        },
        'identifiers' : {'file' : 'identifiers.tab.zip',
                          'url' : 'http://thebiogrid.org/downloads/archives/Latest%20Release/BIOGRID-IDENTIFIERS-LATEST.tab.zip'
        }
    }

    #biogrid-specific identifiers for use in subsetting identifier mapping
    biogrid_ids = [106638,107308,107506,107674,107675,108277,108506,108767,108814,108899,110308,110364,110678,111642,
                   112300,112365,112771,112898,199832,203220,247276,120150,120160,124085]

    def __init__(self, tax_ids=None):
        super().__init__('biogrid')

        self.tax_ids = tax_ids
        self.load_bindings()

        self.dataset = Dataset('biogrid', 'The BioGrid', 'http://thebiogrid.org/')

        # Defaults
        #taxids = [9606,10090,10116,7227,7955,6239,8355]  #our favorite animals
        if self.tax_ids is None:
            self.tax_ids = [9606, 10090]

        if (not (('test_ids' in config.get_config()) and ('gene' in config.get_config()['test_ids']))):
            print("WARN: not configured with gene test ids.")
        else:
            self.test_ids = config.get_config()['test_ids']['gene']

        #data-source specific warnings (will be removed when issues are cleared)
        print("WARN: several MI experimental codes do not exactly map to ECO; using approximations.")
        return

    def fetch(self, is_dl_forced):
        '''
        :return: None
        '''

        for f in self.files.keys():
            file = self.files.get(f)
            self.fetch_from_url(file['url'],
                                ('/').join((self.rawdir,file['file'])),
                                is_dl_forced)
            self.dataset.setFileAccessUrl(file['url'])

            st = os.stat(('/').join((self.rawdir,file['file'])))

        filedate=datetime.utcfromtimestamp(st[ST_CTIME]).strftime("%Y-%m-%d")

        #the version number is encoded in the filename in the zip.
        #for example, the interactions file may unzip to BIOGRID-ALL-3.2.119.mitab.txt,
        #where the version number is 3.2.119
        f=('/').join((self.rawdir,self.files['interactions']['file']))
        with ZipFile(f, 'r') as myzip:
            flist=myzip.namelist()
            #assume that the first entry is the item
            fname=flist[0]
            #get the version from the filename
            version = re.match('BIOGRID-ALL-(\d+\.\d+\.\d+)\.mitab.txt',fname)
        myzip.close()

        self.dataset.setVersion(filedate,str(version.groups()[0]))

        return

    def parse(self,limit=None):
        '''
        abstract method to parse all data from an external resource, that was fetched in
        fetch()
        this should be overridden by subclasses
        :return: None
        '''

        #TODO make each of these items an option... we may want to process them separately
        for testMode in [True,False]:
            self._get_interactions(limit,testMode)
            self._get_identifiers(limit,testMode)

        self.load_bindings()
        print("Loaded", len(self.graph), "nodes")

        return

    def _get_interactions(self,limit,testMode):
        print("INFO: getting interactions")
        line_counter = 0
        f=('/').join((self.rawdir,self.files['interactions']['file']))
        myzip = ZipFile(f,'r')
        #assume that the first entry is the item
        fname=myzip.namelist()[0]
        matchcounter=0

        with myzip.open(fname,'r') as csvfile:
            for line in csvfile:
                #skip comment lines
                if (re.match('^#',line.decode())):
                    print("Skipping header line")
                    continue
                line_counter += 1
                line=line.decode().strip()
                #print(line)
                (interactor_a, interactor_b, alt_ids_a, alt_ids_b,aliases_a, aliases_b,
                 detection_method, pub_author,pub_id,
                 taxid_a, taxid_b, interaction_type,
                 source_db, interaction_id, confidence_val) = line.split('\t')

                #get the actual gene ids, typically formated like: gene/locuslink:351|BIOGRID:106848
                gene_a_num = re.search('locuslink\:(\d+)\|',interactor_a).groups()[0]
                gene_b_num = re.search('locuslink\:(\d+)\|',interactor_b).groups()[0]

                if testMode:
                    g = self.testgraph
                    #skip any genes that don't match our test set
                    if (int(gene_a_num) not in self.test_ids or int(gene_b_num) not in self.test_ids):
                        continue
                else:
                    g = self.graph
                    #when not in test mode, filter by taxon
                    if (int(re.sub('taxid:','', taxid_a.rstrip())) not in self.tax_ids
                        or int(re.sub('taxid:','', taxid_b.rstrip())) not in self.tax_ids):
                        continue
                    else:
                        matchcounter += 1

                gene_a='NCBIGene:'+gene_a_num
                gene_b='NCBIGene:'+gene_b_num

                #get the interaction type
                #psi-mi:"MI:0407"(direct interaction)
                int_type=re.search('MI:\d+',interaction_type).group()
                rel=self._map_MI_to_RO(int_type)

                #scrub pubmed-->PMID prefix
                pub_id = re.sub('pubmed','PMID',pub_id)
                #remove bogus whitespace
                pub_id = pub_id.strip()

                #get the method, and convert to evidence code
                det_code=re.search('MI:\d+',detection_method).group()
                evidence=self._map_MI_to_ECO(det_code)

                #note that the interaction_id is some kind of internal biogrid identifier that does not
                #map to a public URI.  we will construct a monarch identifier from this
                assoc_id = self.make_id(interaction_id)

                assoc = InteractionAssoc(assoc_id,gene_a,gene_b,pub_id,evidence)
                assoc.setRelationship(rel)
                assoc.loadAllProperties(g)    #FIXME - this seems terribly inefficient
                assoc.addInteractionAssociationToGraph(g)
                if not testMode and (limit is not None and line_counter > limit):
                    break

        myzip.close()

        return

    def _get_identifiers(self,limit,testMode):
        '''
        This will process the id mapping file provided by Biogrid.
        The file has a very large header, which we scan past, then pull the identifiers, and make
        equivalence axioms
        :param limit:
        :return:
        '''
        print("INFO: getting identifier mapping")
        line_counter = 0
        f=('/').join((self.rawdir,self.files['identifiers']['file']))
        myzip = ZipFile(f,'r')
        #assume that the first entry is the item
        fname=myzip.namelist()[0]
        matchcounter=0
        foundheader=False

        gu = GraphUtils(curie_map.get())

        #TODO align this species filter with the one above
        #speciesfilters='Homo sapiens,Mus musculus,Drosophila melanogaster,Danio rerio,Caenorhabditis elegans,Xenopus laevis'.split(',')
        speciesfilters='Homo sapiens,Mus musculus'.split(',')
        with myzip.open(fname,'r') as csvfile:
            for line in csvfile:
                #skip header lines
                if (not foundheader):
                    if (re.match('BIOGRID_ID',line.decode())):
                        foundheader=True
                    continue

                line=line.decode().strip()
                #BIOGRID_ID	IDENTIFIER_VALUE	IDENTIFIER_TYPE	ORGANISM_OFFICIAL_NAME
                #1	814566	ENTREZ_GENE	Arabidopsis thaliana
                (biogrid_num,id_num,id_type,organism_label) = line.split('\t')

                if testMode:
                    g = self.testgraph
                    #skip any genes that don't match our test set
                    if (int(biogrid_num) not in self.biogrid_ids):
                        continue
                else:
                    g = self.graph

                #for each one of these, create the node and add equivalent classes
                biogrid_id='BIOGRID:'+biogrid_num
                prefix = self._map_idtype_to_prefix(id_type)

                #TODO make these filters available as commandline options
                #geneidtypefilters='NCBIGene,OMIM,MGI,FlyBase,ZFIN,MGI,HGNC,WormBase,XenBase,ENSEMBL,miRBase'.split(',')
                geneidtypefilters='NCBIGene,MGI,ENSEMBL,ZFIN,HGNC'.split(',')
                #proteinidtypefilters='HPRD,Swiss-Prot,NCBIProtein'
                if ((speciesfilters is not None) and (organism_label.strip() in speciesfilters)):
                    line_counter += 1
                    if ((geneidtypefilters is not None) and (prefix in geneidtypefilters)):
                        mapped_id=(':').join((prefix,id_num))
                        gu.addEquivalentClass(g,biogrid_id,mapped_id)
                    elif (id_type == 'OFFICIAL_SYMBOL'):  #this symbol will only get attached to the biogrid class
                        gu.addClassToGraph(g,biogrid_id,id_num)
                    #elif (id_type == 'SYNONYM'):
                    #    gu.addSynonym(g,biogrid_id,id_num)  #FIXME - i am not sure these are synonyms, altids?

                if not testMode and (limit is not None and line_counter > limit):
                    break

        myzip.close()

        return

    def _map_MI_to_RO(self,mi_id):
        rel = InteractionAssoc(None,None,None,None,None).get_properties()
        mi_ro_map = {
            'MI:0403' : rel['colocalizes_with'], #colocalization
            'MI:0407' : rel['interacts_with'], #direct interaction
            'MI:0794' : rel['genetically_interacts_with'], #synthetic genetic interaction defined by inequality
            'MI:0796' : rel['genetically_interacts_with'], #suppressive genetic interaction defined by inequality
            'MI:0799' : rel['genetically_interacts_with'], #additive genetic interaction defined by inequality
            'MI:0914' : rel['interacts_with'], #association
            'MI:0915' : rel['interacts_with'] #physical association
        }

        ro_id = rel['interacts_with']  #default
        if (mi_id in mi_ro_map):
            ro_id = mi_ro_map.get(mi_id)

        return ro_id

    def _map_MI_to_ECO(self,mi_id):
        eco_id = 'ECO:0000006' #default to experimental evidence
        mi_to_eco_map = {
            'MI:0018' : 'ECO:0000068', #yeast two-hybrid
            'MI:0004' : 'ECO:0000079', #affinity chromatography
            'MI:0047' : 'ECO:0000076', #far western blotting
            'MI:0055' : 'ECO:0000021', #should be FRET, but using physical_interaction FIXME
            'MI:0090' : 'ECO:0000012', #desired: protein complementation, using: functional complementation
            'MI:0096' : 'ECO:0000085', #desired: pull down, using: immunoprecipitation
            'MI:0114' : 'ECO:0000324', #desired: x-ray crystallography, using: imaging assay
            'MI:0254' : 'ECO:0000011', #desired: genetic interference, using: genetic interaction evidence
            'MI:0401' : 'ECO:0000172', #desired: biochemical, using: biochemical trait evidence
            'MI:0415' : 'ECO:0000005', #desired: enzymatic study, using: enzyme assay evidence
            'MI:0428' : 'ECO:0000324', #imaging
            'MI:0686' : 'ECO:0000006', #desired: unspecified, using: experimental evidence
            'MI:1313' : 'ECO:0000006'  #None?
        }
        if (mi_id in mi_to_eco_map):
            eco_id = mi_to_eco_map.get(mi_id)
        else:
            print("WARN: unmapped code",mi_id, ". Defaulting to experimental_evidence")

        return eco_id

    def _map_idtype_to_prefix(self,idtype):
        prefix = idtype
        idtype_to_prefix_map = {
            'XENBASE' : 'XenBase',
            'TREMBL' : 'TrEMBL',
            'MGI' : 'MGI',
            'REFSEQ_DNA_ACCESSION': 'RefSeqNA',
            'MAIZEGDB' : 'MaizeGDB',
            'BEEBASE' : 'BeeBase',
            'ENSEMBL' : 'ENSEMBL',
            'TAIR' : 'TAIR',
            'GENBANK_DNA_GI' : 'NCBIgi',
            'CGNC' : 'CGNC',
            'RGD' : 'RGD',
            'GENBANK_GENOMIC_DNA_GI' : 'NCBIgi',
            'SWISSPROT' : 'Swiss-Prot',
            'MIM' : 'OMIM',
            'FLYBASE' : 'FlyBase',
            'VEGA' : 'VEGA',
            'ANIMALQTLDB' : 'AQTLDB',
            'ENTREZ_GENE_ETG' : 'ETG',
            'HPRD' : 'HPRD',
            'APHIDBASE' : 'APHIDBASE',
            'GENBANK_PROTEIN_ACCESSION' : 'NCBIProtein',
            'ENTREZ_GENE' : 'NCBIGene',
            'SGD' : 'SGD',
            'GENBANK_GENOMIC_DNA_ACCESSION' : 'NCBIGenome',
            'BGD' : 'BGD',
            'WORMBASE' : 'WormBase',
            'ZFIN' : 'ZFIN',
            'DICTYBASE' : 'dictyBase',
            'ECOGENE' : 'ECOGENE',
            'BIOGRID' : 'BIOGRID',
            'GENBANK_DNA_ACCESSION' : 'NCBILocus',
            'VECTORBASE' : 'VectorBase',
            'MIRBASE' : 'miRBase',
            'IMGT/GENE-DB' : 'IGMT',
            'HGNC' : 'HGNC',
            'SYSTEMATIC_NAME' : None,
            'OFFICIAL_SYMBOL' : None,
            'REFSEQ_GENOMIC_DNA_ACCESSION' : 'NCBILocus',
            'GENBANK_PROTEIN_GI' : 'NCBIgi',
            'REFSEQ_PROTEIN_ACCESSION' : 'RefSeqProt',
            'SYNONYM' : None,
            'GRID_LEGACY' : None
        }
        if (idtype in idtype_to_prefix_map):
            prefix = idtype_to_prefix_map.get(idtype)
        else:
            print("WARN: unmapped prefix",prefix)

        return prefix
<|endoftext|>


<commit_before>import pygame

class Display:
	def __init__(self, width, height):
		self.width = width
		self.height = height
		pygame.init()
		pygame.font.init()
		self.screen = pygame.display.set_mode((self.width, self.height))
		pygame.mouse.set_visible(False)

	def __del__(self):
		print "Close PyGame"
		pygame.quit()

	def flip(self):
		pygame.display.flip()

	def misc(self):
		pass
	"""
		#self.surface = pygame.Surface((width, height))
        #self.screen.blit(surf, (320,540))
		#self.surface = pygame.Surface((width, height))

	"""
<commit_msg>Start Cleanup, NumPy Smooth<commit_after>import os, pygame

class Display:
	def __init__(self, width, height):
		self.width = width
		self.height = height
		os.environ["SDL_FBDEV"] = "/dev/fb1"
		pygame.init()
		pygame.font.init()
		self.screen = pygame.display.set_mode((self.width, self.height))
		pygame.mouse.set_visible(False)

	def __del__(self):
		print "Close PyGame"
		pygame.quit()

	def flip(self):
		pygame.display.flip()

	def misc(self):
		pass
	"""
		#self.surface = pygame.Surface((width, height))
        #self.screen.blit(surf, (320,540))
		#self.surface = pygame.Surface((width, height))

	"""
<|endoftext|>


<commit_before># Portions (c) 2014, Alexander Klimenko <alex@erix.ru>
# All rights reserved.
#
# Copyright (c) 2011, SmartFile <btimby@smartfile.com>
# All rights reserved.
#
# This file is part of DjangoDav.
#
# DjangoDav is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# DjangoDav is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with DjangoDav.  If not, see <http://www.gnu.org/licenses/>.

import datetime, time, calendar
from wsgiref.handlers import format_date_time
from django.utils.feedgenerator import rfc2822_date

try:
    from email.utils import parsedate_tz
except ImportError:
    from email.Utils import parsedate_tz
import lxml.builder as lb

# Sun, 06 Nov 1994 08:49:37 GMT  ; RFC 822, updated by RFC 1123
FORMAT_RFC_822 = '%a, %d %b %Y %H:%M:%S GMT'
# Sunday, 06-Nov-94 08:49:37 GMT ; RFC 850, obsoleted by RFC 1036
FORMAT_RFC_850 = '%A %d-%b-%y %H:%M:%S GMT'
# Sun Nov  6 08:49:37 1994       ; ANSI C's asctime() format
FORMAT_ASC = '%a %b %d %H:%M:%S %Y'

WEBDAV_NS = "DAV:"

WEBDAV_NSMAP = {'D': WEBDAV_NS}

D = lb.ElementMaker(namespace=WEBDAV_NS, nsmap=WEBDAV_NSMAP)

def get_property_tag_list(res, *names):
    props = []
    for name in names:
        tag = get_property_tag(res, name)
        if tag is None:
            continue
        props.append(tag)
    return props

def get_property_tag(res, name):
    if name == 'resourcetype':
        if res.is_collection:
            return D(name, D.collection)
        return D(name)
    try:
        if hasattr(res, name):
            return D(name, unicode(getattr(res, name)))
    except AttributeError:
        return

def safe_join(root, *paths):
    """The provided os.path.join() does not work as desired. Any path starting with /
    will simply be returned rather than actually being joined with the other elements."""
    if not root.startswith('/'):
        root = '/' + root
    for path in paths:
        while root.endswith('/'):
            root = root[:-1]
        while path.startswith('/'):
            path = path[1:]
        root += '/' + path
    return root

def url_join(base, *paths):
    """Assuming base is the scheme and host (and perhaps path) we will join the remaining
    path elements to it."""
    paths = safe_join(*paths) if paths else ""
    while base.endswith('/'):
        base = base[:-1]
    return base + paths

def ns_split(tag):
    """Splits the namespace and property name from a clark notation property name."""
    if tag.startswith("{") and "}" in tag:
        ns, name = tag.split("}", 1)
        return ns[1:-1], name
    return "", tag

def ns_join(ns, name):
    """Joins a namespace and property name into clark notation."""
    return '{%s:}%s' % (ns, name)

def rfc3339_date(dt):
    if not dt:
        return ''
    return dt.strftime('%Y-%m-%dT%H:%M:%SZ')

def rfc1123_date(dt):
    if not dt:
        return ''
    return rfc2822_date(dt)

def parse_time(timestring):
    value = None
    for fmt in (FORMAT_RFC_822, FORMAT_RFC_850, FORMAT_ASC):
        try:
            value = time.strptime(timestring, fmt)
        except:
            pass
    if value is None:
        try:
            # Sun Nov  6 08:49:37 1994 +0100      ; ANSI C's asctime() format with timezone
            value = parsedate_tz(timestring)
        except:
            pass
    if value is None:
        return
    return calendar.timegm(value)
<commit_msg>+ except value error on data parse<commit_after># Portions (c) 2014, Alexander Klimenko <alex@erix.ru>
# All rights reserved.
#
# Copyright (c) 2011, SmartFile <btimby@smartfile.com>
# All rights reserved.
#
# This file is part of DjangoDav.
#
# DjangoDav is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# DjangoDav is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with DjangoDav.  If not, see <http://www.gnu.org/licenses/>.

import datetime, time, calendar
from wsgiref.handlers import format_date_time
from django.utils.feedgenerator import rfc2822_date

try:
    from email.utils import parsedate_tz
except ImportError:
    from email.Utils import parsedate_tz
import lxml.builder as lb

# Sun, 06 Nov 1994 08:49:37 GMT  ; RFC 822, updated by RFC 1123
FORMAT_RFC_822 = '%a, %d %b %Y %H:%M:%S GMT'
# Sunday, 06-Nov-94 08:49:37 GMT ; RFC 850, obsoleted by RFC 1036
FORMAT_RFC_850 = '%A %d-%b-%y %H:%M:%S GMT'
# Sun Nov  6 08:49:37 1994       ; ANSI C's asctime() format
FORMAT_ASC = '%a %b %d %H:%M:%S %Y'

WEBDAV_NS = "DAV:"

WEBDAV_NSMAP = {'D': WEBDAV_NS}

D = lb.ElementMaker(namespace=WEBDAV_NS, nsmap=WEBDAV_NSMAP)

def get_property_tag_list(res, *names):
    props = []
    for name in names:
        tag = get_property_tag(res, name)
        if tag is None:
            continue
        props.append(tag)
    return props

def get_property_tag(res, name):
    if name == 'resourcetype':
        if res.is_collection:
            return D(name, D.collection)
        return D(name)
    try:
        if hasattr(res, name):
            return D(name, unicode(getattr(res, name)))
    except AttributeError:
        return

def safe_join(root, *paths):
    """The provided os.path.join() does not work as desired. Any path starting with /
    will simply be returned rather than actually being joined with the other elements."""
    if not root.startswith('/'):
        root = '/' + root
    for path in paths:
        while root.endswith('/'):
            root = root[:-1]
        while path.startswith('/'):
            path = path[1:]
        root += '/' + path
    return root

def url_join(base, *paths):
    """Assuming base is the scheme and host (and perhaps path) we will join the remaining
    path elements to it."""
    paths = safe_join(*paths) if paths else ""
    while base.endswith('/'):
        base = base[:-1]
    return base + paths

def ns_split(tag):
    """Splits the namespace and property name from a clark notation property name."""
    if tag.startswith("{") and "}" in tag:
        ns, name = tag.split("}", 1)
        return ns[1:-1], name
    return "", tag

def ns_join(ns, name):
    """Joins a namespace and property name into clark notation."""
    return '{%s:}%s' % (ns, name)

def rfc3339_date(dt):
    if not dt:
        return ''
    return dt.strftime('%Y-%m-%dT%H:%M:%SZ')

def rfc1123_date(dt):
    if not dt:
        return ''
    return rfc2822_date(dt)

def parse_time(timestring):
    value = None
    for fmt in (FORMAT_RFC_822, FORMAT_RFC_850, FORMAT_ASC):
        try:
            value = time.strptime(timestring, fmt)
        except ValueError:
            pass
    if value is None:
        try:
            # Sun Nov  6 08:49:37 1994 +0100      ; ANSI C's asctime() format with timezone
            value = parsedate_tz(timestring)
        except ValueError:
            pass
    if value is None:
        return
    return calendar.timegm(value)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from django.db.models import CharField
from django.core.exceptions import ImproperlyConfigured
from django.utils.translation import ugettext_lazy as _

from .widgets import LocationWidget

class LocationField(CharField):
    description = _("A geoposition field (latitude and longitude)")

    def __init__(self, base_field=None, *args, **kwargs):
        self.base_field = base_field
        kwargs['max_length'] = 63
        super(LocationField, self).__init__(*args, **kwargs)

    def deconstruct(self):
        name, path, args, kwargs = super(LocationField, self).deconstruct()
        del kwargs["max_length"]

        if self.base_field == '':
            raise ImproperlyConfigured()
            #print('Foo')
            #kwargs['separator'] = self.separator
        return name, path, args, kwargs

    def formfield(self, **kwargs):
        kwargs['widget'] = LocationWidget
        return super(LocationField, self).formfield(**kwargs)
<commit_msg>Remove unused code<commit_after># -*- coding: utf-8 -*-

from django.db.models import CharField
from django.core.exceptions import ImproperlyConfigured
from django.utils.translation import ugettext_lazy as _

from .widgets import LocationWidget

class LocationField(CharField):
    description = _("A geoposition field (latitude and longitude)")

    def __init__(self, base_field=None, *args, **kwargs):
        self.base_field = base_field
        kwargs['max_length'] = 63
        super(LocationField, self).__init__(*args, **kwargs)

    def deconstruct(self):
        name, path, args, kwargs = super(LocationField, self).deconstruct()
        del kwargs["max_length"]

        if self.base_field == '':
            raise ImproperlyConfigured()
        return name, path, args, kwargs

    def formfield(self, **kwargs):
        kwargs['widget'] = LocationWidget
        return super(LocationField, self).formfield(**kwargs)
<|endoftext|>


<commit_before>from django.conf import settings

# e.g. 2015-05-01 00:00:00
START_DATE = settings.START_DATE
# Informix likes DATE("2015-06-01") when using an IDE like Squirrel or RazorSQL
# or DATE("06/01/15")

STUDENTS_ALPHA = """
SELECT
    UNIQUE
    CASE
        WHEN
            NVL(UPPER(stu_serv_rec.bldg), '') = 'CMTR'
        OR
            NVL(stu_serv_rec.bldg, '') = 'undc'
        OR
            NVL(UPPER(stu_serv_rec.bldg), '') = 'OFF'
        OR
            NVL(UPPER(stu_serv_rec.bldg), '') = ''
        THEN
            'Commuter'
        ELSE
            'Resident'
        END
    AS
        residency_status,
    id_rec.lastname, id_rec.firstname, id_rec.middlename,
    id_rec.id, profile_rec.birth_date,cvid_rec.ldap_name,
    cc_student_medical_manager.id as manid,
    cc_student_medical_manager.created_at,
    cc_student_medical_manager.staff_notes,
    cc_student_medical_manager.athlete,
    cc_student_medical_manager.sitrep,
    cc_student_medical_manager.sitrep_athlete,
    cc_student_medical_manager.concussion_baseline,
    cc_student_medical_manager.emergency_contact,
    cc_student_medical_manager.sports,
    cc_student_medical_manager.medical_consent_agreement,
    cc_student_medical_manager.medical_consent_agreement_status,
    cc_student_medical_manager.physical_evaluation_1,
    cc_student_medical_manager.physical_evaluation_2,
    cc_student_medical_manager.cc_student_medical_history,
    cc_student_medical_manager.cc_student_health_insurance,
    cc_student_medical_manager.cc_student_meni_waiver,
    cc_student_medical_manager.cc_student_immunization,
    cc_student_medical_manager.cc_athlete_medical_history,
    cc_student_medical_manager.cc_athlete_privacy_waiver,
    cc_student_medical_manager.cc_athlete_reporting_waiver,
    cc_student_medical_manager.cc_athlete_risk_waiver,
    cc_student_medical_manager.cc_athlete_sicklecell_waiver,
    cc_student_health_insurance.primary_policy_type,
    cc_athlete_sicklecell_waiver.updated_at,
    cc_athlete_sicklecell_waiver.waive,
    cc_athlete_sicklecell_waiver.proof,
    cc_athlete_sicklecell_waiver.results,
    cc_athlete_sicklecell_waiver.results_file
FROM
    id_rec
INNER JOIN
    prog_enr_rec ON  id_rec.id = prog_enr_rec.id
LEFT JOIN
    cvid_rec
ON
    id_rec.id = cvid_rec.cx_id
LEFT JOIN
    stu_acad_rec    ON  id_rec.id   =   stu_acad_rec.id
LEFT JOIN
    stu_serv_rec    ON  id_rec.id   =   stu_serv_rec.id
LEFT JOIN
    profile_rec  ON  id_rec.id = profile_rec.id
LEFT JOIN
    cc_student_medical_manager ON id_rec.id = cc_student_medical_manager.college_id
    AND
        cc_student_medical_manager.created_at > "{}"
LEFT JOIN
    cc_student_health_insurance
  ON
    cc_student_medical_manager.id = cc_student_health_insurance.manager_id
LEFT JOIN
    cc_athlete_sicklecell_waiver ON id_rec.id = cc_athlete_sicklecell_waiver.college_id
    AND
        (cc_athlete_sicklecell_waiver.proof = 1 or cc_athlete_sicklecell_waiver.created_at > "{}")
WHERE
    prog_enr_rec.subprog NOT IN  ("UWPK","RSBD","SLS","PARA","MSW","KUSD","ENRM","CONF","CHWK")
AND
    prog_enr_rec.lv_date IS  NULL
AND
    stu_acad_rec.sess IN  ("RA","RC","AM","GC","PC","TC","GD","GA","GC")
""".format(START_DATE,START_DATE)

STUDENT_VITALS = """
SELECT
    UNIQUE
    CASE
        WHEN
            NVL(UPPER(stu_serv_rec.bldg), '') = 'CMTR'
        OR
            NVL(stu_serv_rec.bldg, '') = 'undc'
        OR
            NVL(UPPER(stu_serv_rec.bldg), '') = ''
        THEN
            'Commuter'
        ELSE
            'Resident'
        END
    AS
        residency_status,stu_serv_rec.stusv_no,
    id_rec.id, id_rec.lastname, id_rec.firstname, id_rec.middlename,
    id_rec.addr_line1, id_rec.addr_line2, id_rec.city, id_rec.st,
    id_rec.zip, id_rec.ctry, id_rec.phone, cvid_rec.ldap_name,
    cc_student_medical_manager.id as manid,
    cc_student_medical_manager.created_at,
    cc_student_medical_manager.sitrep,
    cc_student_medical_manager.sitrep_athlete,
    cc_student_medical_manager.concussion_baseline,
    cc_student_medical_manager.athlete, cc_student_medical_manager.sports,
    cc_student_medical_manager.medical_consent_agreement,
    cc_student_medical_manager.medical_consent_agreement_status,
    cc_student_medical_manager.physical_evaluation_1,
    cc_student_medical_manager.physical_evaluation_status_1,
    cc_student_medical_manager.physical_evaluation_2,
    cc_student_medical_manager.physical_evaluation_status_2,
    cc_student_medical_manager.cc_student_immunization,
    cc_student_medical_manager.cc_student_meni_waiver,
    cc_student_medical_manager.cc_athlete_privacy_waiver,
    cc_student_medical_manager.cc_athlete_reporting_waiver,
    cc_student_medical_manager.cc_athlete_risk_waiver,
    cc_student_medical_manager.cc_athlete_sicklecell_waiver,
    cc_student_health_insurance.primary_card_front_status,
    cc_student_health_insurance.primary_card_back_status,
    cc_athlete_sicklecell_waiver.id as sicklecell_id,
    cc_athlete_sicklecell_waiver.updated_at,
    cc_athlete_sicklecell_waiver.waive,
    cc_athlete_sicklecell_waiver.proof,
    cc_athlete_sicklecell_waiver.results,
    cc_athlete_sicklecell_waiver.results_file,
    cc_athlete_sicklecell_waiver.results_file_status,
    cc_athlete_privacy_waiver.id as privacy_id,
    cc_athlete_privacy_waiver.news_media,
    cc_athlete_privacy_waiver.parents_guardians,
    cc_athlete_privacy_waiver.disclose_records,
    cc_student_meni_waiver.id as menid,
    cc_athlete_risk_waiver.id as risk_id,
    cc_athlete_reporting_waiver.id as reporting_id,
    profile_rec.birth_date,
    profile_rec.sex,
    prog_enr_rec.cl,
    mobile_rec.phone as mobile
FROM
    id_rec
INNER JOIN
    prog_enr_rec
ON
    id_rec.id = prog_enr_rec.id
LEFT JOIN
    cvid_rec
ON
    id_rec.id = cvid_rec.cx_id
LEFT JOIN
    cc_student_medical_manager
ON
    id_rec.id = cc_student_medical_manager.college_id
LEFT JOIN
    cc_student_meni_waiver
ON
    cc_student_medical_manager.id = cc_student_meni_waiver.manager_id
LEFT JOIN
    cc_student_health_insurance
ON
    cc_student_medical_manager.id = cc_student_health_insurance.manager_id
LEFT JOIN
    cc_athlete_sicklecell_waiver
ON
    id_rec.id = cc_athlete_sicklecell_waiver.college_id
AND
    (cc_athlete_sicklecell_waiver.proof = 1 OR cc_athlete_sicklecell_waiver.created_at > "{}")
LEFT JOIN
    cc_athlete_privacy_waiver
ON
    cc_student_medical_manager.id = cc_athlete_privacy_waiver.manager_id
LEFT JOIN
    cc_athlete_risk_waiver
ON
    cc_student_medical_manager.id = cc_athlete_risk_waiver.manager_id
LEFT JOIN
    cc_athlete_reporting_waiver
ON
    cc_student_medical_manager.id = cc_athlete_reporting_waiver.manager_id
LEFT JOIN
    profile_rec
ON
    id_rec.id = profile_rec.id
LEFT JOIN
    stu_serv_rec
ON
    id_rec.id = stu_serv_rec.id
LEFT JOIN
    aa_rec as mobile_rec
ON
    (id_rec.id = mobile_rec.id AND mobile_rec.aa = "ENS")
""".format(START_DATE)

ACADEMIC_YEAR = """
SELECT
    id_rec.lastname, id_rec.firstname,
    cc_student_medical_manager.id,
    cc_student_medical_manager.college_id,
    cc_student_health_insurance.primary_policy_holder
FROM
    id_rec
LEFT JOIN
    cc_student_medical_manager
  ON
    id_rec.id = cc_student_medical_manager.college_id
LEFT JOIN
    cc_student_health_insurance
  ON
    cc_student_medical_manager.id = cc_student_health_insurance.manager_id
WHERE
    cc_student_medical_manager.college_id=
"""
<commit_msg>sql incantation for obtain sports in which students participate<commit_after>from django.conf import settings

# e.g. 2015-05-01 00:00:00
START_DATE = settings.START_DATE
# Informix likes DATE("2015-06-01") when using an IDE like Squirrel or RazorSQL
# or DATE("06/01/15")

STUDENTS_ALPHA = """
SELECT
    UNIQUE
    CASE
        WHEN
            NVL(UPPER(stu_serv_rec.bldg), '') = 'CMTR'
        OR
            NVL(stu_serv_rec.bldg, '') = 'undc'
        OR
            NVL(UPPER(stu_serv_rec.bldg), '') = 'OFF'
        OR
            NVL(UPPER(stu_serv_rec.bldg), '') = ''
        THEN
            'Commuter'
        ELSE
            'Resident'
        END
    AS
        residency_status,
    id_rec.lastname, id_rec.firstname, id_rec.middlename,
    id_rec.id, profile_rec.birth_date,cvid_rec.ldap_name,
    cc_student_medical_manager.id as manid,
    cc_student_medical_manager.created_at,
    cc_student_medical_manager.staff_notes,
    cc_student_medical_manager.athlete,
    cc_student_medical_manager.sitrep,
    cc_student_medical_manager.sitrep_athlete,
    cc_student_medical_manager.concussion_baseline,
    cc_student_medical_manager.emergency_contact,
    cc_student_medical_manager.sports,
    cc_student_medical_manager.medical_consent_agreement,
    cc_student_medical_manager.medical_consent_agreement_status,
    cc_student_medical_manager.physical_evaluation_1,
    cc_student_medical_manager.physical_evaluation_2,
    cc_student_medical_manager.cc_student_medical_history,
    cc_student_medical_manager.cc_student_health_insurance,
    cc_student_medical_manager.cc_student_meni_waiver,
    cc_student_medical_manager.cc_student_immunization,
    cc_student_medical_manager.cc_athlete_medical_history,
    cc_student_medical_manager.cc_athlete_privacy_waiver,
    cc_student_medical_manager.cc_athlete_reporting_waiver,
    cc_student_medical_manager.cc_athlete_risk_waiver,
    cc_student_medical_manager.cc_athlete_sicklecell_waiver,
    cc_student_health_insurance.primary_policy_type,
    cc_athlete_sicklecell_waiver.updated_at,
    cc_athlete_sicklecell_waiver.waive,
    cc_athlete_sicklecell_waiver.proof,
    cc_athlete_sicklecell_waiver.results,
    cc_athlete_sicklecell_waiver.results_file
FROM
    id_rec
INNER JOIN
    prog_enr_rec ON  id_rec.id = prog_enr_rec.id
LEFT JOIN
    cvid_rec
ON
    id_rec.id = cvid_rec.cx_id
LEFT JOIN
    stu_acad_rec    ON  id_rec.id   =   stu_acad_rec.id
LEFT JOIN
    stu_serv_rec    ON  id_rec.id   =   stu_serv_rec.id
LEFT JOIN
    profile_rec  ON  id_rec.id = profile_rec.id
LEFT JOIN
    cc_student_medical_manager ON id_rec.id = cc_student_medical_manager.college_id
    AND
        cc_student_medical_manager.created_at > "{}"
LEFT JOIN
    cc_student_health_insurance
  ON
    cc_student_medical_manager.id = cc_student_health_insurance.manager_id
LEFT JOIN
    cc_athlete_sicklecell_waiver ON id_rec.id = cc_athlete_sicklecell_waiver.college_id
    AND
        (cc_athlete_sicklecell_waiver.proof = 1 or cc_athlete_sicklecell_waiver.created_at > "{}")
WHERE
    prog_enr_rec.subprog NOT IN  ("UWPK","RSBD","SLS","PARA","MSW","KUSD","ENRM","CONF","CHWK")
AND
    prog_enr_rec.lv_date IS  NULL
AND
    stu_acad_rec.sess IN  ("RA","RC","AM","GC","PC","TC","GD","GA","GC")
""".format(START_DATE,START_DATE)

STUDENT_VITALS = """
SELECT
    UNIQUE
    CASE
        WHEN
            NVL(UPPER(stu_serv_rec.bldg), '') = 'CMTR'
        OR
            NVL(stu_serv_rec.bldg, '') = 'undc'
        OR
            NVL(UPPER(stu_serv_rec.bldg), '') = ''
        THEN
            'Commuter'
        ELSE
            'Resident'
        END
    AS
        residency_status,stu_serv_rec.stusv_no,
    id_rec.id, id_rec.lastname, id_rec.firstname, id_rec.middlename,
    id_rec.addr_line1, id_rec.addr_line2, id_rec.city, id_rec.st,
    id_rec.zip, id_rec.ctry, id_rec.phone, cvid_rec.ldap_name,
    cc_student_medical_manager.id as manid,
    cc_student_medical_manager.created_at,
    cc_student_medical_manager.sitrep,
    cc_student_medical_manager.sitrep_athlete,
    cc_student_medical_manager.concussion_baseline,
    cc_student_medical_manager.athlete, cc_student_medical_manager.sports,
    cc_student_medical_manager.medical_consent_agreement,
    cc_student_medical_manager.medical_consent_agreement_status,
    cc_student_medical_manager.physical_evaluation_1,
    cc_student_medical_manager.physical_evaluation_status_1,
    cc_student_medical_manager.physical_evaluation_2,
    cc_student_medical_manager.physical_evaluation_status_2,
    cc_student_medical_manager.cc_student_immunization,
    cc_student_medical_manager.cc_student_meni_waiver,
    cc_student_medical_manager.cc_athlete_privacy_waiver,
    cc_student_medical_manager.cc_athlete_reporting_waiver,
    cc_student_medical_manager.cc_athlete_risk_waiver,
    cc_student_medical_manager.cc_athlete_sicklecell_waiver,
    cc_student_health_insurance.primary_card_front_status,
    cc_student_health_insurance.primary_card_back_status,
    cc_athlete_sicklecell_waiver.id as sicklecell_id,
    cc_athlete_sicklecell_waiver.updated_at,
    cc_athlete_sicklecell_waiver.waive,
    cc_athlete_sicklecell_waiver.proof,
    cc_athlete_sicklecell_waiver.results,
    cc_athlete_sicklecell_waiver.results_file,
    cc_athlete_sicklecell_waiver.results_file_status,
    cc_athlete_privacy_waiver.id as privacy_id,
    cc_athlete_privacy_waiver.news_media,
    cc_athlete_privacy_waiver.parents_guardians,
    cc_athlete_privacy_waiver.disclose_records,
    cc_student_meni_waiver.id as menid,
    cc_athlete_risk_waiver.id as risk_id,
    cc_athlete_reporting_waiver.id as reporting_id,
    profile_rec.birth_date,
    profile_rec.sex,
    prog_enr_rec.cl,
    mobile_rec.phone as mobile
FROM
    id_rec
INNER JOIN
    prog_enr_rec
ON
    id_rec.id = prog_enr_rec.id
LEFT JOIN
    cvid_rec
ON
    id_rec.id = cvid_rec.cx_id
LEFT JOIN
    cc_student_medical_manager
ON
    id_rec.id = cc_student_medical_manager.college_id
LEFT JOIN
    cc_student_meni_waiver
ON
    cc_student_medical_manager.id = cc_student_meni_waiver.manager_id
LEFT JOIN
    cc_student_health_insurance
ON
    cc_student_medical_manager.id = cc_student_health_insurance.manager_id
LEFT JOIN
    cc_athlete_sicklecell_waiver
ON
    id_rec.id = cc_athlete_sicklecell_waiver.college_id
AND
    (cc_athlete_sicklecell_waiver.proof = 1 OR cc_athlete_sicklecell_waiver.created_at > "{}")
LEFT JOIN
    cc_athlete_privacy_waiver
ON
    cc_student_medical_manager.id = cc_athlete_privacy_waiver.manager_id
LEFT JOIN
    cc_athlete_risk_waiver
ON
    cc_student_medical_manager.id = cc_athlete_risk_waiver.manager_id
LEFT JOIN
    cc_athlete_reporting_waiver
ON
    cc_student_medical_manager.id = cc_athlete_reporting_waiver.manager_id
LEFT JOIN
    profile_rec
ON
    id_rec.id = profile_rec.id
LEFT JOIN
    stu_serv_rec
ON
    id_rec.id = stu_serv_rec.id
LEFT JOIN
    aa_rec as mobile_rec
ON
    (id_rec.id = mobile_rec.id AND mobile_rec.aa = "ENS")
""".format(START_DATE)

ACADEMIC_YEAR = """
SELECT
    id_rec.lastname, id_rec.firstname,
    cc_student_medical_manager.id,
    cc_student_medical_manager.college_id,
    cc_student_health_insurance.primary_policy_holder
FROM
    id_rec
LEFT JOIN
    cc_student_medical_manager
  ON
    id_rec.id = cc_student_medical_manager.college_id
LEFT JOIN
    cc_student_health_insurance
  ON
    cc_student_medical_manager.id = cc_student_health_insurance.manager_id
WHERE
    cc_student_medical_manager.college_id=
"""

sports = """
SELECT
   TRIM(IT.invl) AS sport_code,
   TRIM(IT.txt) AS sport_name,
   INR.id AS id,
   INR.beg_date,
   INR.end_date
FROM
   invl_table IT
INNER JOIN
    involve_rec INR
ON
    TRIM(IT.invl) = TRIM(INR.invl)
AND
    IT.sanc_sport = 'Y'
WHERE
   TODAY BETWEEN IT.active_date AND NVL(IT.inactive_date, TODAY)
AND
   TODAY < NVL(INR.end_date, TODAY)
AND
   INR.id =
"""
<|endoftext|>


<commit_before>#
#    ICRAR - International Centre for Radio Astronomy Research
#    (c) UWA - The University of Western Australia, 2017
#    Copyright by UWA (in the framework of the ICRAR)
#    All rights reserved
#
#    This library is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation; either
#    version 2.1 of the License, or (at your option) any later version.
#
#    This library is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
#    MA 02111-1307  USA
#
"""Module implementing the PyFuncApp class"""

import base64
import collections
import importlib
import inspect
import logging

import dill
import six
import six.moves.cPickle as pickle  # @UnresolvedImport

from .. import droputils
from ..drop import BarrierAppDROP
from ..exceptions import InvalidDropException

_getargsspec = inspect.getargspec if six.PY2 else inspect.getfullargspec

logger = logging.getLogger(__name__)

def serialize_func(f):

    if isinstance(f, six.string_types):
        parts = f.split('.')
        f = getattr(importlib.import_module('.'.join(parts[:-1])), parts[-1])

    fser = dill.dumps(f)
    fdefaults = {}
    a = _getargsspec(f)
    if a.defaults:
        fdefaults = dict(zip(a.args[-len(a.defaults):], a.defaults))
    return fser, fdefaults

def import_using_name(app, fname):
    # The name has the form pack1.pack2.mod.func
    parts = fname.split('.')
    if len(parts) < 2:
        msg = '%s does not contain a module name' % fname
        raise InvalidDropException(app, msg)

    modname, fname = '.'.join(parts[:-1]), parts[-1]
    try:
        mod = importlib.import_module(modname, __name__)
        return getattr(mod, fname)
    except ImportError as e:
        raise InvalidDropException(app, 'Error when loading module %s: %s' % (modname, str(e)))
    except AttributeError:
        raise InvalidDropException(app, 'Module %s has no member %s' % (modname, fname))

def import_using_code(code):
    return dill.loads(code)

class PyFuncApp(BarrierAppDROP):
    """
    An application that wraps a simple python function.

    The inputs of the application are treated as the arguments of the function.
    Conversely, the output of the function is treated as the output of the
    application. If the application has more than one output, the result of
    calling the function is treated as an iterable, with each individual object
    being written to its corresponding output.

    Users indicate the function to be wrapped via the ``func_name`` parameter.
    Otherwise, users can also *send* over the python code using the ``func_code``
    parameter. The code needs to be base64-encoded and produced with the marshal
    module of the same Python version used to run DALiuGE.

    Both inputs and outputs are serialized using the pickle protocol.
    """

    def initialize(self, **kwargs):
        BarrierAppDROP.initialize(self, **kwargs)

        self.fname = fname = self._getArg(kwargs, 'func_name', None)
        fcode = self._getArg(kwargs, 'func_code', None)
        if not fname and not fcode:
            raise InvalidDropException(self, 'No function specified (either via name or code)')

        if not fcode:
            self.f = import_using_name(self, fname)
        else:
            if not isinstance(fcode, six.binary_type):
                fcode = base64.b64decode(six.b(fcode))
            self.f = import_using_code(fcode)

        # Mapping from argname to default value. Should match only the last part
        # of the argnames list
        self.fdefaults = self._getArg(kwargs, 'func_defaults', {}) or {}
        logger.debug("Default values for function: %r", self.fdefaults)

        # Mapping between argument name and input drop uids
        self.func_arg_mapping = self._getArg(kwargs, 'func_arg_mapping', {})
        logger.debug("Input mapping: %r", self.func_arg_mapping)

    def run(self):

        # Inputs are un-pickled and treated as the arguments of the function
        # Their order must be preserved, so we use an OrderedDict
        all_contents = lambda x: pickle.loads(droputils.allDropContents(x))
        inputs = collections.OrderedDict()
        for uid, i in self._inputs.items():
            inputs[uid] = all_contents(i)

        # Keyword arguments are made up by the default values plus the inputs
        # that match one of the keyword argument names
        kwargs = {name: inputs.pop(uid)
                  for name, uid in self.func_arg_mapping.items()
                  if name in self.fdefaults}

        # The rest of the inputs are the positional arguments
        args = list(inputs.values())

        logger.debug("Running %s with args=%r, kwargs=%r", self.fname, args, kwargs)
        result = self.f(*args, **kwargs)

        # Depending on how many outputs we have we treat our result
        # as an iterable or as a single object. Each result is pickled
        # and written to its corresponding output
        outputs = self.outputs
        if len(outputs) == 1:
            result = [result]
        for r, o in zip(result, outputs):
            o.write(pickle.dumps(r))  # @UndefinedVariable<commit_msg>Correctly handling **kwargs in PyFunc applications<commit_after>#
#    ICRAR - International Centre for Radio Astronomy Research
#    (c) UWA - The University of Western Australia, 2017
#    Copyright by UWA (in the framework of the ICRAR)
#    All rights reserved
#
#    This library is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation; either
#    version 2.1 of the License, or (at your option) any later version.
#
#    This library is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
#    MA 02111-1307  USA
#
"""Module implementing the PyFuncApp class"""

import base64
import collections
import importlib
import inspect
import logging

import dill
import six
import six.moves.cPickle as pickle  # @UnresolvedImport

from .. import droputils
from ..drop import BarrierAppDROP
from ..exceptions import InvalidDropException

_getargsspec = inspect.getargspec if six.PY2 else inspect.getfullargspec

logger = logging.getLogger(__name__)

def serialize_func(f):

    if isinstance(f, six.string_types):
        parts = f.split('.')
        f = getattr(importlib.import_module('.'.join(parts[:-1])), parts[-1])

    fser = dill.dumps(f)
    fdefaults = {}
    a = _getargsspec(f)
    if a.defaults:
        fdefaults = dict(zip(a.args[-len(a.defaults):], a.defaults))
    return fser, fdefaults

def import_using_name(app, fname):
    # The name has the form pack1.pack2.mod.func
    parts = fname.split('.')
    if len(parts) < 2:
        msg = '%s does not contain a module name' % fname
        raise InvalidDropException(app, msg)

    modname, fname = '.'.join(parts[:-1]), parts[-1]
    try:
        mod = importlib.import_module(modname, __name__)
        return getattr(mod, fname)
    except ImportError as e:
        raise InvalidDropException(app, 'Error when loading module %s: %s' % (modname, str(e)))
    except AttributeError:
        raise InvalidDropException(app, 'Module %s has no member %s' % (modname, fname))

def import_using_code(code):
    return dill.loads(code)

class PyFuncApp(BarrierAppDROP):
    """
    An application that wraps a simple python function.

    The inputs of the application are treated as the arguments of the function.
    Conversely, the output of the function is treated as the output of the
    application. If the application has more than one output, the result of
    calling the function is treated as an iterable, with each individual object
    being written to its corresponding output.

    Users indicate the function to be wrapped via the ``func_name`` parameter.
    Otherwise, users can also *send* over the python code using the ``func_code``
    parameter. The code needs to be base64-encoded and produced with the marshal
    module of the same Python version used to run DALiuGE.

    Both inputs and outputs are serialized using the pickle protocol.
    """

    def initialize(self, **kwargs):
        BarrierAppDROP.initialize(self, **kwargs)

        self.fname = fname = self._getArg(kwargs, 'func_name', None)
        fcode = self._getArg(kwargs, 'func_code', None)
        if not fname and not fcode:
            raise InvalidDropException(self, 'No function specified (either via name or code)')

        if not fcode:
            self.f = import_using_name(self, fname)
        else:
            if not isinstance(fcode, six.binary_type):
                fcode = base64.b64decode(six.b(fcode))
            self.f = import_using_code(fcode)

        # Mapping from argname to default value. Should match only the last part
        # of the argnames list
        self.fdefaults = self._getArg(kwargs, 'func_defaults', {}) or {}
        logger.debug("Default values for function: %r", self.fdefaults)

        # Mapping between argument name and input drop uids
        self.func_arg_mapping = self._getArg(kwargs, 'func_arg_mapping', {})
        logger.debug("Input mapping: %r", self.func_arg_mapping)

    def run(self):

        # Inputs are un-pickled and treated as the arguments of the function
        # Their order must be preserved, so we use an OrderedDict
        all_contents = lambda x: pickle.loads(droputils.allDropContents(x))
        inputs = collections.OrderedDict()
        for uid, i in self._inputs.items():
            inputs[uid] = all_contents(i)

        # Keyword arguments are made up by the default values plus the inputs
        # that match one of the keyword argument names
        argnames = _getargsspec(self.f).args
        kwargs = {name: inputs.pop(uid)
                  for name, uid in self.func_arg_mapping.items()
                  if name in self.fdefaults or name not in argnames}

        # The rest of the inputs are the positional arguments
        args = list(inputs.values())

        logger.debug("Running %s with args=%r, kwargs=%r", self.fname, args, kwargs)
        result = self.f(*args, **kwargs)

        # Depending on how many outputs we have we treat our result
        # as an iterable or as a single object. Each result is pickled
        # and written to its corresponding output
        outputs = self.outputs
        if len(outputs) == 1:
            result = [result]
        for r, o in zip(result, outputs):
            o.write(pickle.dumps(r))  # @UndefinedVariable<|endoftext|>


<commit_before># Copyright 2014 Rackspace Inc.
#
# Author: Eric Larson <eric.larson@rackspace.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from unittest import mock

import dns
from oslo_config import cfg
from oslo_config import fixture as cfg_fixture
import oslotest.base

from designate import exceptions
from designate.mdns import handler
from designate import objects
from designate.tests import fixtures
from designate.worker import rpcapi as worker_rpcapi

CONF = cfg.CONF

class MdnsHandleTest(oslotest.base.BaseTestCase):
    def setUp(self):
        super(MdnsHandleTest, self).setUp()
        self.stdlog = fixtures.StandardLogging()
        self.useFixture(self.stdlog)
        self.useFixture(cfg_fixture.Config(CONF))
        self.context = mock.Mock()
        self.storage = mock.Mock()
        self.tg = mock.Mock()
        self.handler = handler.RequestHandler(self.storage, self.tg)

    def test_worker_api(self):
        self.assertIsNone(self.handler._worker_api)
        self.assertIsInstance(self.handler.worker_api,
                              worker_rpcapi.WorkerAPI)
        self.assertIsNotNone(self.handler._worker_api)
        self.assertIsInstance(self.handler.worker_api,
                              worker_rpcapi.WorkerAPI)

    @mock.patch.object(dns.resolver.Resolver, 'query')
    def test_notify(self, mock_query):
        self.storage.find_zone.return_value = objects.Zone(
            id='e2bed4dc-9d01-11e4-89d3-123b93f75cba',
            serial=2,
            masters=objects.ZoneMasterList.from_list([
                {'host': '1.0.0.0', 'port': 53},
            ])
        )
        mock_query.return_value = [
            mock.Mock(serial=1)
        ]

        request = dns.message.make_query(
            'www.example.org.', dns.rdatatype.SOA
        )
        request.environ = dict(addr=['1.0.0.0'], context=self.context)

        response = self.handler._handle_notify(request)

        self.assertEqual(dns.rcode.NOERROR, tuple(response)[0].rcode())

        self.assertIn(
            'Scheduling AXFR for e2bed4dc-9d01-11e4-89d3-123b93f75cba '
            'from 1.0.0.0:53',
            self.stdlog.logger.output
        )

    @mock.patch.object(dns.resolver.Resolver, 'query')
    def test_notify_same_serial(self, mock_query):
        self.storage.find_zone.return_value = objects.Zone(
            id='e2bed4dc-9d01-11e4-89d3-123b93f75cba',
            serial=1,
            masters=objects.ZoneMasterList.from_list([
                {'host': '1.0.0.0', 'port': 53},
            ])
        )
        mock_query.return_value = [
            mock.Mock(serial=1)
        ]

        request = dns.message.make_query(
            'www.example.org.', dns.rdatatype.SOA
        )
        request.environ = dict(addr=['1.0.0.0'], context=self.context)

        response = self.handler._handle_notify(request)

        self.assertEqual(dns.rcode.NOERROR, tuple(response)[0].rcode())

        self.assertIn(
            'Serial 1 is the same for master and us for '
            'e2bed4dc-9d01-11e4-89d3-123b93f75cba',
            self.stdlog.logger.output
        )

    def test_notify_no_questions(self):
        request = dns.message.make_query(
            'www.example.org.', dns.rdatatype.SOA
        )
        request.environ = dict(context=self.context)
        request.question = []

        response = self.handler._handle_notify(request)

        self.assertEqual(dns.rcode.FORMERR, tuple(response)[0].rcode())

    def test_notify_zone_not_found(self):
        self.storage.find_zone.side_effect = exceptions.ZoneNotFound

        request = dns.message.make_query(
            'www.example.org.', dns.rdatatype.SOA
        )
        request.environ = dict(context=self.context)

        response = self.handler._handle_notify(request)

        self.assertEqual(dns.rcode.NOTAUTH, tuple(response)[0].rcode())

    def test_notify_no_master_addr(self):
        self.storage.find_zone.return_value = objects.Zone(
            masters=objects.ZoneMasterList.from_list([
                {'host': '1.0.0.0', 'port': 53},
            ])
        )

        request = dns.message.make_query(
            'www.example.org.', dns.rdatatype.SOA
        )
        request.environ = dict(addr=['127.0.0.1', 53], context=self.context)

        response = self.handler._handle_notify(request)

        self.assertEqual(dns.rcode.REFUSED, tuple(response)[0].rcode())

        self.assertIn(
            'NOTIFY for None from non-master server 127.0.0.1, refusing.',
            self.stdlog.logger.output
        )

    def test_axfr_zone_not_found(self):
        self.storage.find_zone.side_effect = exceptions.ZoneNotFound

        request = dns.message.make_query(
            'www.example.org.', dns.rdatatype.AXFR
        )
        request.environ = dict(context=self.context)

        response = tuple(self.handler._handle_axfr(request))

        self.assertEqual(dns.rcode.REFUSED, response[0].rcode())

        self.assertIn(
            'ZoneNotFound while handling axfr request. '
            'Question was www.example.org. IN AXFR',
            self.stdlog.logger.output
        )

    def test_axfr_forbidden(self):
        self.storage.find_zone.side_effect = exceptions.Forbidden

        request = dns.message.make_query(
            'www.example.org.', dns.rdatatype.AXFR
        )
        request.environ = dict(context=self.context)

        response = tuple(self.handler._handle_axfr(request))

        self.assertEqual(dns.rcode.REFUSED, response[0].rcode())

        self.assertIn(
            'Forbidden while handling axfr request. '
            'Question was www.example.org. IN AXFR',
            self.stdlog.logger.output
        )

    def test_get_max_message_size(self):
        CONF.set_override('max_message_size', 32768, 'service:mdns')

        self.assertEqual(
            32768, self.handler._get_max_message_size(had_tsig=False)
        )

    def test_get_max_message_size_larger_than_allowed(self):
        CONF.set_override('max_message_size', 65535 * 2, 'service:mdns')

        self.assertEqual(
            65535, self.handler._get_max_message_size(had_tsig=False)
        )

        self.assertIn(
            'MDNS max message size must not be greater than 65535',
            self.stdlog.logger.output
        )

    def test_get_max_message_with_tsig(self):
        CONF.set_override('max_message_size', 65535, 'service:mdns')

        self.assertEqual(
            65300, self.handler._get_max_message_size(had_tsig=True)
        )

class TestRequestHandlerCall(oslotest.base.BaseTestCase):
    def setUp(self):
        super(TestRequestHandlerCall, self).setUp()
        self.handler = handler.RequestHandler(mock.Mock(), mock.Mock())

        # Use a simple handlers that doesn't require a real request
        self.handler._handle_query_error = mock.Mock(return_value='Error')
        self.handler._handle_axfr = mock.Mock(return_value=['AXFR'])
        self.handler._handle_record_query = mock.Mock(
            return_value=['Record Query'])
        self.handler._handle_notify = mock.Mock(return_value=['Notify'])

    def test__call___unhandled_opcodes(self):
        unhandled_codes = [
            dns.opcode.STATUS,
            dns.opcode.IQUERY,
            dns.opcode.UPDATE,
        ]

        request = mock.Mock()
        for code in unhandled_codes:
            request.opcode.return_value = code  # return an error

            self.assertEqual(['Error'], list(self.handler(request)))
            self.handler._handle_query_error.assert_called_with(
                request, dns.rcode.REFUSED
            )

    def test__call__query_error_with_more_than_one_question(self):
        request = mock.Mock()
        request.opcode.return_value = dns.opcode.QUERY
        request.question = [mock.Mock(), mock.Mock()]

        self.assertEqual(['Error'], list(self.handler(request)))
        self.handler._handle_query_error.assert_called_with(
            request, dns.rcode.REFUSED
        )

    def test__call__query_error_with_data_claas_not_in(self):
        request = mock.Mock()
        request.opcode.return_value = dns.opcode.QUERY
        request.question = [mock.Mock(rdclass=dns.rdataclass.ANY)]

        self.assertEqual(['Error'], list(self.handler(request)))
        self.handler._handle_query_error.assert_called_with(
            request, dns.rcode.REFUSED
        )

    def test__call__axfr(self):
        request = mock.Mock()
        request.opcode.return_value = dns.opcode.QUERY
        request.question = [
            mock.Mock(rdclass=dns.rdataclass.IN, rdtype=dns.rdatatype.AXFR)
        ]

        self.assertEqual(['AXFR'], list(self.handler(request)))

    def test__call__ixfr(self):
        request = mock.Mock()
        request.opcode.return_value = dns.opcode.QUERY
        request.question = [
            mock.Mock(rdclass=dns.rdataclass.IN, rdtype=dns.rdatatype.IXFR)
        ]

        self.assertEqual(['AXFR'], list(self.handler(request)))

    def test__call__record_query(self):
        request = mock.Mock()
        request.opcode.return_value = dns.opcode.QUERY
        request.question = [
            mock.Mock(rdclass=dns.rdataclass.IN, rdtype=dns.rdatatype.A)
        ]

        self.assertEqual(['Record Query'], list(self.handler(request)))

    def test__call__notify(self):
        request = mock.Mock()
        request.opcode.return_value = dns.opcode.NOTIFY

        self.assertEqual(['Notify'], list(self.handler(request)))

    def test_convert_to_rrset_no_records(self):
        zone = objects.Zone.from_dict({'ttl': 1234})
        recordset = objects.RecordSet(
            name='www.example.org.',
            type='A',
            records=objects.RecordList(objects=[
            ])
        )

        r_rrset = self.handler._convert_to_rrset(zone, recordset)

        self.assertIsNone(r_rrset)

    def test_convert_to_rrset(self):
        zone = objects.Zone.from_dict({'ttl': 1234})
        recordset = objects.RecordSet(
            name='www.example.org.',
            type='A',
            records=objects.RecordList(objects=[
                objects.Record(data='192.0.2.1'),
                objects.Record(data='192.0.2.2'),
            ])
        )

        r_rrset = self.handler._convert_to_rrset(zone, recordset)

        self.assertEqual(2, len(r_rrset))

class HandleRecordQueryTest(oslotest.base.BaseTestCase):
    def setUp(self):
        super(HandleRecordQueryTest, self).setUp()
        self.context = mock.Mock()
        self.storage = mock.Mock()
        self.handler = handler.RequestHandler(self.storage, mock.Mock())

    def test_handle_record_query_empty_recordlist(self):
        # bug #1550441
        self.storage.find_recordset.return_value = objects.RecordSet(
            name='www.example.org.',
            type='A',
            records=objects.RecordList(objects=[
            ])
        )

        request = dns.message.make_query('www.example.org.', dns.rdatatype.A)
        request.environ = dict(context=self.context)
        response_gen = self.handler._handle_record_query(request)
        for response in response_gen:
            # This was raising an exception due to bug #1550441
            out = response.to_wire(max_size=65535)

            self.assertEqual(33, len(out))

    def test_handle_record_query_zone_not_found(self):
        self.storage.find_recordset.return_value = objects.RecordSet(
            name='www.example.org.',
            type='A',
            records=objects.RecordList(objects=[
                objects.Record(data='192.0.2.2'),
            ])
        )
        self.storage.find_zone.side_effect = exceptions.ZoneNotFound

        request = dns.message.make_query('www.example.org.', dns.rdatatype.A)
        request.environ = dict(context=self.context)
        response = tuple(self.handler._handle_record_query(request))

        self.assertEqual(1, len(response))
        self.assertEqual(dns.rcode.REFUSED, response[0].rcode())

    def test_handle_record_query_forbidden(self):
        self.storage.find_recordset.return_value = objects.RecordSet(
            name='www.example.org.',
            type='A',
            records=objects.RecordList(objects=[
                objects.Record(data='192.0.2.2'),
            ])
        )
        self.storage.find_zone.side_effect = exceptions.Forbidden

        request = dns.message.make_query('www.example.org.', dns.rdatatype.A)
        request.environ = dict(context=self.context)
        response = tuple(self.handler._handle_record_query(request))

        self.assertEqual(1, len(response))
        self.assertEqual(dns.rcode.REFUSED, response[0].rcode())

    def test_handle_record_query_find_recordsed_forbidden(self):
        self.storage.find_recordset.side_effect = exceptions.Forbidden

        request = dns.message.make_query('www.example.org.', dns.rdatatype.A)
        request.environ = dict(context=self.context)
        response = tuple(self.handler._handle_record_query(request))

        self.assertEqual(1, len(response))
        self.assertEqual(dns.rcode.REFUSED, response[0].rcode())

    def test_handle_record_query_find_recordsed_not_found(self):
        self.storage.find_recordset.side_effect = exceptions.NotFound

        request = dns.message.make_query('www.example.org.', dns.rdatatype.A)
        request.environ = dict(context=self.context)
        response = tuple(self.handler._handle_record_query(request))

        self.assertEqual(1, len(response))
        self.assertEqual(dns.rcode.REFUSED, response[0].rcode())
<commit_msg>Fix mdns test_handler setup<commit_after># Copyright 2014 Rackspace Inc.
#
# Author: Eric Larson <eric.larson@rackspace.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from unittest import mock

import dns
from oslo_config import cfg
from oslo_config import fixture as cfg_fixture
from oslo_messaging import conffixture as messaging_fixture
import oslotest.base

from designate import exceptions
from designate.mdns import handler
from designate import objects
from designate.tests import fixtures
from designate.worker import rpcapi as worker_rpcapi

CONF = cfg.CONF

class MdnsHandleTest(oslotest.base.BaseTestCase):
    def setUp(self):
        super(MdnsHandleTest, self).setUp()
        self.stdlog = fixtures.StandardLogging()
        self.useFixture(self.stdlog)
        self.useFixture(cfg_fixture.Config(CONF))
        self.context = mock.Mock()
        self.storage = mock.Mock()
        self.tg = mock.Mock()
        self.handler = handler.RequestHandler(self.storage, self.tg)
        self.messaging_conf = messaging_fixture.ConfFixture(CONF)
        self.messaging_conf.transport_url = 'fake:/'
        self.messaging_conf.response_timeout = 5
        self.useFixture(self.messaging_conf)
        self.useFixture(fixtures.RPCFixture(CONF))

    def test_worker_api(self):
        self.assertIsNone(self.handler._worker_api)
        self.assertIsInstance(self.handler.worker_api,
                              worker_rpcapi.WorkerAPI)
        self.assertIsNotNone(self.handler._worker_api)
        self.assertIsInstance(self.handler.worker_api,
                              worker_rpcapi.WorkerAPI)

    @mock.patch.object(dns.resolver.Resolver, 'query')
    def test_notify(self, mock_query):
        self.storage.find_zone.return_value = objects.Zone(
            id='e2bed4dc-9d01-11e4-89d3-123b93f75cba',
            serial=2,
            masters=objects.ZoneMasterList.from_list([
                {'host': '1.0.0.0', 'port': 53},
            ])
        )
        mock_query.return_value = [
            mock.Mock(serial=1)
        ]

        request = dns.message.make_query(
            'www.example.org.', dns.rdatatype.SOA
        )
        request.environ = dict(addr=['1.0.0.0'], context=self.context)

        response = self.handler._handle_notify(request)

        self.assertEqual(dns.rcode.NOERROR, tuple(response)[0].rcode())

        self.assertIn(
            'Scheduling AXFR for e2bed4dc-9d01-11e4-89d3-123b93f75cba '
            'from 1.0.0.0:53',
            self.stdlog.logger.output
        )

    @mock.patch.object(dns.resolver.Resolver, 'query')
    def test_notify_same_serial(self, mock_query):
        self.storage.find_zone.return_value = objects.Zone(
            id='e2bed4dc-9d01-11e4-89d3-123b93f75cba',
            serial=1,
            masters=objects.ZoneMasterList.from_list([
                {'host': '1.0.0.0', 'port': 53},
            ])
        )
        mock_query.return_value = [
            mock.Mock(serial=1)
        ]

        request = dns.message.make_query(
            'www.example.org.', dns.rdatatype.SOA
        )
        request.environ = dict(addr=['1.0.0.0'], context=self.context)

        response = self.handler._handle_notify(request)

        self.assertEqual(dns.rcode.NOERROR, tuple(response)[0].rcode())

        self.assertIn(
            'Serial 1 is the same for master and us for '
            'e2bed4dc-9d01-11e4-89d3-123b93f75cba',
            self.stdlog.logger.output
        )

    def test_notify_no_questions(self):
        request = dns.message.make_query(
            'www.example.org.', dns.rdatatype.SOA
        )
        request.environ = dict(context=self.context)
        request.question = []

        response = self.handler._handle_notify(request)

        self.assertEqual(dns.rcode.FORMERR, tuple(response)[0].rcode())

    def test_notify_zone_not_found(self):
        self.storage.find_zone.side_effect = exceptions.ZoneNotFound

        request = dns.message.make_query(
            'www.example.org.', dns.rdatatype.SOA
        )
        request.environ = dict(context=self.context)

        response = self.handler._handle_notify(request)

        self.assertEqual(dns.rcode.NOTAUTH, tuple(response)[0].rcode())

    def test_notify_no_master_addr(self):
        self.storage.find_zone.return_value = objects.Zone(
            masters=objects.ZoneMasterList.from_list([
                {'host': '1.0.0.0', 'port': 53},
            ])
        )

        request = dns.message.make_query(
            'www.example.org.', dns.rdatatype.SOA
        )
        request.environ = dict(addr=['127.0.0.1', 53], context=self.context)

        response = self.handler._handle_notify(request)

        self.assertEqual(dns.rcode.REFUSED, tuple(response)[0].rcode())

        self.assertIn(
            'NOTIFY for None from non-master server 127.0.0.1, refusing.',
            self.stdlog.logger.output
        )

    def test_axfr_zone_not_found(self):
        self.storage.find_zone.side_effect = exceptions.ZoneNotFound

        request = dns.message.make_query(
            'www.example.org.', dns.rdatatype.AXFR
        )
        request.environ = dict(context=self.context)

        response = tuple(self.handler._handle_axfr(request))

        self.assertEqual(dns.rcode.REFUSED, response[0].rcode())

        self.assertIn(
            'ZoneNotFound while handling axfr request. '
            'Question was www.example.org. IN AXFR',
            self.stdlog.logger.output
        )

    def test_axfr_forbidden(self):
        self.storage.find_zone.side_effect = exceptions.Forbidden

        request = dns.message.make_query(
            'www.example.org.', dns.rdatatype.AXFR
        )
        request.environ = dict(context=self.context)

        response = tuple(self.handler._handle_axfr(request))

        self.assertEqual(dns.rcode.REFUSED, response[0].rcode())

        self.assertIn(
            'Forbidden while handling axfr request. '
            'Question was www.example.org. IN AXFR',
            self.stdlog.logger.output
        )

    def test_get_max_message_size(self):
        CONF.set_override('max_message_size', 32768, 'service:mdns')

        self.assertEqual(
            32768, self.handler._get_max_message_size(had_tsig=False)
        )

    def test_get_max_message_size_larger_than_allowed(self):
        CONF.set_override('max_message_size', 65535 * 2, 'service:mdns')

        self.assertEqual(
            65535, self.handler._get_max_message_size(had_tsig=False)
        )

        self.assertIn(
            'MDNS max message size must not be greater than 65535',
            self.stdlog.logger.output
        )

    def test_get_max_message_with_tsig(self):
        CONF.set_override('max_message_size', 65535, 'service:mdns')

        self.assertEqual(
            65300, self.handler._get_max_message_size(had_tsig=True)
        )

class TestRequestHandlerCall(oslotest.base.BaseTestCase):
    def setUp(self):
        super(TestRequestHandlerCall, self).setUp()
        self.handler = handler.RequestHandler(mock.Mock(), mock.Mock())

        # Use a simple handlers that doesn't require a real request
        self.handler._handle_query_error = mock.Mock(return_value='Error')
        self.handler._handle_axfr = mock.Mock(return_value=['AXFR'])
        self.handler._handle_record_query = mock.Mock(
            return_value=['Record Query'])
        self.handler._handle_notify = mock.Mock(return_value=['Notify'])

    def test__call___unhandled_opcodes(self):
        unhandled_codes = [
            dns.opcode.STATUS,
            dns.opcode.IQUERY,
            dns.opcode.UPDATE,
        ]

        request = mock.Mock()
        for code in unhandled_codes:
            request.opcode.return_value = code  # return an error

            self.assertEqual(['Error'], list(self.handler(request)))
            self.handler._handle_query_error.assert_called_with(
                request, dns.rcode.REFUSED
            )

    def test__call__query_error_with_more_than_one_question(self):
        request = mock.Mock()
        request.opcode.return_value = dns.opcode.QUERY
        request.question = [mock.Mock(), mock.Mock()]

        self.assertEqual(['Error'], list(self.handler(request)))
        self.handler._handle_query_error.assert_called_with(
            request, dns.rcode.REFUSED
        )

    def test__call__query_error_with_data_claas_not_in(self):
        request = mock.Mock()
        request.opcode.return_value = dns.opcode.QUERY
        request.question = [mock.Mock(rdclass=dns.rdataclass.ANY)]

        self.assertEqual(['Error'], list(self.handler(request)))
        self.handler._handle_query_error.assert_called_with(
            request, dns.rcode.REFUSED
        )

    def test__call__axfr(self):
        request = mock.Mock()
        request.opcode.return_value = dns.opcode.QUERY
        request.question = [
            mock.Mock(rdclass=dns.rdataclass.IN, rdtype=dns.rdatatype.AXFR)
        ]

        self.assertEqual(['AXFR'], list(self.handler(request)))

    def test__call__ixfr(self):
        request = mock.Mock()
        request.opcode.return_value = dns.opcode.QUERY
        request.question = [
            mock.Mock(rdclass=dns.rdataclass.IN, rdtype=dns.rdatatype.IXFR)
        ]

        self.assertEqual(['AXFR'], list(self.handler(request)))

    def test__call__record_query(self):
        request = mock.Mock()
        request.opcode.return_value = dns.opcode.QUERY
        request.question = [
            mock.Mock(rdclass=dns.rdataclass.IN, rdtype=dns.rdatatype.A)
        ]

        self.assertEqual(['Record Query'], list(self.handler(request)))

    def test__call__notify(self):
        request = mock.Mock()
        request.opcode.return_value = dns.opcode.NOTIFY

        self.assertEqual(['Notify'], list(self.handler(request)))

    def test_convert_to_rrset_no_records(self):
        zone = objects.Zone.from_dict({'ttl': 1234})
        recordset = objects.RecordSet(
            name='www.example.org.',
            type='A',
            records=objects.RecordList(objects=[
            ])
        )

        r_rrset = self.handler._convert_to_rrset(zone, recordset)

        self.assertIsNone(r_rrset)

    def test_convert_to_rrset(self):
        zone = objects.Zone.from_dict({'ttl': 1234})
        recordset = objects.RecordSet(
            name='www.example.org.',
            type='A',
            records=objects.RecordList(objects=[
                objects.Record(data='192.0.2.1'),
                objects.Record(data='192.0.2.2'),
            ])
        )

        r_rrset = self.handler._convert_to_rrset(zone, recordset)

        self.assertEqual(2, len(r_rrset))

class HandleRecordQueryTest(oslotest.base.BaseTestCase):
    def setUp(self):
        super(HandleRecordQueryTest, self).setUp()
        self.context = mock.Mock()
        self.storage = mock.Mock()
        self.handler = handler.RequestHandler(self.storage, mock.Mock())

    def test_handle_record_query_empty_recordlist(self):
        # bug #1550441
        self.storage.find_recordset.return_value = objects.RecordSet(
            name='www.example.org.',
            type='A',
            records=objects.RecordList(objects=[
            ])
        )

        request = dns.message.make_query('www.example.org.', dns.rdatatype.A)
        request.environ = dict(context=self.context)
        response_gen = self.handler._handle_record_query(request)
        for response in response_gen:
            # This was raising an exception due to bug #1550441
            out = response.to_wire(max_size=65535)

            self.assertEqual(33, len(out))

    def test_handle_record_query_zone_not_found(self):
        self.storage.find_recordset.return_value = objects.RecordSet(
            name='www.example.org.',
            type='A',
            records=objects.RecordList(objects=[
                objects.Record(data='192.0.2.2'),
            ])
        )
        self.storage.find_zone.side_effect = exceptions.ZoneNotFound

        request = dns.message.make_query('www.example.org.', dns.rdatatype.A)
        request.environ = dict(context=self.context)
        response = tuple(self.handler._handle_record_query(request))

        self.assertEqual(1, len(response))
        self.assertEqual(dns.rcode.REFUSED, response[0].rcode())

    def test_handle_record_query_forbidden(self):
        self.storage.find_recordset.return_value = objects.RecordSet(
            name='www.example.org.',
            type='A',
            records=objects.RecordList(objects=[
                objects.Record(data='192.0.2.2'),
            ])
        )
        self.storage.find_zone.side_effect = exceptions.Forbidden

        request = dns.message.make_query('www.example.org.', dns.rdatatype.A)
        request.environ = dict(context=self.context)
        response = tuple(self.handler._handle_record_query(request))

        self.assertEqual(1, len(response))
        self.assertEqual(dns.rcode.REFUSED, response[0].rcode())

    def test_handle_record_query_find_recordsed_forbidden(self):
        self.storage.find_recordset.side_effect = exceptions.Forbidden

        request = dns.message.make_query('www.example.org.', dns.rdatatype.A)
        request.environ = dict(context=self.context)
        response = tuple(self.handler._handle_record_query(request))

        self.assertEqual(1, len(response))
        self.assertEqual(dns.rcode.REFUSED, response[0].rcode())

    def test_handle_record_query_find_recordsed_not_found(self):
        self.storage.find_recordset.side_effect = exceptions.NotFound

        request = dns.message.make_query('www.example.org.', dns.rdatatype.A)
        request.environ = dict(context=self.context)
        response = tuple(self.handler._handle_record_query(request))

        self.assertEqual(1, len(response))
        self.assertEqual(dns.rcode.REFUSED, response[0].rcode())
<|endoftext|>