<commit_before>import UserDict

from pygr import annotation, seqdb, sequence, sqlgraph, worldbase
from pygr.dbfile import ReadOnlyError

gRes = None

class UCSCStrandDescr(object):

    def __get__(self, obj, objtype):
        if obj.strand == '+':
            return 1
        else:
            return -1

class UCSCSeqIntervalRow(sqlgraph.TupleO):
    orientation = UCSCStrandDescr()

class UCSCProteinSeq(sqlgraph.ProteinSQLSequence):
    '''Representation of UCSC protein-sequence tables such as ensGene,
    which lack the length column.'''

    def __len__(self):
        return self._select('length(seq)')

class UCSCEnsemblInterface(object):

    def __init__(self, ucsc_genome_name, ens_species=None,
                 ucsc_serverInfo=None, ens_serverInfo=None):
        '''Set up everything needed to produce UCSC/Ensembl
        annotation databases. ucsc_genome_name should follow the worldbase
        naming convention. If ens_species is not specified, we will try
        to autodetect it.'''
        # Only one instance can be active at a time for now.
        global gRes
        if gRes is not None:
            raise ValueError("A UCSCEnsemblInterface object already exists")
        else:
            gRes = self
        # Connect to both servers and prepare database names.
        if ucsc_serverInfo is not None:
            self.ucsc_server = ucsc_serverInfo
        else:
            self.ucsc_server = sqlgraph.DBServerInfo(
                host='genome-mysql.cse.ucsc.edu', user='genome')
        if ens_serverInfo is not None:
            self.ens_server = ens_serverInfo
        else:
            self.ens_server = sqlgraph.DBServerInfo(
                host='ensembldb.ensembl.org', port=5306, user='anonymous')
        self.ucsc_db = ucsc_genome_name.split('.')[-1]
        self.ens_db = self.get_ensembl_db_name(ens_species)
        # Connect to all the necessary tables.
        self.ucsc_ensGene_trans = sqlgraph.SQLTable('%s.ensGene' %
                                                    self.ucsc_db,
                                                   serverInfo=self.ucsc_server,
                                                    primaryKey='name',
                                                  itemClass=UCSCSeqIntervalRow)
        self.ucsc_ensGene_gene = sqlgraph.SQLTable('%s.ensGene' % self.ucsc_db,
                                                   serverInfo=self.ucsc_server,
                                                   primaryKey='name2',
                                                  itemClass=UCSCSeqIntervalRow)
        self.ucsc_ensGtp = sqlgraph.SQLTable('%s.ensGtp' % self.ucsc_db,
                                             serverInfo=self.ucsc_server,
                                             primaryKey='protein')
        self.ucsc_ensPep = sqlgraph.SQLTable('%s.ensPep' % self.ucsc_db,
                                             serverInfo=self.ucsc_server,
                                             itemClass=UCSCProteinSeq,
                                             itemSliceClass=seqdb.SeqDBSlice)
        self.ens_exon_stable_id = sqlgraph.SQLTable('%s.exon_stable_id'
                                                    % self.ens_db,
                                                    serverInfo=self.ens_server,
                                                    primaryKey='stable_id')
        self.ens_transcript_stable_id = sqlgraph.SQLTable(
            '%s.transcript_stable_id' % self.ens_db,
            serverInfo=self.ens_server, primaryKey='stable_id')
        # Mappings.
        self.protein_transcript_id_map = sqlgraph.MapView(self.ucsc_ensGtp,
            self.ucsc_ensGene_trans, 'select transcript from %s.ensGtp \
            where protein=%%s' % self.ucsc_db, inverseSQL='select protein \
            from %s.ensGtp where transcript=%%s' % self.ucsc_db)
        self.transcripts_in_genes_map = sqlgraph.GraphView(
            self.ucsc_ensGtp, self.ucsc_ensGene_gene,
            "select transcript from %s.ensGtp where gene=%%s" % self.ucsc_db)
        self.ens_transcripts_of_exons_map = sqlgraph.GraphView(
            self.ens_exon_stable_id, self.ucsc_ensGene_trans, """\
select trans.stable_id from %s.exon_stable_id exon, \
%s.transcript_stable_id trans, %s.exon_transcript et where \
exon.exon_id=et.exon_id and trans.transcript_id=et.transcript_id and \
exon.stable_id=%%s""" % (self.ens_db, self.ens_db, self.ens_db))
        self.ens_exons_in_transcripts_map = sqlgraph.GraphView(
            self.ens_transcript_stable_id, self.ens_exon_stable_id, """\
select exon.stable_id from %s.exon_stable_id exon, %s.transcript_stable_id \
trans, %s.exon_transcript et where exon.exon_id=et.exon_id and \
trans.transcript_id=et.transcript_id and trans.stable_id=%%s order by \
et.rank""" % (self.ens_db, self.ens_db, self.ens_db))
        # We will need this too.
        self.genome_seq = worldbase(ucsc_genome_name)
        protein_slicedb = EnsemblProteinSliceDB('%s.ensGene' %
                                                self.ucsc_db,
                                                serverInfo=self.ucsc_server,
                                                primaryKey='name',
                                                itemClass=UCSCSeqIntervalRow)
        exon_slicedb = EnsemblExonOnDemandSliceDB()
        # Finally, initialise all UCSC-Ensembl databases.
        self.trans_db = annotation.AnnotationDB(self.ucsc_ensGene_trans,
                                                self.genome_seq,
                                                checkFirstID=False,
                                                sliceAttrDict=dict(
                                                    id='chrom',
                                                    start='txStart',
                                                    stop='txEnd'),
                                      itemClass=EnsemblTranscriptAnnotationSeq)
        self.gene_db = annotation.AnnotationDB(self.ucsc_ensGene_gene,
                                               self.genome_seq,
                                               checkFirstID=False,
                                               sliceAttrDict=dict(
                                                   id='chrom',
                                                   start='txStart',
                                                   stop='txEnd'))
        self.prot_db = annotation.AnnotationDB(protein_slicedb,
                                               self.genome_seq,
                                               checkFirstID=False,
                                               sliceAttrDict=dict(
                                                   id='chrom',
                                                   start='txStart',
                                                   stop='txEnd'))
        self.exon_db = annotation.AnnotationDB(exon_slicedb,
                                               self.genome_seq,
                                               checkFirstID=False)

    def get_ensembl_db_name(self, ens_prefix):
        '''Used by __init__(), obtains Ensembl database name matching
        the specified UCSC genome version'''
        ucsc_versions = sqlgraph.SQLTable('hgFixed.trackVersion',
                                          serverInfo=self.ucsc_server,
                                          primaryKey='db')
        ens_version = ucsc_versions[self.ucsc_db].version
        if ens_prefix is None:
            # Note: this assumes 'source' in hgFixed.trackVersion contains
            # the URI of the Ensembl data set and that the last path component
            # of that URI is the species name of that data set.
            ens_prefix = ucsc_versions[self.ucsc_db].source.split('/')[-2]
        cursor = self.ens_server.cursor()
        cursor.execute("show databases like '%s_core_%s_%%'" % (ens_prefix,
                                                                ens_version))
        return cursor.fetchall()[0][0]

    def get_ensembl_exon_ids(self, transcript_id):
        '''Obtain a list of stable IDs of exons associated with the
        specified transcript, ordered by rank.'''
        matching_edges = self.ens_exons_in_transcripts_map[
            self.ens_transcript_stable_id[transcript_id]]
        ids = []
        for exon in matching_edges.keys():
            ids.append(exon.stable_id)
        return ids

    def transcript_database(self):
        'Return an AnnotationDB of transcript annotations.'
        return self.trans_db

    def gene_database(self):
        'Return an AnnotationDB of gene annotations.'
        return self.gene_db

    def protein_database(self):
        'Return an AnnotationDB of protein annotations.'
        return self.prot_db

    def exon_database(self):
        'Return an AnnotationDB of exon annotations.'
        return self.exon_db

class EnsemblTranscriptAnnotationSeqDescr(object):

    def __get__(self, obj, objtype):
        '''Concatenate exon sequences of a transcript to obtain
        its sequence.'''
        exon_count = obj.exonCount
        exon_starts = obj.exonStarts.split(',')[:exon_count]
        exon_ends = obj.exonEnds.split(',')[:exon_count]
        trans_seq = ''
        for i in range(0, exon_count):
            trans_seq += str(sequence.absoluteSlice(obj._anno_seq,
                                                    int(exon_starts[i]),
                                                    int(exon_ends[i])))
        return sequence.Sequence(trans_seq, obj.name)   # FIXME: cache this?

class EnsemblTranscriptAnnotationExonDescr(object):

    def __get__(self, obj, objtype):
        'Return a list of exons contained in this transcript.'
        exon_ids = gRes.get_ensembl_exon_ids(obj.name)
        return exon_ids

class EnsemblTranscriptAnnotationSeq(annotation.AnnotationSeq):
    '''An AnnotationSeq class for transcript annotations, implementing
    custom 'sequence' and 'children' properties.'''
    sequence = EnsemblTranscriptAnnotationSeqDescr()
    children = EnsemblTranscriptAnnotationExonDescr()

class EnsemblProteinSliceDB(sqlgraph.SQLTable):
    '''A sliceDB class for protein annotations. Basically, an SQLTable
    pointing to transcript data along with transparent mapping of keys.'''

    def __getitem__(self, k):
        tid = gRes.protein_transcript_id_map[gRes.ucsc_ensGtp[k]].name
        return sqlgraph.SQLTable.__getitem__(self, tid)

    def keys(self):
        prot_keys = []
        trans_keys = SQLTable.keys(self)
        for tid in trans_keys:
            pid = (~gRes.protein_transcript_id_map[
                gRes.ucsc_ensGene[tid]]).name
            prot_keys.append(pid)
        return prot_keys

class EnsemblExonSliceInfo(object):

    def __init__(self, id, start, stop, orientation, parents=None,
                 children=None):
        self.id = id
        self.start = start
        self.stop = stop
        self.orientation = orientation
        self.parents = parents
        self.children = children

class EnsemblExonOnDemandSliceDB(object, UserDict.DictMixin):

    def __init__(self):
        self.data = {}

    def __getitem__(self, k):
        try:
            return self.data[k]
        except KeyError:
            # Not cached yet, extract the exon from transcript data.
            transcripts = gRes.ens_transcripts_of_exons_map[
                gRes.ens_exon_stable_id[k]].keys()
            transcript_exons = self.get_transcript_exons(transcripts[0])
            # Cache all exons from that transcript to save time in the future.
            for exon_id in transcript_exons:
                if exon_id not in self.data:
                    transcript_exons[exon_id].parents = transcripts
                    self.data[exon_id] = transcript_exons[exon_id]
            gRes.genome_seq.cacheHint({transcripts[0].id:
                                           (transcripts[0].txStart,
                                            transcripts[0].txEnd)},
                                          transcripts[0])
            return self.data[k]

    def __setitem__(self, k, v):
        '''Method required by UserDict.DictMixin. Throws an exception
        (read-only sliceDB).'''
        raise ReadOnlyError('EnsemblExonOnDemandSliceDB is read-only')

    def __delitem__(self, k):
        '''Method required by UserDict.DictMixin. Throws an exception
        (read-only sliceDB).'''
        raise ReadOnlyError('EnsemblExonOnDemandSliceDB is read-only')

    def keys(self):
        'Returns keys present in the cache. FIXME: add support for SQL ones?'
        return self.data.keys()

    def get_transcript_exons(self, transcript):
        '''Parse the provided transcript, extract exon data from it
        and return it as a dictionary of slices.'''
        chromosome = transcript.chrom
        exon_count = transcript.exonCount
        exon_starts = transcript.exonStarts.split(',')[:exon_count]
        exon_ends = transcript.exonEnds.split(',')[:exon_count]
        exons = {}
        exon_ids = gRes.get_ensembl_exon_ids(transcript.name)
        for i in range(0, exon_count):
            e = EnsemblExonSliceInfo(chromosome, exon_starts[i], exon_ends[i],
                                     transcript.orientation)
            exons[exon_ids[i]] = e
        return exons
<commit_msg>Modified gene-annotation code to add three new attributes, (to be) useful whenever a gene is related to by multiple transcripts: minTxStart, maxTxEnd and children (the latter returning a list of associated transcripts). By the way, the first two should be achievable by the attrAlias keyword of SQLTable - I've had problems getting it to work correctly, though.<commit_after>import UserDict

from pygr import annotation, seqdb, sequence, sqlgraph, worldbase
from pygr.dbfile import ReadOnlyError

gRes = None

class UCSCStrandDescr(object):

    def __get__(self, obj, objtype):
        if obj.strand == '+':
            return 1
        else:
            return -1

class UCSCSeqIntervalRow(sqlgraph.TupleO):
    orientation = UCSCStrandDescr()

class UCSCGeneIntervalRow(sqlgraph.TupleO):
    orientation = UCSCStrandDescr()

    def __init__(self, *args, **kwargs):
        sqlgraph.TupleO.__init__(self, *args, **kwargs)
        # FIXME: this should be doable with attrAlias...
        cursor = gRes.ucsc_server.cursor()
        cursor.execute("select min(txStart) from %s.ensGene where name2='%s'" % (gRes.ucsc_db, self.name2))
        self.minTxStart = cursor.fetchall()[0][0]
        cursor.execute("select max(txEnd) from %s.ensGene where name2='%s'" % (gRes.ucsc_db, self.name2))
        self.maxTxEnd = cursor.fetchall()[0][0]
        self.children = gRes.get_gene_transcript_ids(self.name2)

class UCSCProteinSeq(sqlgraph.ProteinSQLSequence):
    '''Representation of UCSC protein-sequence tables such as ensGene,
    which lack the length column.'''

    def __len__(self):
        return self._select('length(seq)')

class UCSCEnsemblInterface(object):

    def __init__(self, ucsc_genome_name, ens_species=None,
                 ucsc_serverInfo=None, ens_serverInfo=None):
        '''Set up everything needed to produce UCSC/Ensembl
        annotation databases. ucsc_genome_name should follow the worldbase
        naming convention. If ens_species is not specified, we will try
        to autodetect it.'''
        # Only one instance can be active at a time for now.
        global gRes
        if gRes is not None:
            raise ValueError("A UCSCEnsemblInterface object already exists")
        else:
            gRes = self
        # Connect to both servers and prepare database names.
        if ucsc_serverInfo is not None:
            self.ucsc_server = ucsc_serverInfo
        else:
            self.ucsc_server = sqlgraph.DBServerInfo(
                host='genome-mysql.cse.ucsc.edu', user='genome')
        if ens_serverInfo is not None:
            self.ens_server = ens_serverInfo
        else:
            self.ens_server = sqlgraph.DBServerInfo(
                host='ensembldb.ensembl.org', port=5306, user='anonymous')
        self.ucsc_db = ucsc_genome_name.split('.')[-1]
        self.ens_db = self.get_ensembl_db_name(ens_species)
        # Connect to all the necessary tables.
        self.ucsc_ensGene_trans = sqlgraph.SQLTable('%s.ensGene' %
                                                    self.ucsc_db,
                                                   serverInfo=self.ucsc_server,
                                                    primaryKey='name',
                                                  itemClass=UCSCSeqIntervalRow)
        self.ucsc_ensGene_gene = sqlgraph.SQLTable('%s.ensGene' % self.ucsc_db,
                                                   serverInfo=self.ucsc_server,
                                                   primaryKey='name2',
                                                  itemClass=UCSCGeneIntervalRow)
        self.ucsc_ensGtp_gene = sqlgraph.SQLTable('%s.ensGtp' % self.ucsc_db,
                                             serverInfo=self.ucsc_server,
                                             primaryKey='gene')
        self.ucsc_ensGtp_prot = sqlgraph.SQLTable('%s.ensGtp' % self.ucsc_db,
                                             serverInfo=self.ucsc_server,
                                             primaryKey='protein')
        self.ucsc_ensPep = sqlgraph.SQLTable('%s.ensPep' % self.ucsc_db,
                                             serverInfo=self.ucsc_server,
                                             itemClass=UCSCProteinSeq,
                                             itemSliceClass=seqdb.SeqDBSlice)
        self.ens_exon_stable_id = sqlgraph.SQLTable('%s.exon_stable_id'
                                                    % self.ens_db,
                                                    serverInfo=self.ens_server,
                                                    primaryKey='stable_id')
        self.ens_transcript_stable_id = sqlgraph.SQLTable(
            '%s.transcript_stable_id' % self.ens_db,
            serverInfo=self.ens_server, primaryKey='stable_id')
        # Mappings.
        self.protein_transcript_id_map = sqlgraph.MapView(self.ucsc_ensGtp_prot,
            self.ucsc_ensGene_trans, 'select transcript from %s.ensGtp \
            where protein=%%s' % self.ucsc_db, inverseSQL='select protein \
            from %s.ensGtp where transcript=%%s' % self.ucsc_db)
        self.transcripts_in_genes_map = sqlgraph.GraphView(
            self.ucsc_ensGtp_gene, self.ucsc_ensGene_trans,
            "select transcript from %s.ensGtp where gene=%%s" % self.ucsc_db)
        self.ens_transcripts_of_exons_map = sqlgraph.GraphView(
            self.ens_exon_stable_id, self.ucsc_ensGene_trans, """\
select trans.stable_id from %s.exon_stable_id exon, \
%s.transcript_stable_id trans, %s.exon_transcript et where \
exon.exon_id=et.exon_id and trans.transcript_id=et.transcript_id and \
exon.stable_id=%%s""" % (self.ens_db, self.ens_db, self.ens_db))
        self.ens_exons_in_transcripts_map = sqlgraph.GraphView(
            self.ens_transcript_stable_id, self.ens_exon_stable_id, """\
select exon.stable_id from %s.exon_stable_id exon, %s.transcript_stable_id \
trans, %s.exon_transcript et where exon.exon_id=et.exon_id and \
trans.transcript_id=et.transcript_id and trans.stable_id=%%s order by \
et.rank""" % (self.ens_db, self.ens_db, self.ens_db))
        # We will need this too.
        self.genome_seq = worldbase(ucsc_genome_name)
        protein_slicedb = EnsemblProteinSliceDB('%s.ensGene' %
                                                self.ucsc_db,
                                                serverInfo=self.ucsc_server,
                                                primaryKey='name',
                                                itemClass=UCSCSeqIntervalRow)
        exon_slicedb = EnsemblExonOnDemandSliceDB()
        # Finally, initialise all UCSC-Ensembl databases.
        self.trans_db = annotation.AnnotationDB(self.ucsc_ensGene_trans,
                                                self.genome_seq,
                                                checkFirstID=False,
                                                sliceAttrDict=dict(
                                                    id='chrom',
                                                    start='txStart',
                                                    stop='txEnd'),
                                      itemClass=EnsemblTranscriptAnnotationSeq)
        self.gene_db = annotation.AnnotationDB(self.ucsc_ensGene_gene,
                                               self.genome_seq,
                                               checkFirstID=False,
                                               sliceAttrDict=dict(
                                                   id='chrom',
                                                   start='txStart',
                                                   stop='txEnd'))
        self.prot_db = annotation.AnnotationDB(protein_slicedb,
                                               self.genome_seq,
                                               checkFirstID=False,
                                               sliceAttrDict=dict(
                                                   id='chrom',
                                                   start='txStart',
                                                   stop='txEnd'))
        self.exon_db = annotation.AnnotationDB(exon_slicedb,
                                               self.genome_seq,
                                               checkFirstID=False)

    def get_ensembl_db_name(self, ens_prefix):
        '''Used by __init__(), obtains Ensembl database name matching
        the specified UCSC genome version'''
        ucsc_versions = sqlgraph.SQLTable('hgFixed.trackVersion',
                                          serverInfo=self.ucsc_server,
                                          primaryKey='db')
        ens_version = ucsc_versions[self.ucsc_db].version
        if ens_prefix is None:
            # Note: this assumes 'source' in hgFixed.trackVersion contains
            # the URI of the Ensembl data set and that the last path component
            # of that URI is the species name of that data set.
            ens_prefix = ucsc_versions[self.ucsc_db].source.split('/')[-2]
        cursor = self.ens_server.cursor()
        cursor.execute("show databases like '%s_core_%s_%%'" % (ens_prefix,
                                                                ens_version))
        return cursor.fetchall()[0][0]

    def get_ensembl_exon_ids(self, transcript_id):
        '''Obtain a list of stable IDs of exons associated with the
        specified transcript, ordered by rank.'''
        matching_edges = self.ens_exons_in_transcripts_map[
            self.ens_transcript_stable_id[transcript_id]]
        ids = []
        for exon in matching_edges.keys():
            ids.append(exon.stable_id)
        return ids

    def get_gene_transcript_ids(self, gene_id):
        '''Obtain a list of stable IDs of transcripts associated
        with the specified gene.'''
        matching_edges = self.transcripts_in_genes_map[
            self.ucsc_ensGtp_gene[gene_id]]
        ids = []
        for transcript in matching_edges.keys():
            ids.append(transcript.name)
        return ids

    def transcript_database(self):
        'Return an AnnotationDB of transcript annotations.'
        return self.trans_db

    def gene_database(self):
        'Return an AnnotationDB of gene annotations.'
        return self.gene_db

    def protein_database(self):
        'Return an AnnotationDB of protein annotations.'
        return self.prot_db

    def exon_database(self):
        'Return an AnnotationDB of exon annotations.'
        return self.exon_db

class EnsemblTranscriptAnnotationSeqDescr(object):

    def __get__(self, obj, objtype):
        '''Concatenate exon sequences of a transcript to obtain
        its sequence.'''
        exon_count = obj.exonCount
        exon_starts = obj.exonStarts.split(',')[:exon_count]
        exon_ends = obj.exonEnds.split(',')[:exon_count]
        trans_seq = ''
        for i in range(0, exon_count):
            trans_seq += str(sequence.absoluteSlice(obj._anno_seq,
                                                    int(exon_starts[i]),
                                                    int(exon_ends[i])))
        return sequence.Sequence(trans_seq, obj.name)   # FIXME: cache this?

class EnsemblTranscriptAnnotationExonDescr(object):

    def __get__(self, obj, objtype):
        'Return a list of exons contained in this transcript.'
        exon_ids = gRes.get_ensembl_exon_ids(obj.name)
        return exon_ids

class EnsemblTranscriptAnnotationSeq(annotation.AnnotationSeq):
    '''An AnnotationSeq class for transcript annotations, implementing
    custom 'sequence' and 'children' properties.'''
    sequence = EnsemblTranscriptAnnotationSeqDescr()
    children = EnsemblTranscriptAnnotationExonDescr()

class EnsemblProteinSliceDB(sqlgraph.SQLTable):
    '''A sliceDB class for protein annotations. Basically, an SQLTable
    pointing to transcript data along with transparent mapping of keys.'''

    def __getitem__(self, k):
        tid = gRes.protein_transcript_id_map[gRes.ucsc_ensGtp_prot[k]].name
        return sqlgraph.SQLTable.__getitem__(self, tid)

    def keys(self):
        prot_keys = []
        trans_keys = SQLTable.keys(self)
        for tid in trans_keys:
            pid = (~gRes.protein_transcript_id_map[
                gRes.ucsc_ensGene[tid]]).name
            prot_keys.append(pid)
        return prot_keys

class EnsemblExonSliceInfo(object):

    def __init__(self, id, start, stop, orientation, parents=None,
                 children=None):
        self.id = id
        self.start = start
        self.stop = stop
        self.orientation = orientation
        self.parents = parents
        self.children = children

class EnsemblExonOnDemandSliceDB(object, UserDict.DictMixin):

    def __init__(self):
        self.data = {}

    def __getitem__(self, k):
        try:
            return self.data[k]
        except KeyError:
            # Not cached yet, extract the exon from transcript data.
            transcripts = gRes.ens_transcripts_of_exons_map[
                gRes.ens_exon_stable_id[k]].keys()
            transcript_exons = self.get_transcript_exons(transcripts[0])
            # Cache all exons from that transcript to save time in the future.
            for exon_id in transcript_exons:
                if exon_id not in self.data:
                    transcript_exons[exon_id].parents = transcripts
                    self.data[exon_id] = transcript_exons[exon_id]
            gRes.genome_seq.cacheHint({transcripts[0].id:
                                           (transcripts[0].txStart,
                                            transcripts[0].txEnd)},
                                          transcripts[0])
            return self.data[k]

    def __setitem__(self, k, v):
        '''Method required by UserDict.DictMixin. Throws an exception
        (read-only sliceDB).'''
        raise ReadOnlyError('EnsemblExonOnDemandSliceDB is read-only')

    def __delitem__(self, k):
        '''Method required by UserDict.DictMixin. Throws an exception
        (read-only sliceDB).'''
        raise ReadOnlyError('EnsemblExonOnDemandSliceDB is read-only')

    def keys(self):
        'Returns keys present in the cache. FIXME: add support for SQL ones?'
        return self.data.keys()

    def get_transcript_exons(self, transcript):
        '''Parse the provided transcript, extract exon data from it
        and return it as a dictionary of slices.'''
        chromosome = transcript.chrom
        exon_count = transcript.exonCount
        exon_starts = transcript.exonStarts.split(',')[:exon_count]
        exon_ends = transcript.exonEnds.split(',')[:exon_count]
        exons = {}
        exon_ids = gRes.get_ensembl_exon_ids(transcript.name)
        for i in range(0, exon_count):
            e = EnsemblExonSliceInfo(chromosome, exon_starts[i], exon_ends[i],
                                     transcript.orientation)
            exons[exon_ids[i]] = e
        return exons
<|endoftext|>


<commit_before># Copyright © 2017-2019 Zuercher Hochschule fuer Angewandte Wissenschaften.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import os
import shutil
from typing import Dict

from compose.cli.main import TopLevelCommand, project_from_options
# from kubernetes import client, config

from adapters.log import LOG
# from adapters.datasource import STORE

class Backend(object):
    def create(self, instance_id: str, content: str, c_type: str, **kwargs) -> None:
        pass

    def info(self, instance_id: str, **kwargs) -> Dict[str, str]:
        pass

    def delete(self, instance_id: str, **kwargs) -> None:
        pass

class DockerBackend(Backend):
    def __init__(self) -> None:
        super().__init__()
        self.options = {
            "--no-deps": False,
            "--abort-on-container-exit": False,  # do not set to True
            "SERVICE": "",
            "--remove-orphans": False,
            "--no-recreate": True,
            "--force-recreate": False,
            "--build": False,
            '--no-build': False,
            '--no-color': False,
            # "--rmi": "none",
            "--volumes": "/private",
            # "--follow": False,
            # "--timestamps": False,
            # "--tail": "all",
            "-d": True,
            "-q": False,  # quiet
            "--scale": [],  # don't need this but have to supply
        }

        # TODO this location needs to be configurable
        self.manifest_cache = '/tmp'

    def create(self, instance_id: str, content: str, c_type: str, **kwargs) -> None:
        if c_type == 'docker-compose':
            LOG.warn('WARNING: this is for local-only deployments.')
            return self._create_compose(inst_id=instance_id, content=content)
        else:
            raise NotImplementedError('The type ({type}) of cluster manager is unknown'.format(type=c_type))

    def _create_compose(self, inst_id: str, content: str) -> None:
        """
        This creates a set of containers using docker compose.
        Note: the use of the docker compose python module is unsupported by docker inc.
        :param content: the docker compose file as a string
        :return:
        """

        # XXX this is a dirty hack - mongodb strips newlines!
        LOG.debug("# content = content.replace('
', '\\n') is a dirty hack - mongodb strips newlines!")
        content = content.replace('
', '\n')
        LOG.debug(content)

        # when we get the manifest, we have to dump it to a temporary file
        # to allow for multiple stack instances we need to have multiple projects!
        # this means multiple directories
        mani_dir = self.manifest_cache + '/' + inst_id

        if not os.path.exists(mani_dir):
            os.makedirs(mani_dir)
        else:
            LOG.info('The instance is already running with the following project: {mani_dir}'.format(mani_dir=mani_dir))

        LOG.debug('writing to: {compo}'.format(compo=mani_dir + '/docker-compose.yml'))
        m = open(mani_dir + '/docker-compose.yml', 'wt')
        m.write(content)
        m.close()
        project = project_from_options(mani_dir, self.options)
        cmd = TopLevelCommand(project)
        cmd.up(self.options)  # WARNING: this method can call sys.exit() but only if --abort-on-container-exit is True

    def info(self, instance_id: str, **kwargs) -> Dict[str, str]:
        mani_dir = self.manifest_cache + '/' + instance_id

        if not os.path.exists(mani_dir):
            LOG.error('requested directory does not exist: {mani_dir}'.format(mani_dir=mani_dir))
            # TODO raise exception
            return

        LOG.debug('info from: {compo}'.format(compo=mani_dir + '/docker-compose.yml'))
        project = project_from_options(mani_dir, self.options)
        # cmd = TopLevelCommand(project)

        containers = project.containers(service_names=self.options['SERVICE'], stopped=True)

        # containers = sorted(
        #     self.project.containers(service_names=options['SERVICE'], stopped=True) +
        #     self.project.containers(service_names=options['SERVICE'], one_off=OneOffFilter.only),
        #     key=attrgetter('name'))

        # XXX questionable flattening of information - is it necessary? looks fugly...
        info = dict()
        for c in containers:
            LOG.debug('{name} container image name: {img_name}'
                      .format(name=c.name, img_name=c.image_config['RepoTags'][0]))
            info[c.name+'_image_name'] = c.image_config['RepoTags'][0]
            LOG.debug('{name} container image: {img}'.format(name=c.name, img=c.image))
            info[c.name + '_image_id'] = c.image

            LOG.debug(c.ports)

            for k, v in c.ports.items():
                if isinstance(v, list):
                    for i in v:
                        info[c.name + '_' + k + '/HostIp'] = i['HostIp']
                        info[c.name + '_' + k + '/HostPort'] = i['HostPort']
                else:
                    info[c.name + '_' + k] = v

            LOG.debug('{name} container command: {cmd}'.format(name=c.name, cmd=c.human_readable_command))
            info[c.name + '_cmd'] = c.human_readable_command

            LOG.debug('{name} container state: {state}'.format(name=c.name, state=c.human_readable_state))
            info[c.name + '_state'] = c.human_readable_state

        states = set([v for k, v in info.items() if k.endswith('state')])

        # states from compose.container.Container: 'Paused', 'Restarting', 'Ghost', 'Up', 'Exit %s'
        # states for OSBA: in progress, succeeded, and failed
        for state in states:
            if state.startswith('Exit'):
                # there's been an error with docker
                info['srv_inst.state.state'] = 'failed'
                info['srv_inst.state.description'] = 'There was an error in creating the instance {error}'.format(error=state)
                # return 'Error with docker: {error}'.format(error=state), 500

        if len(states) == 1:  # if all states of the same value
            if states.pop() == 'Up':  # if running: Up
                info['srv_inst.state.state'] = 'succeeded'
                info['srv_inst.state.description'] = 'The service instance has been created successfully'
        else:
            # still waiting for completion
            info['srv_inst.state.state'] = 'in progress'
            info['srv_inst.state.description'] = 'The service instance is being created.'

        return info

    def delete(self, instance_id: str, **kwargs) -> None:
        mani_dir = self.manifest_cache + '/' + instance_id

        if not os.path.exists(mani_dir):
            LOG.error('requested directory does not exist: {mani_dir}'.format(mani_dir=mani_dir))
            # TODO raise exception
            return

        self.options["--force"] = True
        self.options["--rmi"] = "none"
        LOG.info('destroying: {compo}'.format(compo=mani_dir + '/docker-compose.yml'))
        project = project_from_options(mani_dir, self.options)
        cmd = TopLevelCommand(project)
        cmd.down(self.options)
        try:
            shutil.rmtree(mani_dir)
        except PermissionError:
            # Done to let travis pass
            # TODO make location of mani_dir configurable
            LOG.warn('Could not delete the directory {dir}'.format(dir=mani_dir))

class KubernetesBackend(Backend):
    def __init__(self) -> None:
        super().__init__()
#         # self.directory = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'data'))
#         # prefix = "k8s_"
#         # Identify all kubernetes files to process
#         # self.files = [os.path.join(self.directory, filename) for filename in os.listdir(self.directory)
#         #               if filename.startswith(prefix) and filename.endswith(".yaml")]
#         # k8s_config_file = os.path.abspath(os.path.join(self.directory, 'k8s_config'))
#         # self.api = pykube.HTTPClient(pykube.KubeConfig.from_file(k8s_config_file))
#         config.load_kube_config()
#         self.k8s_beta = client.ExtensionsV1beta1Api()

    def create(self, instance_id: str, content: str, c_type: str, **kwargs) -> None:
        super().create(instance_id, content, c_type)
#
#         # LOG.debug("# content = content.replace('
', '\\n') is a dirty hack - mongodb strips newlines!")
#         # content = content.replace('
', '\n')
#         #
#         # dep = yaml.load(content)
#         # resp = self.k8s_beta.create_namespaced_deployment(body=dep, namespace="default")
#         # print("Deployment created. status='%s'" % str(resp.metadata.uid))
#         # print("Deployment created. status='%s'" % str(resp.status))
#         # return resp.metadata.uid

    def info(self, instance_id: str, **kwargs) -> Dict[str, str]:
        super().info(instance_id)

    def delete(self, instance_id: str, **kwargs) -> None:
        super().delete(instance_id)
#         # content = ''
#         # dep = yaml.load(content)
#         # resp = self.k8s_beta.delete_collection_namespaced_deployment(body=dep, namespace="default")

class DummyBackend(Backend):
    def __init__(self) -> None:
        super().__init__()

    def create(self, instance_id: str, content: str, c_type: str, **kwargs) -> None:
        super().create(instance_id, content, c_type)
        LOG.info('DummyBackend driver: create called')

    def info(self, instance_id: str, **kwargs) -> Dict[str, str]:
        super().info(instance_id)
        LOG.info('DummyBackend driver: info called')

        info = {
            'testid123_spark-worker_1_image_name': 'elastest/ebs-spark-base:0.5.0',
            'testid123_spark-worker_1_image_id': 'sha256:138a91572bd6bdce7d7b49a44b91a4caf4abdf1a75f105991e18be971353d5cb',
            'testid123_spark-worker_1_8080/tcp': None,
            'testid123_spark-worker_1_8081/tcp/HostIp': '0.0.0.0',
            'testid123_spark-worker_1_8081/tcp/HostPort': '32784',
            'testid123_spark-worker_1_cmd': '/usr/bin/supervisord --configuration=/opt/conf/slave.conf',
            'testid123_spark-worker_1_state': 'Up',
            'spark-master_image_name': 'elastest/ebs-spark-base:0.5.0',
            'spark-master_image_id': 'sha256:138a91572bd6bdce7d7b49a44b91a4caf4abdf1a75f105991e18be971353d5cb',
            'spark-master_8080/tcp/HostIp': '0.0.0.0',
            'spark-master_8080/tcp/HostPort': '8080',
            'spark-master_cmd': '/usr/bin/supervisord --configuration=/opt/conf/master.conf',
            'spark-master_state': 'Up',
            'srv_inst.state.state': 'succeeded',
            'srv_inst.state.description': 'The service instance has been created successfully'
        }

        # TODO FIXME below until return is duplicated code!!!!
        states = set([v for k, v in info.items() if k.endswith('state')])

        # states from compose.container.Container: 'Paused', 'Restarting', 'Ghost', 'Up', 'Exit %s'
        # states for OSBA: in progress, succeeded, and failed
        for state in states:
            if state.startswith('Exit'):
                # there's been an error with docker
                info['srv_inst.state.state'] = 'failed'
                info['srv_inst.state.description'] = 'There was an error in creating the instance {error}'.format(
                    error=state)
                # return 'Error with docker: {error}'.format(error=state), 500

        if len(states) == 1:  # if all states of the same value
            if states.pop() == 'Up':  # if running: Up
                info['srv_inst.state.state'] = 'succeeded'
                info['srv_inst.state.description'] = 'The service instance has been created successfully'
        else:
            # still waiting for completion
            info['srv_inst.state.state'] = 'in progress'
            info['srv_inst.state.description'] = 'The service instance is being created.'

        LOG.info('Dummy data:\n{dummy}'.format(dummy=info))
        return info

    def delete(self, instance_id: str, **kwargs) -> None:
        super().delete(instance_id)
        LOG.info('DummyBackend driver: delete called')

class ResourceManager(Backend):

    def __init__(self) -> None:
        self.backends = {
            'docker': DockerBackend(),
            'kubernetes': KubernetesBackend(),
            'dummy': DummyBackend(),
        }
        # set an alias to the docker-compose driver
        LOG.info('Adding docker-compose alias to DockerBackend')
        self.backends['docker-compose'] = self.backends.get('docker')
        LOG.info('Adding k8s alias to KubernetesBackend')
        self.backends['k8s'] = self.backends.get('kubernetes')

    def create(self, instance_id: str, content: str, c_type: str, **kwargs):
        be = self.backends.get(c_type, self.backends['dummy'])
        be.create(instance_id, content, c_type)

    def info(self, instance_id: str, **kwargs) -> Dict[str, str]:
        if 'manifest_type' in kwargs:
            manifest_type = kwargs.get('manifest_type', 'dummy')
            be = self.backends.get(manifest_type, self.backends['dummy'])
        else:
            raise RuntimeError('manifest_type parameter not specified in call to info()')
        return be.info(instance_id)

    def delete(self, instance_id: str, **kwargs):
        if 'manifest_type' in kwargs:
            manifest_type = kwargs.get('manifest_type', 'dummy')
            be = self.backends.get(manifest_type, self.backends['dummy'])
        else:
            raise RuntimeError('manifest_type parameter not specified in call to info()')

        be.delete(instance_id)

    # def _get_manifest_type(self, instance_id: str) -> str:
    #     srv_inst = self.store.get_service_instance(instance_id=instance_id)
    #     # TODO this will fail if srv_inst len is 0
    #     mani_id = srv_inst[0].context['manifest_id']
    #     # TODO this will fail if mani is None
    #     mani = self.store.get_manifest(manifest_id=mani_id)[0]
    #     return mani.manifest_type

RM = ResourceManager()
<commit_msg>adds more type signalling removes error issued by mypy cruft removal<commit_after># Copyright © 2017-2019 Zuercher Hochschule fuer Angewandte Wissenschaften.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import os
import shutil
from typing import Dict

from compose.cli.main import TopLevelCommand, project_from_options
# from kubernetes import client, config

from adapters.log import LOG
# from adapters.datasource import STORE

class Backend(object):
    def create(self, instance_id: str, content: str, c_type: str, **kwargs) -> None:
        pass

    def info(self, instance_id: str, **kwargs) -> Dict[str, str]:
        pass

    def delete(self, instance_id: str, **kwargs) -> None:
        pass

class DockerBackend(Backend):
    def __init__(self) -> None:
        super().__init__()
        self.options = {
            "--no-deps": False,
            "--abort-on-container-exit": False,  # do not set to True
            "SERVICE": "",
            "--remove-orphans": False,
            "--no-recreate": True,
            "--force-recreate": False,
            "--build": False,
            '--no-build': False,
            '--no-color': False,
            # "--rmi": "none",
            "--volumes": "/private",
            # "--follow": False,
            # "--timestamps": False,
            # "--tail": "all",
            "-d": True,
            "-q": False,  # quiet
            "--scale": [],  # don't need this but have to supply
        }

        # TODO this location needs to be configurable
        self.manifest_cache = '/tmp'

    def create(self, instance_id: str, content: str, c_type: str, **kwargs) -> None:
        if c_type == 'docker-compose':
            LOG.warn('WARNING: this is for local-only deployments.')
            return self._create_compose(inst_id=instance_id, content=content)
        else:
            raise NotImplementedError('The type ({type}) of cluster manager is unknown'.format(type=c_type))

    def _create_compose(self, inst_id: str, content: str) -> None:
        """
        This creates a set of containers using docker compose.
        Note: the use of the docker compose python module is unsupported by docker inc.
        :param content: the docker compose file as a string
        :return:
        """

        # XXX this is a dirty hack - mongodb strips newlines!
        LOG.debug("# content = content.replace('
', '\\n') is a dirty hack - mongodb strips newlines!")
        content = content.replace('
', '\n')
        LOG.debug(content)

        # when we get the manifest, we have to dump it to a temporary file
        # to allow for multiple stack instances we need to have multiple projects!
        # this means multiple directories
        mani_dir = self.manifest_cache + '/' + inst_id

        if not os.path.exists(mani_dir):
            os.makedirs(mani_dir)
        else:
            LOG.info('The instance is already running with the following project: {mani_dir}'.format(mani_dir=mani_dir))

        LOG.debug('writing to: {compo}'.format(compo=mani_dir + '/docker-compose.yml'))
        m = open(mani_dir + '/docker-compose.yml', 'wt')
        m.write(content)
        m.close()
        project = project_from_options(mani_dir, self.options)
        cmd = TopLevelCommand(project)
        cmd.up(self.options)  # WARNING: this method can call sys.exit() but only if --abort-on-container-exit is True

    def info(self, instance_id: str, **kwargs) -> Dict[str, str]:
        mani_dir = self.manifest_cache + '/' + instance_id

        if not os.path.exists(mani_dir):
            LOG.warn('requested directory does not exist: {mani_dir}'.format(mani_dir=mani_dir))
            return {}

        LOG.debug('info from: {compo}'.format(compo=mani_dir + '/docker-compose.yml'))
        project = project_from_options(mani_dir, self.options)
        # cmd = TopLevelCommand(project)

        containers = project.containers(service_names=self.options['SERVICE'], stopped=True)

        # containers = sorted(
        #     self.project.containers(service_names=options['SERVICE'], stopped=True) +
        #     self.project.containers(service_names=options['SERVICE'], one_off=OneOffFilter.only),
        #     key=attrgetter('name'))

        # XXX questionable flattening of information - is it necessary? looks fugly...
        info = dict()
        for c in containers:
            LOG.debug('{name} container image name: {img_name}'
                      .format(name=c.name, img_name=c.image_config['RepoTags'][0]))
            info[c.name+'_image_name'] = c.image_config['RepoTags'][0]
            LOG.debug('{name} container image: {img}'.format(name=c.name, img=c.image))
            info[c.name + '_image_id'] = c.image

            LOG.debug(c.ports)

            for k, v in c.ports.items():
                if isinstance(v, list):
                    for i in v:
                        info[c.name + '_' + k + '/HostIp'] = i['HostIp']
                        info[c.name + '_' + k + '/HostPort'] = i['HostPort']
                else:
                    info[c.name + '_' + k] = v

            LOG.debug('{name} container command: {cmd}'.format(name=c.name, cmd=c.human_readable_command))
            info[c.name + '_cmd'] = c.human_readable_command

            LOG.debug('{name} container state: {state}'.format(name=c.name, state=c.human_readable_state))
            info[c.name + '_state'] = c.human_readable_state

        states = set([v for k, v in info.items() if k.endswith('state')])

        # states from compose.container.Container: 'Paused', 'Restarting', 'Ghost', 'Up', 'Exit %s'
        # states for OSBA: in progress, succeeded, and failed
        for state in states:
            if state.startswith('Exit'):
                # there's been an error with docker
                info['srv_inst.state.state'] = 'failed'
                info['srv_inst.state.description'] = 'There was an error in creating the instance {error}'.format(error=state)
                # return 'Error with docker: {error}'.format(error=state), 500

        if len(states) == 1:  # if all states of the same value
            if states.pop() == 'Up':  # if running: Up
                info['srv_inst.state.state'] = 'succeeded'
                info['srv_inst.state.description'] = 'The service instance has been created successfully'
        else:
            # still waiting for completion
            info['srv_inst.state.state'] = 'in progress'
            info['srv_inst.state.description'] = 'The service instance is being created.'

        return info

    def delete(self, instance_id: str, **kwargs) -> None:
        mani_dir = self.manifest_cache + '/' + instance_id

        if not os.path.exists(mani_dir):
            LOG.warn('requested directory does not exist: {mani_dir}'.format(mani_dir=mani_dir))
            # raise Exception('requested directory does not exist: {mani_dir}'.format(mani_dir=mani_dir))
            return

        self.options["--force"] = True
        self.options["--rmi"] = "none"
        LOG.info('destroying: {compo}'.format(compo=mani_dir + '/docker-compose.yml'))
        project = project_from_options(mani_dir, self.options)
        cmd = TopLevelCommand(project)
        cmd.down(self.options)
        try:
            shutil.rmtree(mani_dir)
        except PermissionError:
            # Done to let travis pass
            # TODO make location of mani_dir configurable
            LOG.warn('Could not delete the directory {dir}'.format(dir=mani_dir))

class KubernetesBackend(Backend):
    def __init__(self) -> None:
        super().__init__()
#         # self.directory = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'data'))
#         # prefix = "k8s_"
#         # Identify all kubernetes files to process
#         # self.files = [os.path.join(self.directory, filename) for filename in os.listdir(self.directory)
#         #               if filename.startswith(prefix) and filename.endswith(".yaml")]
#         # k8s_config_file = os.path.abspath(os.path.join(self.directory, 'k8s_config'))
#         # self.api = pykube.HTTPClient(pykube.KubeConfig.from_file(k8s_config_file))
#         config.load_kube_config()
#         self.k8s_beta = client.ExtensionsV1beta1Api()

    def create(self, instance_id: str, content: str, c_type: str, **kwargs) -> None:
        super().create(instance_id, content, c_type)
#
#         # LOG.debug("# content = content.replace('
', '\\n') is a dirty hack - mongodb strips newlines!")
#         # content = content.replace('', '\n')
#         #
#         # dep = yaml.load(content)
#         # resp = self.k8s_beta.create_namespaced_deployment(body=dep, namespace="default")
#         # print("Deployment created. status='%s'" % str(resp.metadata.uid))
#         # print("Deployment created. status='%s'" % str(resp.status))
#         # return resp.metadata.uid

    def info(self, instance_id: str, **kwargs) -> Dict[str, str]:
        super().info(instance_id)
        return {}

    def delete(self, instance_id: str, **kwargs) -> None:
        super().delete(instance_id)
#         # content = ''
#         # dep = yaml.load(content)
#         # resp = self.k8s_beta.delete_collection_namespaced_deployment(body=dep, namespace="default")

class DummyBackend(Backend):
    def __init__(self) -> None:
        super().__init__()

    def create(self, instance_id: str, content: str, c_type: str, **kwargs) -> None:
        super().create(instance_id, content, c_type)
        LOG.info('DummyBackend driver: create called')

    def info(self, instance_id: str, **kwargs) -> Dict[str, str]:
        super().info(instance_id)
        LOG.info('DummyBackend driver: info called')

        info = {
            'testid123_spark-worker_1_image_name': 'elastest/ebs-spark-base:0.5.0',
            'testid123_spark-worker_1_image_id': 'sha256:138a91572bd6bdce7d7b49a44b91a4caf4abdf1a75f105991e18be971353d5cb',
            'testid123_spark-worker_1_8080/tcp': None,
            'testid123_spark-worker_1_8081/tcp/HostIp': '0.0.0.0',
            'testid123_spark-worker_1_8081/tcp/HostPort': '32784',
            'testid123_spark-worker_1_cmd': '/usr/bin/supervisord --configuration=/opt/conf/slave.conf',
            'testid123_spark-worker_1_state': 'Up',
            'spark-master_image_name': 'elastest/ebs-spark-base:0.5.0',
            'spark-master_image_id': 'sha256:138a91572bd6bdce7d7b49a44b91a4caf4abdf1a75f105991e18be971353d5cb',
            'spark-master_8080/tcp/HostIp': '0.0.0.0',
            'spark-master_8080/tcp/HostPort': '8080',
            'spark-master_cmd': '/usr/bin/supervisord --configuration=/opt/conf/master.conf',
            'spark-master_state': 'Up',
            'srv_inst.state.state': 'succeeded',
            'srv_inst.state.description': 'The service instance has been created successfully'
        }

        # TODO FIXME below until return is duplicated code!!!!
        states = set([v for k, v in info.items() if k.endswith('state')])

        # states from compose.container.Container: 'Paused', 'Restarting', 'Ghost', 'Up', 'Exit %s'
        # states for OSBA: in progress, succeeded, and failed
        for state in states:
            if state.startswith('Exit'):
                # there's been an error with docker
                info['srv_inst.state.state'] = 'failed'
                info['srv_inst.state.description'] = 'There was an error in creating the instance {error}'.format(
                    error=state)
                # return 'Error with docker: {error}'.format(error=state), 500

        if len(states) == 1:  # if all states of the same value
            if states.pop() == 'Up':  # if running: Up
                info['srv_inst.state.state'] = 'succeeded'
                info['srv_inst.state.description'] = 'The service instance has been created successfully'
        else:
            # still waiting for completion
            info['srv_inst.state.state'] = 'in progress'
            info['srv_inst.state.description'] = 'The service instance is being created.'

        LOG.info('Dummy data:\n{dummy}'.format(dummy=info))
        return info

    def delete(self, instance_id: str, **kwargs) -> None:
        super().delete(instance_id)
        LOG.info('DummyBackend driver: delete called')

class ResourceManager(Backend):

    def __init__(self) -> None:
        self.backends = {
            'docker': DockerBackend(),
            'kubernetes': KubernetesBackend(),
            'dummy': DummyBackend(),
        }
        # set an alias to the docker-compose driver
        LOG.info('Adding docker-compose alias to DockerBackend')
        self.backends['docker-compose'] = self.backends.get('docker')
        LOG.info('Adding k8s alias to KubernetesBackend')
        self.backends['k8s'] = self.backends.get('kubernetes')

    def create(self, instance_id: str, content: str, c_type: str, **kwargs):
        be = self.backends.get(c_type, self.backends['dummy'])
        be.create(instance_id, content, c_type)

    def info(self, instance_id: str, **kwargs) -> Dict[str, str]:
        if 'manifest_type' in kwargs:
            manifest_type = kwargs.get('manifest_type', 'dummy')
            be = self.backends.get(manifest_type, self.backends['dummy'])
        else:
            raise RuntimeError('manifest_type parameter not specified in call to info()')
        return be.info(instance_id)

    def delete(self, instance_id: str, **kwargs):
        if 'manifest_type' in kwargs:
            manifest_type = kwargs.get('manifest_type', 'dummy')
            be = self.backends.get(manifest_type, self.backends['dummy'])
        else:
            raise RuntimeError('manifest_type parameter not specified in call to info()')

        be.delete(instance_id)

RM = ResourceManager()
<|endoftext|>


<commit_before>import json
import bleach
from bottle import Bottle, run, redirect, response, request
import common as c

app = Bottle()

def v2to3(items):
    convert = {
        'plugname': 'plugin_name',
        'name': 'slug',
        'repo': 'server',
        'link': 'dbo_page',
        'slug': 'name',
    }
    data = []
    for item in items:
        if item in convert:
            data.append(convert[item])
        else:
            data.append(item)
    return data

def v3to2(items):
    for item in items:
        if 'dbo_page' in item:
            item['link'] = item['dbo_page']
            del(item['dbo_page'])
        if 'server' in item:
            item['repo'] = item['server']
            del(item['server'])
        if 'plugin_name' in item:
            item['plugname'] = item['plugin_name']
            del(item['plugin_name'])
        if 'slug' in item:
            item['name'] = item['slug']
            del(item['slug'])
        data.append(item)
    return data

@app.hook('before_request')
def set_json_header():
    # We will need these set for everything in the API ;)
    response.set_header('Content-Type', 'application/json')
    response.set_header('Access-Control-Allow-Origin', '*')

@app.get('/')
@app.get('/geninfo')
@app.get('/geninfo/')
def generation_info():
    '''Generation Information
    Returns the generation information as requested.  User can optionall request
    to look X number of versions back.
    '''
    size = c.sint(bleach.clean(request.query.size or None))
    return c.jsonify(c.list_geninfo(size))

@app.get('/<server>/plugins')
@app.get('/<server>/plugins/')
def plugin_list(server=None):
    '''Plugin Listing
    Returns the plugin listing.  Can optionally be limited to a specific server
    binary compatability type.
    '''
    fields = bleach.clean(request.query.fields or 'slug,plugin_name,description').split(','):
    fields = v2to3(fields)
    start = c.sint(bleach.clean(request.query.start or None))
    size = c.sint(bleach.clean(request.query.size or None))
    sort = bleach.clean(request.query.sort or 'slug')
    data = c.list_plugins(server, fields, sort)
    if size is not None and start is not None:
        return c.jsonify(v3to2(data[start:start+size]))
    return c.jsonify(v3to2(data))

@app.get('/<server>/plugin/<slug>')
@app.get('/<server>/plugin/<slug>/')
@app.get('/<server>/plugin/<slug>/<version>')
@app.get('/<server>/plugin/<slug>/<version>/')
def plugin_details(server, slug, version=None):
    '''Plugin Details
    Returns the document for a specific plugin.  Optionally can return only a
    specific version as part of the data as well.
    '''
    fields = bleach.clean(request.query.fields or '').split(','):
    fields = v2to3(fields)
    data = c.plugin_details(server, slug, version, fields)
    return c.jsonify(v3to2([data])[0])

@app.get('/plugins/<server>/<slug>/<version>/download')
def plugin_download(server, slug, version):
    '''Plugin Download Redirector
    Will attempt to redirect to the plugin download for the version specified.
    If no version exists, then it will throw a 404 error.
    '''
    plugin = c.plugin_details()
    if version.lower() == 'latest':
        link = [plugin['versions'][0]['download'],]
    else:
        versions = plugin['versions']
        link = [v['download'] for v in versions if v['version'] == version]
    if len(link) > 0:
        redirect(link[0])
    else:
        raise bottle.HTTPError(404, '{"error": "could not find version"}')

@app.get('/authors')
@app.get('/authors/')
def author_list():
    '''Author Listing
    Returns a full listing of the authors in the system and the number of
    plugins that they have in the database.
    '''
    return c.jsonify([a['name'] for a in c.list_authors()])

@app.get('/<server>/author/<name>')
@app.get('/<server>/author/<name>/')
def author_plugins(server, name):
    '''Author Plugin Listing
    Returns the plugins associated with a specific author.  Optionally can also
    be restricted to a specific server binary compatability.
    '''
    fields = bleach.clean(request.query.fields or 'name,plugname,description').split(','):
    fields = v2to3(fields)
    start = c.sint(bleach.clean(request.query.start or None))
    size = c.sint(bleach.clean(request.query.size or None))
    sort = bleach.clean(request.query.sort or 'slug')
    data = v3to2(c.list_author_plugins(server, name, fields, sort))
    if size is not None and start is not None:
        return c.jsonify(v3to2(data[start:start+size]))
    return c.jsonify(v3to2(data))

@app.get('/categories')
@app.get('/categories/')
def category_list():
    '''Category Listing
    Returns the categories in the database and the number of plugins that each
    category holds.
    '''
    return c.jsonify([a['name'] for a in c.list_categories()])

@app.get('/<server>/category/<name>')
@app.get('/<server>/category/<name>/')
def category_plugins(server, name):
    '''Category Plugin listing
    returns the list of plugins that match a specific category.  Optionally a
    specific server binary compatability can be specified.
    '''
    fields = bleach.clean(request.query.fields or 'name,plugname,description').split(','):
    fields = v2to3(fields)
    start = c.sint(bleach.clean(request.query.start or None))
    size = c.sint(bleach.clean(request.query.size or None))
    sort = bleach.clean(request.query.sort or 'slug')
    data = c.list_category_plugins(server, name, fields, sort))
    if size is not None and start is not None:
        return c.jsonify(v3to2(data[start:start+size]))
    return c.jsonify(v3to2(data))

@app.get('/search/<base>/<field>/<action>/<value>')
@app.get('/search/<base>/<field>/<action>/<value>/')
def search(base=None, field=None, action=None, value=None):
    '''Plugin search
    A generalized search system that accepts both single-criteria get requests
    as well as multi-criteria posts.
    '''
    fields = bleach.clean(request.query.fields or 'name,plugname,description').split(','):
    fields = v2to3(fields)
    start = c.sint(bleach.clean(request.query.start or None))
    size = c.sint(bleach.clean(request.query.size or None))
    sort = bleach.clean(request.query.sort or 'slug')
    field = bleach.clean(field)
    value = bleach.clean(value)
    base = bleach.clean(base)
    if base == 'version':
        field = 'versions.%s' % field
    filters = [
        {'field': field, 'action': action, 'value': value}
    ]
    try:
        data = c.plugin_search(filters, fields, sort)
    except:
        raise bottle.HTTPError(400, '{"error": "invalid search"}')
    else:
        if start is not None and size is not None:
            return c.jsonify(v3to2(data[start:start+size]))
        return c.jsonify(v3to2(data))
<commit_msg>Forgot to remove those pesky :<commit_after>import json
import bleach
from bottle import Bottle, run, redirect, response, request
import common as c

app = Bottle()

def v2to3(items):
    convert = {
        'plugname': 'plugin_name',
        'name': 'slug',
        'repo': 'server',
        'link': 'dbo_page',
        'slug': 'name',
    }
    data = []
    for item in items:
        if item in convert:
            data.append(convert[item])
        else:
            data.append(item)
    return data

def v3to2(items):
    for item in items:
        if 'dbo_page' in item:
            item['link'] = item['dbo_page']
            del(item['dbo_page'])
        if 'server' in item:
            item['repo'] = item['server']
            del(item['server'])
        if 'plugin_name' in item:
            item['plugname'] = item['plugin_name']
            del(item['plugin_name'])
        if 'slug' in item:
            item['name'] = item['slug']
            del(item['slug'])
        data.append(item)
    return data

@app.hook('before_request')
def set_json_header():
    # We will need these set for everything in the API ;)
    response.set_header('Content-Type', 'application/json')
    response.set_header('Access-Control-Allow-Origin', '*')

@app.get('/')
@app.get('/geninfo')
@app.get('/geninfo/')
def generation_info():
    '''Generation Information
    Returns the generation information as requested.  User can optionall request
    to look X number of versions back.
    '''
    size = c.sint(bleach.clean(request.query.size or None))
    return c.jsonify(c.list_geninfo(size))

@app.get('/<server>/plugins')
@app.get('/<server>/plugins/')
def plugin_list(server=None):
    '''Plugin Listing
    Returns the plugin listing.  Can optionally be limited to a specific server
    binary compatability type.
    '''
    fields = bleach.clean(request.query.fields or 'slug,plugin_name,description').split(',')
    fields = v2to3(fields)
    start = c.sint(bleach.clean(request.query.start or None))
    size = c.sint(bleach.clean(request.query.size or None))
    sort = bleach.clean(request.query.sort or 'slug')
    data = c.list_plugins(server, fields, sort)
    if size is not None and start is not None:
        return c.jsonify(v3to2(data[start:start+size]))
    return c.jsonify(v3to2(data))

@app.get('/<server>/plugin/<slug>')
@app.get('/<server>/plugin/<slug>/')
@app.get('/<server>/plugin/<slug>/<version>')
@app.get('/<server>/plugin/<slug>/<version>/')
def plugin_details(server, slug, version=None):
    '''Plugin Details
    Returns the document for a specific plugin.  Optionally can return only a
    specific version as part of the data as well.
    '''
    fields = bleach.clean(request.query.fields or '').split(','):
    fields = v2to3(fields)
    data = c.plugin_details(server, slug, version, fields)
    return c.jsonify(v3to2([data])[0])

@app.get('/plugins/<server>/<slug>/<version>/download')
def plugin_download(server, slug, version):
    '''Plugin Download Redirector
    Will attempt to redirect to the plugin download for the version specified.
    If no version exists, then it will throw a 404 error.
    '''
    plugin = c.plugin_details()
    if version.lower() == 'latest':
        link = [plugin['versions'][0]['download'],]
    else:
        versions = plugin['versions']
        link = [v['download'] for v in versions if v['version'] == version]
    if len(link) > 0:
        redirect(link[0])
    else:
        raise bottle.HTTPError(404, '{"error": "could not find version"}')

@app.get('/authors')
@app.get('/authors/')
def author_list():
    '''Author Listing
    Returns a full listing of the authors in the system and the number of
    plugins that they have in the database.
    '''
    return c.jsonify([a['name'] for a in c.list_authors()])

@app.get('/<server>/author/<name>')
@app.get('/<server>/author/<name>/')
def author_plugins(server, name):
    '''Author Plugin Listing
    Returns the plugins associated with a specific author.  Optionally can also
    be restricted to a specific server binary compatability.
    '''
    fields = bleach.clean(request.query.fields or 'name,plugname,description').split(',')
    fields = v2to3(fields)
    start = c.sint(bleach.clean(request.query.start or None))
    size = c.sint(bleach.clean(request.query.size or None))
    sort = bleach.clean(request.query.sort or 'slug')
    data = v3to2(c.list_author_plugins(server, name, fields, sort))
    if size is not None and start is not None:
        return c.jsonify(v3to2(data[start:start+size]))
    return c.jsonify(v3to2(data))

@app.get('/categories')
@app.get('/categories/')
def category_list():
    '''Category Listing
    Returns the categories in the database and the number of plugins that each
    category holds.
    '''
    return c.jsonify([a['name'] for a in c.list_categories()])

@app.get('/<server>/category/<name>')
@app.get('/<server>/category/<name>/')
def category_plugins(server, name):
    '''Category Plugin listing
    returns the list of plugins that match a specific category.  Optionally a
    specific server binary compatability can be specified.
    '''
    fields = bleach.clean(request.query.fields or 'name,plugname,description').split(',')
    fields = v2to3(fields)
    start = c.sint(bleach.clean(request.query.start or None))
    size = c.sint(bleach.clean(request.query.size or None))
    sort = bleach.clean(request.query.sort or 'slug')
    data = c.list_category_plugins(server, name, fields, sort))
    if size is not None and start is not None:
        return c.jsonify(v3to2(data[start:start+size]))
    return c.jsonify(v3to2(data))

@app.get('/search/<base>/<field>/<action>/<value>')
@app.get('/search/<base>/<field>/<action>/<value>/')
def search(base=None, field=None, action=None, value=None):
    '''Plugin search
    A generalized search system that accepts both single-criteria get requests
    as well as multi-criteria posts.
    '''
    fields = bleach.clean(request.query.fields or 'name,plugname,description').split(',')
    fields = v2to3(fields)
    start = c.sint(bleach.clean(request.query.start or None))
    size = c.sint(bleach.clean(request.query.size or None))
    sort = bleach.clean(request.query.sort or 'slug')
    field = bleach.clean(field)
    value = bleach.clean(value)
    base = bleach.clean(base)
    if base == 'version':
        field = 'versions.%s' % field
    filters = [
        {'field': field, 'action': action, 'value': value}
    ]
    try:
        data = c.plugin_search(filters, fields, sort)
    except:
        raise bottle.HTTPError(400, '{"error": "invalid search"}')
    else:
        if start is not None and size is not None:
            return c.jsonify(v3to2(data[start:start+size]))
        return c.jsonify(v3to2(data))
<|endoftext|>


<commit_before>#!/usr/bin/env python

#_MIT License
#_
#_Copyright (c) 2017 Dan Persons (dpersonsdev@gmail.com)
#_
#_Permission is hereby granted, free of charge, to any person obtaining a copy
#_of this software and associated documentation files (the "Software"), to deal
#_in the Software without restriction, including without limitation the rights
#_to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#_copies of the Software, and to permit persons to whom the Software is
#_furnished to do so, subject to the following conditions:
#_
#_The above copyright notice and this permission notice shall be included in all
#_copies or substantial portions of the Software.
#_
#_THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#_IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#_FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#_AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#_LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#_OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
#_SOFTWARE.

import time
from time import strftime
from time import sleep
from time import daylight
from time import timezone
from time import altzone
from random import randrange
from datetime import datetime
import MySQLdb as mdb
import json
import threading
import os
from sys import exit
#import signal

class SiemTrigger:

    def __init__(self, db, rule):
        """Initialize trigger object"""

        self.db = db
        self.rule = rule
        self.tzone = None

    def watch_rule(self):
        """Watch a trigger rule"""

        # Set time zone:
        if daylight:
            self.tzone = \
                    str(int(float(altzone) / 60 // 60)).rjust(2,
                            '0') + \
                    str(int(float(altzone) / 60 % 60)).ljust(2, '0')
        else:
            self.tzone = \
                    str(int(float(timezone) / 60 // 60)).rjust(2,
                            '0') + \
                    str(int(float(timezone) / 60 % 60)).ljust(2, '0')
        if not '-' in self.tzone:
            self.tzone = '+' + self.tzone

        while True:

            # Check the rule:
            self.check_rule()

            # Wait until the next interval
            sleep(int(self.rule['time_int']) * 60)

    def check_rule(self):
        """Check a trigger rule"""

        if not self.tzone:
            # Set time zone:
            if daylight:
                self.tzone = \
                        str(int(float(altzone) / 60 // 60)).rjust(2,
                                '0') + \
                        str(int(float(altzone) / 60 % 60)).ljust(2, '0')
            else:
                self.tzone = \
                        str(int(float(timezone) / 60 // 60)).rjust(2,
                                '0') + \
                        str(int(float(timezone) / 60 % 60)).ljust(2, '0')
            if not '-' in self.tzone:
                self.tzone = '+' + self.tzone

        # Query the database:
        con = mdb.connect(self.db['host'], self.db['user'],
                self.db['password'], self.db['database'])
        with con:
            cur = con.cursor()
            cur.execute(self.rule['sql_query'])
            rows = cur.fetchall()
            cur.close()
        con.close()

        # Evaluate the results:
        if len(rows) > int(self.rule['event_limit']):
            idtags = json.dumps([int(row[0]) for row in rows])

            datestamp = datetime.now().strftime('%Y%m%d%H%M%S')
            magnitude = ((len(rows) // \
                    (self.rule['event_limit'] + 1) // 2) + 5) * \
                    ( 7 - self.rule['severity'])

            outstatement = 'INSERT INTO ' + \
                    self.rule['out_table'] + \
                    '(date_stamp, t_zone, ' + \
                    'source_rule, severity, source_table, event_limit, ' + \
                    'event_count, magnitude, time_int, message, source_ids) ' + \
                    'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)'

            # Send an event to the database:
            con = mdb.connect(self.db['host'], self.db['user'],
                    self.db['password'], self.db['database'])
            with con:
                cur = con.cursor()
                cur.execute(outstatement, (datestamp, self.tzone,
                    self.rule['rule_name'], self.rule['severity'],
                    self.rule['source_table'],
                    self.rule['event_limit'], len(rows), magnitude,
                    self.rule['time_int'], self.rule['message'],
                    idtags))
                cur.close()
            con.close()

def start_rule(db, rule, oneshot):
    """Initialize trigger object and start watching"""

    # Create table if it doesn't exist:
    con = mdb.connect(db['host'], db['user'], db['password'], db['database'])
    with con:
        cur = con.cursor()
        cur.execute('CREATE TABLE IF NOT EXISTS ' + rule['out_table'] + \
                '(id INT PRIMARY KEY AUTO_INCREMENT, ' + \
                'date_stamp TIMESTAMP, ' + \
                't_zone NVARCHAR(5), ' + \
                'source_rule NVARCHAR(25), ' + \
                'severity TINYINT UNSIGNED, ' + \
                'source_table NVARCHAR(25), ' + \
                'event_limit INT UNSIGNED, ' + \
                'event_count INT UNSIGNED, ' + \
                'magnitude INT UNSIGNED, ' + \
                'time_int INT UNSIGNED, ' + \
                'message NVARCHAR(1000), ' + \
                'source_ids NVARCHAR(2000))')
        cur.close()
    con.close()

    sentry = SiemTrigger(db, rule)

    if oneshot:
        sentry.check_rule()
    elif int(rule['time_int']) == 0:
        pass

    else:
        # Before starting, sleep randomly up to rule interval to stagger
        # database use:
        sleep(randrange(0, int(rule['time_int']) * 60))

        sentry.watch_rule()
<commit_msg>Update magnitude calculation formula<commit_after>#!/usr/bin/env python

#_MIT License
#_
#_Copyright (c) 2017 Dan Persons (dpersonsdev@gmail.com)
#_
#_Permission is hereby granted, free of charge, to any person obtaining a copy
#_of this software and associated documentation files (the "Software"), to deal
#_in the Software without restriction, including without limitation the rights
#_to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#_copies of the Software, and to permit persons to whom the Software is
#_furnished to do so, subject to the following conditions:
#_
#_The above copyright notice and this permission notice shall be included in all
#_copies or substantial portions of the Software.
#_
#_THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#_IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#_FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#_AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#_LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#_OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
#_SOFTWARE.

import time
from time import strftime
from time import sleep
from time import daylight
from time import timezone
from time import altzone
from random import randrange
from datetime import datetime
import MySQLdb as mdb
import json
import threading
import os
from sys import exit
#import signal

class SiemTrigger:

    def __init__(self, db, rule):
        """Initialize trigger object"""

        self.db = db
        self.rule = rule
        self.tzone = None

    def watch_rule(self):
        """Watch a trigger rule"""

        # Set time zone:
        if daylight:
            self.tzone = \
                    str(int(float(altzone) / 60 // 60)).rjust(2,
                            '0') + \
                    str(int(float(altzone) / 60 % 60)).ljust(2, '0')
        else:
            self.tzone = \
                    str(int(float(timezone) / 60 // 60)).rjust(2,
                            '0') + \
                    str(int(float(timezone) / 60 % 60)).ljust(2, '0')
        if not '-' in self.tzone:
            self.tzone = '+' + self.tzone

        while True:

            # Check the rule:
            self.check_rule()

            # Wait until the next interval
            sleep(int(self.rule['time_int']) * 60)

    def check_rule(self):
        """Check a trigger rule"""

        if not self.tzone:
            # Set time zone:
            if daylight:
                self.tzone = \
                        str(int(float(altzone) / 60 // 60)).rjust(2,
                                '0') + \
                        str(int(float(altzone) / 60 % 60)).ljust(2, '0')
            else:
                self.tzone = \
                        str(int(float(timezone) / 60 // 60)).rjust(2,
                                '0') + \
                        str(int(float(timezone) / 60 % 60)).ljust(2, '0')
            if not '-' in self.tzone:
                self.tzone = '+' + self.tzone

        # Query the database:
        con = mdb.connect(self.db['host'], self.db['user'],
                self.db['password'], self.db['database'])
        with con:
            cur = con.cursor()
            cur.execute(self.rule['sql_query'])
            rows = cur.fetchall()
            cur.close()
        con.close()

        # Evaluate the results:
        if len(rows) > int(self.rule['event_limit']):
            idtags = json.dumps([int(row[0]) for row in rows])

            datestamp = datetime.now().strftime('%Y%m%d%H%M%S')
            magnitude = (((len(rows) // 2) // \
                    (self.rule['event_limit'] + 1) // 2) + 5) * \
                    ( 7 - self.rule['severity'])

            outstatement = 'INSERT INTO ' + \
                    self.rule['out_table'] + \
                    '(date_stamp, t_zone, ' + \
                    'source_rule, severity, source_table, event_limit, ' + \
                    'event_count, magnitude, time_int, message, source_ids) ' + \
                    'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)'

            # Send an event to the database:
            con = mdb.connect(self.db['host'], self.db['user'],
                    self.db['password'], self.db['database'])
            with con:
                cur = con.cursor()
                cur.execute(outstatement, (datestamp, self.tzone,
                    self.rule['rule_name'], self.rule['severity'],
                    self.rule['source_table'],
                    self.rule['event_limit'], len(rows), magnitude,
                    self.rule['time_int'], self.rule['message'],
                    idtags))
                cur.close()
            con.close()

def start_rule(db, rule, oneshot):
    """Initialize trigger object and start watching"""

    # Create table if it doesn't exist:
    con = mdb.connect(db['host'], db['user'], db['password'], db['database'])
    with con:
        cur = con.cursor()
        cur.execute('CREATE TABLE IF NOT EXISTS ' + rule['out_table'] + \
                '(id INT PRIMARY KEY AUTO_INCREMENT, ' + \
                'date_stamp TIMESTAMP, ' + \
                't_zone NVARCHAR(5), ' + \
                'source_rule NVARCHAR(25), ' + \
                'severity TINYINT UNSIGNED, ' + \
                'source_table NVARCHAR(25), ' + \
                'event_limit INT UNSIGNED, ' + \
                'event_count INT UNSIGNED, ' + \
                'magnitude INT UNSIGNED, ' + \
                'time_int INT UNSIGNED, ' + \
                'message NVARCHAR(1000), ' + \
                'source_ids NVARCHAR(2000))')
        cur.close()
    con.close()

    sentry = SiemTrigger(db, rule)

    if oneshot:
        sentry.check_rule()
    elif int(rule['time_int']) == 0:
        pass

    else:
        # Before starting, sleep randomly up to rule interval to stagger
        # database use:
        sleep(randrange(0, int(rule['time_int']) * 60))

        sentry.watch_rule()
<|endoftext|>


<commit_before># coding:utf-8
'''
ScratchからBezelieを動かすサンプル
Copyright (c) 2016 Daisuke IMAI
This software is released under the MIT License.
http://opensource.org/licenses/mit-license.php
'''

import sys
import time
import threading
import scratch
import bezelie

class Receiver(object):
    '''Scratchからの受信データの処理
    '''
    def __init__(self, bezelie):
        self.bezelie = bezelie

    def broadcast_handler(self, message):
        print('[receive] broadcast:', message)

    def sonsor_update_handler(self, **sensor_data):
        for name, value in sensor_data.items():
            print('[receive] sensor-update:', name, value)

if __name__ == '__main__':

    scratchhost = '127.0.0.1'

    # 引数処理
    param = sys.argv
    if len(param) > 1:
        # 引数がついていればScratchの接続先として利用
        scratchhost = param[1]

    # Bezelieと接続しコントロールするためのインスタンス生成
    bez = bezelie.Control()
    bez.setTrim(head=7, back=2, stage=2)

    # Scratch接続のためのインスタンス生成
    rcv = Receiver(bez)
    rsc = scratch.RemoteSensorConnection(rcv.broadcast_handler, rcv.sonsor_update_handler)
    rsc.connect(host=scratchhost)

    while(True):
        try:
            time.sleep(1)
        except KeyboardInterrupt:
            print " Interrupted by Keyboard"
            rsc.disconnect()
            exit()
<commit_msg>broadcastならびに変数の変更に対応<commit_after># coding:utf-8
'''
ScratchからBezelieを動かすサンプル
Copyright (c) 2016 Daisuke IMAI
This software is released under the MIT License.
http://opensource.org/licenses/mit-license.php
'''

import sys
import time
import threading
import scratch
import bezelie

class Receiver(object):
    '''Scratchからの受信データの処理
    '''
    def __init__(self, bezelie):
        self.bezelie = bezelie

    def broadcast_handler(self, message):
        print('[receive] broadcast:', message)
        command_data = message.split(':')
        if len(command_data) > 2:
            if command_data[0] == u'move':
                if len(command_data) == 3:
                    if command_data[1] == u'head':
                        try:
                            self.bezelie.moveHead(int(command_data[2]))
                        except:
                            pass
                    if command_data[1] == u'back':
                        try:
                            self.bezelie.moveBack(int(command_data[2]))
                        except:
                            pass
                    if command_data[1] == u'stage':
                        try:
                            self.bezelie.moveStage(int(command_data[2]))
                        except:
                            pass
            if command_data[0] == u'talk':
                # talk関連コマンドを入れればtalkも可能に
                pass
            if command_data[0] == u'led':
                # led関連コマンドを入れればledも可能に
                pass

    def sonsor_update_handler(self, **sensor_data):
        for name, value in sensor_data.items():
            print('[receive] sensor-update:', name, value)
            command_data = name.split(':')
            if len(command_data) > 2:
                if command_data[0] == 'bezelie':
                    if command_data[1] == u'head':
                        self.bezelie.moveHead(value)
                    if command_data[1] == u'back':
                        self.bezelie.moveBack(value)
                    if command_data[1] == u'stage':
                        self.bezelie.moveStage(value)

if __name__ == '__main__':

    scratchhost = '127.0.0.1'

    # 引数処理
    param = sys.argv
    if len(param) > 1:
        # 引数がついていればScratchの接続先として利用
        scratchhost = param[1]

    # Bezelieと接続しコントロールするためのインスタンス生成
    bez = bezelie.Control()
    bez.setTrim(head=7, back=2, stage=2)

    # Scratch接続のためのインスタンス生成
    rcv = Receiver(bez)
    rsc = scratch.RemoteSensorConnection(rcv.broadcast_handler, rcv.sonsor_update_handler)
    rsc.connect(host=scratchhost)

    while(True):
        try:
            time.sleep(1)
        except KeyboardInterrupt:
            print " Interrupted by Keyboard"
            rsc.disconnect()
            exit()
<|endoftext|>