<commit_before>
# A consolidation of the data sources for the project
# includes:
# NetworkX!
# RDFlib!
# Other things!
#
# Works like Configure:
# Inherit from the DataUser class to access data of all kinds (listed above)

import sqlite3
import networkx as nx
import PyOpenWorm
from PyOpenWorm import Configureable, Configure, ConfigValue, BadConf
import hashlib
import csv
import urllib2
from rdflib import URIRef, Literal, Graph, Namespace, ConjunctiveGraph
from rdflib.namespace import RDFS,RDF
from datetime import datetime as DT
import datetime
import os
import logging as L

__all__ = ["Data", "DataUser", "TrixSource", "SPARQLSource", "SleepyCatSource", "SQLiteSource", "DefaultSource"]

class _B(ConfigValue):
    def __init__(self, f):
        self.v = False
        self.f = f

    def get(self):
        if not self.v:
            self.v = self.f()

        return self.v
    def invalidate(self):
        self.v = False

ZERO = datetime.timedelta(0)
class _UTC(datetime.tzinfo):
    """UTC"""

    def utcoffset(self, dt):
        return ZERO

    def tzname(self, dt):
        return "UTC"

    def dst(self, dt):
        return ZERO
utc = _UTC()

propertyTypes = {"send" : 'http://openworm.org/entities/356',
        "Neuropeptide" : 'http://openworm.org/entities/354',
        "Receptor" : 'http://openworm.org/entities/361',
        "is a" : 'http://openworm.org/entities/1515',
        "neuromuscular junction" : 'http://openworm.org/entities/1516',
        "Innexin" : 'http://openworm.org/entities/355',
        "Neurotransmitter" : 'http://openworm.org/entities/313',
        "gap junction" : 'http://openworm.org/entities/357'}

def grouper(iterable, n, fillvalue=None):
    "Collect data into fixed-length chunks or blocks"
    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
    args = [iter(iterable)] * n
    while True:
        l = []
        try:
            for x in args:
                l.append(next(x))
        except:
            pass
        yield l
        if len(l) < n:
            break

class DataUser(Configureable):
    def __init__(self, conf = False):
        """ An convenience wrapper for users of the database

        Classes which use the database should inherit from DataUser.
        """
        Configureable.__init__(self, conf=conf)
        if not isinstance(self.conf,Data):
            raise BadConf(self)

    @property
    def rdf(self):
        return self.conf['rdf.graph']

    def _remove_from_store(self, g):
        # Note the assymetry with _add_to_store. You must add actual elements, but deletes
        # can be performed as a query
        for group in grouper(g, 1000):
            temp_graph = Graph()
            for x in group:
                if x is not None:
                    temp_graph.add(x)
                else:
                    break
            s = " DELETE DATA {" + temp_graph.serialize(format="nt") + " } "
            L.debug("deleting. s = " + s)
            self.conf['rdf.graph'].update(s)

    def _add_to_store(self, g, graph_name=False):
        for group in grouper(g, int(self.conf.get('rdf.upload_block_statement_count',100))):
            temp_graph = Graph()
            for x in group:
                if x is not None:
                    temp_graph.add(x)
                else:
                    break
            if graph_name:
                s = " INSERT DATA { GRAPH "+graph_name.n3()+" {" + temp_graph.serialize(format="nt") + " } } "
            else:
                s = " INSERT DATA { " + temp_graph.serialize(format="nt") + " } "
            L.debug("update query = " + s)
            self.conf['rdf.graph'].update(s)

    def add_reference(self, g, reference_iri):
        """
        Add a citation to a set of statements in the database
        :param triples: A set of triples to annotate
        """
        new_statements = Graph()
        ns = self.conf['rdf.namespace']
        for statement in g:
            statement_node = self._reify(new_statements,statement)
            new_statements.add((URIRef(reference_iri), ns['asserts'], statement_node))

        self.add_statements(g + new_statements)

    #def _add_unannotated_statements(self, graph):
    # A UTC class.

    def retract_statements(self, graph):
        """
        Remove a set of statements from the database.
        :param graph: An iterable of triples
        """
        self._remove_from_store_by_query(graph)
    def _remove_from_store_by_query(self, q):
        import logging as L
        s = " DELETE WHERE {" + q + " } "
        L.debug("deleting. s = " + s)
        self.conf['rdf.graph'].update(s)

    def add_statements(self, graph):
        """
        Add a set of statements to the database.
        Annotates the addition with uploader name, etc
        :param graph: An iterable of triples
        """
        self._add_to_store(graph)

    def _reify(self,g,s):
        """
        Add a statement object to g that binds to s
        """
        n = self.conf['new_graph_uri'](s)
        g.add((n, RDF['type'], RDF['Statement']))
        g.add((n, RDF['subject'], s[0]))
        g.add((n, RDF['predicate'], s[1]))
        g.add((n, RDF['object'], s[2]))
        return n

class Data(Configure, Configureable):
    def __init__(self, conf=False):
        Configure.__init__(self)
        Configureable.__init__(self,conf)
        # We copy over all of the configuration that we were given
        self.copy(self.conf)
        self.namespace = Namespace("http://openworm.org/entities/")
        self.molecule_namespace = Namespace("http://openworm.org/entities/molecules/")
        self['nx'] = _B(self._init_networkX)
        self['rdf.namespace'] = self.namespace
        self['molecule_name'] = self._molecule_hash
        self['new_graph_uri'] = self._molecule_hash
        self._init_rdf_graph()

    @classmethod
    def open(cls,file_name):
        c = Configure.open(file_name)
        return cls(c)

    def openDatabase(self):
        self.source.open()
        L.debug("opening " + str(self.source))

    def closeDatabase(self):
        self.source.close()

    def _init_rdf_graph(self):
        # Set these in case they were left out
        c = self.conf
        c['rdf.source'] = c.get('rdf.source', 'default')
        c['rdf.store'] = c.get('rdf.store', 'default')
        c['rdf.store_conf'] = c.get('rdf.store_conf', 'default')

        self.sources = {'sqlite' : SQLiteSource(c),
                'sparql_endpoint' : SPARQLSource(c),
                'Sleepycat' : SleepyCatSource(c),
                'default' : DefaultSource(c),
                'TriX' : TrixSource(c)}
        i = self.sources[c['rdf.source']]
        self.source = i
        self.link('semantic_net_new', 'semantic_net', 'rdf.graph')
        self['rdf.graph'] = i
        return i

    def _molecule_hash(self, data):
        return URIRef(self.molecule_namespace[hashlib.sha224(str(data)).hexdigest()])

    def _init_networkX(self):
        g = nx.DiGraph()

        # Neuron table
        csvfile = urllib2.urlopen(self.conf['neuronscsv'])

        reader = csv.reader(csvfile, delimiter=';', quotechar='|')
        for row in reader:
            neurontype = ""
            # Detects neuron function
            if "sensory" in row[1].lower():
                neurontype += "sensory"
            if "motor" in row[1].lower():
                neurontype += "motor"
            if "interneuron" in row[1].lower():
                neurontype += "interneuron"
            if len(neurontype) == 0:
                neurontype = "unknown"

            if len(row[0]) > 0: # Only saves valid neuron names
                g.add_node(row[0], ntype = neurontype)

        # Connectome table
        csvfile = urllib2.urlopen(self.conf['connectomecsv'])

        reader = csv.reader(csvfile, delimiter=';', quotechar='|')
        for row in reader:
            g.add_edge(row[0], row[1], weight = row[3])
            g[row[0]][row[1]]['synapse'] = row[2]
            g[row[0]][row[1]]['neurotransmitter'] = row[4]
        return g

def modification_date(filename):
    t = os.path.getmtime(filename)
    return datetime.datetime.fromtimestamp(t)

class RDFSource(Configureable,PyOpenWorm.ConfigValue):
    """ Base class for data sources.

    Alternative sources should dervie from this class
    """
    i = 0
    def __init__(self, conf=False):
        if self.i == 1:
            raise Exception(self.i)
        self.i+=1
        Configureable.__init__(self, conf=conf)
        self.graph = False

    def get(self):
        if self.graph == False:
            raise Exception("Must call openDatabase on Data object before using the database")
        return self.graph

    def close(self):
        if self.graph == False:
            return
        self.graph.close()
        self.graph = False

class TrixSource(RDFSource):
    """ Reads from a TriX file or the configured store is more recent, from that.

    .. note::

        configure with  "rdf.source" = "TriX"

        The database store is configured with::

            "rdf.store" = <your rdflib store name here>
            "rdf.store_conf" = <your rdflib store configuration here>

    """
    # XXX How to write back out to this?
    i = 0

    def open(self):
        if not self.graph:
            self.graph = True
            import glob
            # Check the ages of the files. Read the more recent one.
            g0 = ConjunctiveGraph(store=self.conf['rdf.store'])
            database_store = self.conf['rdf.store_conf']
            trix_file = self.conf['trix_location']
            # store_time only works for stores that are on the local
            # machine.
            try:
                store_time = modification_date(database_store)
                # If the store is newer than the serialization
                # get the newest file in the store
                for x in glob.glob(database_store +"/*"):
                    mod = modification_date(x)
                    if store_time < mod:
                        store_time = mod
            except:
                store_time = DT.min

            trix_time = modification_date(trix_file)

            g0.open(database_store, create=True)

            if store_time > trix_time:
                # just use the store
                pass
            else:
                # delete the database and read in the new one
                # read in the serialized format
                g0.parse(trix_file,format="trix")

            self.graph = g0

        return self.graph

class SPARQLSource(RDFSource):
    """ Reads from and queries against a remote data store

        .. note::

            configure with  "rdf.source" = "sparql_endpoint"
    """
    def open(self):
        # XXX: If we have a source that's read only, should we need to set the store separately??
        g0 = ConjunctiveGraph('SPARQLUpdateStore')
        g0.open(tuple(self.conf['rdf.store_conf']))
        self.graph = g0
        return self.graph

class SleepyCatSource(RDFSource):
    """ Reads from and queries against a local sleepy cat database

        configure with  "rdf.source" = "sparql_endpoint"
    """
    def open(self):
        import logging
        # XXX: If we have a source that's read only, should we need to set the store separately??
        g0 = ConjunctiveGraph('Sleepycat')
        g0.open(self.conf['rdf.store_conf'],create=True)
        self.graph = g0
        logging.debug("Opened SleepyCatSource")

class SQLiteSource(RDFSource):
    def open(self):
        conn = sqlite3.connect(self.conf['sqldb'])
        cur = conn.cursor()

        #first step, grab all entities and add them to the graph
        n = self.conf['rdf.namespace']

        cur.execute("SELECT DISTINCT ID, Entity FROM tblentity")
        g0 = ConjunctiveGraph(self.conf['rdf.store'])
        g0.open(self.conf['rdf.store_conf'], create=True)

        for r in cur.fetchall():
            #first item is a number -- needs to be converted to a string
           first = str(r[0])
           #second item is text
           second = str(r[1])

           # This is the backbone of any RDF graph.  The unique
           # ID for each entity is encoded as a URI and every other piece of
           # knowledge about that entity is connected via triples to that URI
           # In this case, we connect the common name of that entity to the
           # root URI via the RDFS label property.
           g0.add( (n[first], RDFS.label, Literal(second)) )

        #second step, get the relationships between them and add them to the graph
        cur.execute("SELECT DISTINCT EnID1, Relation, EnID2, Citations FROM tblrelationship")

        gi = ''

        i = 0
        for r in cur.fetchall():
           #all items are numbers -- need to be converted to a string
           first = str(r[0])
           second = str(r[1])
           third = str(r[2])
           prov = str(r[3])

           ui = self.conf['molecule_name'](prov)
           gi = Graph(g0.store, ui)

           gi.add( (n[first], n[second], n[third]) )

           g0.add([ui, RDFS.label, Literal(str(i))])
           if (prov != ''):
               g0.add([ui, n[u'text_reference'], Literal(prov)])

           i = i + 1

        cur.close()
        conn.close()
        self.graph = g0

class DefaultSource(RDFSource,ConfigValue):
    def open(self):
        self.graph = ConjunctiveGraph(self.conf['rdf.store'])
        self.graph.open(self.conf['rdf.store_conf'],create=True)
<commit_msg>Small update to docs<commit_after>
# A consolidation of the data sources for the project
# includes:
# NetworkX!
# RDFlib!
# Other things!
#
# Works like Configure:
# Inherit from the DataUser class to access data of all kinds (listed above)

import sqlite3
import networkx as nx
import PyOpenWorm
from PyOpenWorm import Configureable, Configure, ConfigValue, BadConf
import hashlib
import csv
import urllib2
from rdflib import URIRef, Literal, Graph, Namespace, ConjunctiveGraph
from rdflib.namespace import RDFS,RDF
from datetime import datetime as DT
import datetime
import os
import logging as L

__all__ = ["Data", "DataUser", "RDFSource", "TrixSource", "SPARQLSource", "SleepyCatSource", "DefaultSource"]

class _B(ConfigValue):
    def __init__(self, f):
        self.v = False
        self.f = f

    def get(self):
        if not self.v:
            self.v = self.f()

        return self.v
    def invalidate(self):
        self.v = False

ZERO = datetime.timedelta(0)
class _UTC(datetime.tzinfo):
    """UTC"""

    def utcoffset(self, dt):
        return ZERO

    def tzname(self, dt):
        return "UTC"

    def dst(self, dt):
        return ZERO
utc = _UTC()

propertyTypes = {"send" : 'http://openworm.org/entities/356',
        "Neuropeptide" : 'http://openworm.org/entities/354',
        "Receptor" : 'http://openworm.org/entities/361',
        "is a" : 'http://openworm.org/entities/1515',
        "neuromuscular junction" : 'http://openworm.org/entities/1516',
        "Innexin" : 'http://openworm.org/entities/355',
        "Neurotransmitter" : 'http://openworm.org/entities/313',
        "gap junction" : 'http://openworm.org/entities/357'}

def grouper(iterable, n, fillvalue=None):
    "Collect data into fixed-length chunks or blocks"
    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
    args = [iter(iterable)] * n
    while True:
        l = []
        try:
            for x in args:
                l.append(next(x))
        except:
            pass
        yield l
        if len(l) < n:
            break

class DataUser(Configureable):
    def __init__(self, conf = False):
        """ An convenience wrapper for users of the database

        Classes which use the database should inherit from DataUser.
        """
        Configureable.__init__(self, conf=conf)
        if not isinstance(self.conf,Data):
            raise BadConf(self)

    @property
    def rdf(self):
        return self.conf['rdf.graph']

    def _remove_from_store(self, g):
        # Note the assymetry with _add_to_store. You must add actual elements, but deletes
        # can be performed as a query
        for group in grouper(g, 1000):
            temp_graph = Graph()
            for x in group:
                if x is not None:
                    temp_graph.add(x)
                else:
                    break
            s = " DELETE DATA {" + temp_graph.serialize(format="nt") + " } "
            L.debug("deleting. s = " + s)
            self.conf['rdf.graph'].update(s)

    def _add_to_store(self, g, graph_name=False):
        for group in grouper(g, int(self.conf.get('rdf.upload_block_statement_count',100))):
            temp_graph = Graph()
            for x in group:
                if x is not None:
                    temp_graph.add(x)
                else:
                    break
            if graph_name:
                s = " INSERT DATA { GRAPH "+graph_name.n3()+" {" + temp_graph.serialize(format="nt") + " } } "
            else:
                s = " INSERT DATA { " + temp_graph.serialize(format="nt") + " } "
            L.debug("update query = " + s)
            self.conf['rdf.graph'].update(s)

    def add_reference(self, g, reference_iri):
        """
        Add a citation to a set of statements in the database
        :param triples: A set of triples to annotate
        """
        new_statements = Graph()
        ns = self.conf['rdf.namespace']
        for statement in g:
            statement_node = self._reify(new_statements,statement)
            new_statements.add((URIRef(reference_iri), ns['asserts'], statement_node))

        self.add_statements(g + new_statements)

    #def _add_unannotated_statements(self, graph):
    # A UTC class.

    def retract_statements(self, graph):
        """
        Remove a set of statements from the database.
        :param graph: An iterable of triples
        """
        self._remove_from_store_by_query(graph)
    def _remove_from_store_by_query(self, q):
        import logging as L
        s = " DELETE WHERE {" + q + " } "
        L.debug("deleting. s = " + s)
        self.conf['rdf.graph'].update(s)

    def add_statements(self, graph):
        """
        Add a set of statements to the database.
        Annotates the addition with uploader name, etc
        :param graph: An iterable of triples
        """
        self._add_to_store(graph)

    def _reify(self,g,s):
        """
        Add a statement object to g that binds to s
        """
        n = self.conf['new_graph_uri'](s)
        g.add((n, RDF['type'], RDF['Statement']))
        g.add((n, RDF['subject'], s[0]))
        g.add((n, RDF['predicate'], s[1]))
        g.add((n, RDF['object'], s[2]))
        return n

class Data(Configure, Configureable):
    """
    Provides configuration for access to the database.

    Usally doesn't need to be accessed directly
    """
    def __init__(self, conf=False):
        Configure.__init__(self)
        Configureable.__init__(self,conf)
        # We copy over all of the configuration that we were given
        self.copy(self.conf)
        self.namespace = Namespace("http://openworm.org/entities/")
        self.molecule_namespace = Namespace("http://openworm.org/entities/molecules/")
        self['nx'] = _B(self._init_networkX)
        self['rdf.namespace'] = self.namespace
        self['molecule_name'] = self._molecule_hash
        self['new_graph_uri'] = self._molecule_hash
        self._init_rdf_graph()

    @classmethod
    def open(cls,file_name):
        """ Open a file storing configuration in a JSON format """
        c = Configure.open(file_name)
        return cls(c)

    def openDatabase(self):
        """ Open a the configured database """
        self.source.open()
        L.debug("opening " + str(self.source))

    def closeDatabase(self):
        """ Close a the configured database """
        self.source.close()

    def _init_rdf_graph(self):
        # Set these in case they were left out
        c = self.conf
        c['rdf.source'] = c.get('rdf.source', 'default')
        c['rdf.store'] = c.get('rdf.store', 'default')
        c['rdf.store_conf'] = c.get('rdf.store_conf', 'default')

        self.sources = {'sqlite' : SQLiteSource(c),
                'sparql_endpoint' : SPARQLSource(c),
                'Sleepycat' : SleepyCatSource(c),
                'default' : DefaultSource(c),
                'TriX' : TrixSource(c)}
        i = self.sources[c['rdf.source']]
        self.source = i
        self.link('semantic_net_new', 'semantic_net', 'rdf.graph')
        self['rdf.graph'] = i
        return i

    def _molecule_hash(self, data):
        return URIRef(self.molecule_namespace[hashlib.sha224(str(data)).hexdigest()])

    def _init_networkX(self):
        g = nx.DiGraph()

        # Neuron table
        csvfile = urllib2.urlopen(self.conf['neuronscsv'])

        reader = csv.reader(csvfile, delimiter=';', quotechar='|')
        for row in reader:
            neurontype = ""
            # Detects neuron function
            if "sensory" in row[1].lower():
                neurontype += "sensory"
            if "motor" in row[1].lower():
                neurontype += "motor"
            if "interneuron" in row[1].lower():
                neurontype += "interneuron"
            if len(neurontype) == 0:
                neurontype = "unknown"

            if len(row[0]) > 0: # Only saves valid neuron names
                g.add_node(row[0], ntype = neurontype)

        # Connectome table
        csvfile = urllib2.urlopen(self.conf['connectomecsv'])

        reader = csv.reader(csvfile, delimiter=';', quotechar='|')
        for row in reader:
            g.add_edge(row[0], row[1], weight = row[3])
            g[row[0]][row[1]]['synapse'] = row[2]
            g[row[0]][row[1]]['neurotransmitter'] = row[4]
        return g

def modification_date(filename):
    t = os.path.getmtime(filename)
    return datetime.datetime.fromtimestamp(t)

class RDFSource(Configureable,PyOpenWorm.ConfigValue):
    """ Base class for data sources.

    Alternative sources should dervie from this class
    """
    i = 0
    def __init__(self, conf=False):
        if self.i == 1:
            raise Exception(self.i)
        self.i+=1
        Configureable.__init__(self, conf=conf)
        self.graph = False

    def get(self):
        if self.graph == False:
            raise Exception("Must call openDatabase on Data object before using the database")
        return self.graph

    def close(self):
        if self.graph == False:
            return
        self.graph.close()
        self.graph = False

class TrixSource(RDFSource):
    """ Reads from a TriX file or the configured store is more recent, from that.

    .. note::

        configure with  "rdf.source" = "TriX"

        The database store is configured with::

            "rdf.store" = <your rdflib store name here>
            "rdf.store_conf" = <your rdflib store configuration here>

    """
    # XXX How to write back out to this?
    i = 0

    def open(self):
        if not self.graph:
            self.graph = True
            import glob
            # Check the ages of the files. Read the more recent one.
            g0 = ConjunctiveGraph(store=self.conf['rdf.store'])
            database_store = self.conf['rdf.store_conf']
            trix_file = self.conf['trix_location']
            # store_time only works for stores that are on the local
            # machine.
            try:
                store_time = modification_date(database_store)
                # If the store is newer than the serialization
                # get the newest file in the store
                for x in glob.glob(database_store +"/*"):
                    mod = modification_date(x)
                    if store_time < mod:
                        store_time = mod
            except:
                store_time = DT.min

            trix_time = modification_date(trix_file)

            g0.open(database_store, create=True)

            if store_time > trix_time:
                # just use the store
                pass
            else:
                # delete the database and read in the new one
                # read in the serialized format
                g0.parse(trix_file,format="trix")

            self.graph = g0

        return self.graph

class SPARQLSource(RDFSource):
    """ Reads from and queries against a remote data store

        .. note::

            configure with  "rdf.source" = "sparql_endpoint"
    """
    def open(self):
        # XXX: If we have a source that's read only, should we need to set the store separately??
        g0 = ConjunctiveGraph('SPARQLUpdateStore')
        g0.open(tuple(self.conf['rdf.store_conf']))
        self.graph = g0
        return self.graph

class SleepyCatSource(RDFSource):
    """ Reads from and queries against a local Sleepycat database

        .. note:: configure with  "rdf.source" = "SQLiteSource"

        The database location can be configured like::

            "rdf.store_conf" = <your database location here>
    """
    def open(self):
        import logging
        # XXX: If we have a source that's read only, should we need to set the store separately??
        g0 = ConjunctiveGraph('Sleepycat')
        g0.open(self.conf['rdf.store_conf'],create=True)
        self.graph = g0
        logging.debug("Opened SleepyCatSource")

class SQLiteSource(RDFSource):
    def open(self):
        conn = sqlite3.connect(self.conf['sqldb'])
        cur = conn.cursor()

        #first step, grab all entities and add them to the graph
        n = self.conf['rdf.namespace']

        cur.execute("SELECT DISTINCT ID, Entity FROM tblentity")
        g0 = ConjunctiveGraph(self.conf['rdf.store'])
        g0.open(self.conf['rdf.store_conf'], create=True)

        for r in cur.fetchall():
            #first item is a number -- needs to be converted to a string
           first = str(r[0])
           #second item is text
           second = str(r[1])

           # This is the backbone of any RDF graph.  The unique
           # ID for each entity is encoded as a URI and every other piece of
           # knowledge about that entity is connected via triples to that URI
           # In this case, we connect the common name of that entity to the
           # root URI via the RDFS label property.
           g0.add( (n[first], RDFS.label, Literal(second)) )

        #second step, get the relationships between them and add them to the graph
        cur.execute("SELECT DISTINCT EnID1, Relation, EnID2, Citations FROM tblrelationship")

        gi = ''

        i = 0
        for r in cur.fetchall():
           #all items are numbers -- need to be converted to a string
           first = str(r[0])
           second = str(r[1])
           third = str(r[2])
           prov = str(r[3])

           ui = self.conf['molecule_name'](prov)
           gi = Graph(g0.store, ui)

           gi.add( (n[first], n[second], n[third]) )

           g0.add([ui, RDFS.label, Literal(str(i))])
           if (prov != ''):
               g0.add([ui, n[u'text_reference'], Literal(prov)])

           i = i + 1

        cur.close()
        conn.close()
        self.graph = g0

class DefaultSource(RDFSource,ConfigValue):
    """ Reads from and queries against a configured database.

    .. note::

        The default configuration.

        Can also configure with "rdf.source" = "default"

        The database store is configured with::

            "rdf.store" = <your rdflib store name here>
            "rdf.store_conf" = <your rdflib store configuration here>

    """
    def open(self):
        self.graph = ConjunctiveGraph(self.conf['rdf.store'])
        self.graph.open(self.conf['rdf.store_conf'],create=True)
<|endoftext|>


<commit_before>import requests
import glob
import hashlib
import os

# this hash function receives the name of the file and returns the hash code
def get_hash(name):
    readsize = 64 * 1024
    with open(name, 'rb') as f:
        size = os.path.getsize(name)
        data = f.read(readsize)
        f.seek(-readsize, os.SEEK_END)
        data += f.read(readsize)
    return hashlib.md5(data).hexdigest()

base_url = 'http://api.thesubdb.com/'
user_agent = "SubDB/1.0 (PySubDBAutoFill/0.1; http://github.com/nikolajlauridsenn/PySubDB"

format = input("file format: ")
files = glob.glob('*.' + format)
headers = {'User-Agent': user_agent}
fail_count = 0

for file in files:
    f_hash = get_hash(file)
    payload = {'action': 'download', 'hash': f_hash, 'language': 'en'}
    print("Getting subtitle for " + file)
    response = requests.get(base_url, headers=headers, params=payload)
    if response.status_code == 200:
        f = open(file + '.srt', 'wb')
        f.write(response.content)
        f.close
        print("done")
    else:
        print("There's unfortunately no subtitle for this file at the moment")
        fail_count += 1

    if fail_count > 8:
        break
<commit_msg>Improving error handling<commit_after>import requests
import glob
import hashlib
import os
from time import sleep

# this hash function receives the name of the file and returns the hash code
def get_hash(name):
    readsize = 64 * 1024
    with open(name, 'rb') as f:
        size = os.path.getsize(name)
        data = f.read(readsize)
        f.seek(-readsize, os.SEEK_END)
        data += f.read(readsize)
    return hashlib.md5(data).hexdigest()

base_url = 'http://api.thesubdb.com/'
user_agent = "SubDB/1.0 (PySubDBAutoFill/0.1; http://github.com/nikolajlauridsenn/PySubDB"

format = input("file format: ")
files = glob.glob('*.' + format)
headers = {'User-Agent': user_agent}
fail_count = 0
max_fails = 10

for file in files:
    f_hash = get_hash(file)
    payload = {'action': 'download', 'hash': f_hash, 'language': 'en'}
    print("Getting subtitle for " + file)
    response = requests.get(base_url, headers=headers, params=payload)
    if response.status_code == 200:
        f = open(file + '.srt', 'wb')
        f.write(response.content)
        f.close
        print("done")
        fail_count = 0
    else:
        print("There's unfortunately no subtitle for this file at the moment")
        fail_count += 1

    if fail_count > max_fails:
        print("More than " + str(max_fails) + " files has failed in a row, exiting script.")
        break
sleep(2)
<|endoftext|>


<commit_before>import json
import requests
import subprocess

print('Hitting discord\'s API to determine how many shards are needed.')
key = ''
with open("key.txt", "r") as target:
	for line in target:
		line = line.strip()
		key = line

url = 'https://discordapp.com/api/v6/gateway/bot?token=' + key
response = requests.get(url)

processes = []
if (response.ok):
	jData = json.loads(response.content)
	shard_count = int(jData['shards'])
	for x in range(0, shard_count):
		processes.append(subprocess.Popen(['python', 'bot.py', str(x), str(shard_count)]))
	print("Launched processes")
	print(processes.pid)
else:
	print("Can't reach the Gateway endpoint. Giving up.")<commit_msg>Basic logging improvement<commit_after>import json
import requests
import subprocess

print('Hitting discord\'s API to determine how many shards are needed.')
key = ''
with open("key.txt", "r") as target:
	for line in target:
		line = line.strip()
		key = line

url = 'https://discordapp.com/api/v6/gateway/bot?token=' + key
response = requests.get(url)

processes = []
if (response.ok):
	jData = json.loads(response.content)
	shard_count = int(jData['shards'])
	for x in range(0, shard_count):
		processes.append(subprocess.Popen(['python', 'bot.py', str(x), str(shard_count)]))
	print("Launched processes")
	with open("logs/pids.txt", "w") as target:
		for process in processes:
			target.write('{}\n'.format(process.pid))
else:
	print("Can't reach the Gateway endpoint. Giving up.")<|endoftext|>


<commit_before># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

import configparser

from UM.Signal import Signal, signalemitter
from UM.Logger import Logger

from UM.SaveFile import SaveFile

##      Preferences are application based settings that are saved for future use.
#       Typical preferences would be window size, standard machine, etc.
@signalemitter
class Preferences():
    def __init__(self):
        super().__init__()

        self._file = None
        self._parser = None
        self._preferences = {}

    def addPreference(self, key, default_value):
        preference = self._findPreference(key)
        if preference:
            Logger.log("d", "Adding preference %s, but it already exists.", key)
            preference.setDefault(default_value)
            return

        group, key = self._splitKey(key)
        if group not in self._preferences:
            self._preferences[group] = {}

        self._preferences[group][key] = _Preference(key, default_value)

    ##  Changes the default value of a preference.
    #
    #   If the preference is currently set to the old default, the value of the
    #   preference will be set to the new default.
    #
    #   \param key The key of the preference to set the default of.
    #   \param default_value The new default value of the preference.
    def setDefault(self, key, default_value):
        preference = self._findPreference(key)
        if not preference: #Key not found.
            Logger.log("w", "Tried to set the default value of non-existing setting %s.", key)
            return
        if preference.getValue() == preference.getDefault():
            self.setValue(key, default_value)
        preference.setDefault(default_value)

    def setValue(self, key, value):
        preference = self._findPreference(key)

        if preference:
            preference.setValue(value)
            self.preferenceChanged.emit(key)
        else:
            Logger.log("w", "Tried to set the value of non-existing setting %s.", key)

    def getValue(self, key):
        preference = self._findPreference(key)

        if preference:
            value = preference.getValue()
            if value == "True":
                value = True
            elif value == "False":
                value = False
            return value

        Logger.log("w", "Tried to get the value of non-existing setting %s.", key)
        return None

    def resetPreference(self, key):
        preference = self._findPreference(key)

        if preference:
            preference.setValue(preference.getDefault())
            self.preferenceChanged.emit(key)

    def readFromFile(self, file):
        self._loadFile(file)

        if not self._parser:
            return

        for group, group_entries in self._parser.items():
            if group == "DEFAULT":
                continue

            if group not in self._preferences:
                self._preferences[group] = {}

            for key, value in group_entries.items():
                if key not in self._preferences[group]:
                    self._preferences[group][key] = _Preference(key)

                self._preferences[group][key].setValue(value)
                self.preferenceChanged.emit("{0}/{1}".format(group, key))

    def writeToFile(self, file):
        parser = configparser.ConfigParser(interpolation = None) #pylint: disable=bad-whitespace
        for group, group_entries in self._preferences.items():
            parser[group] = {}
            for key, pref in group_entries.items():
                if pref.getValue() != pref.getDefault():
                    parser[group][key] = str(pref.getValue())

        parser["general"]["version"] = "2"

        try:
            with SaveFile(file, "wt") as save_file:
                parser.write(save_file)
        except Exception as e:
            Logger.log("e", "Failed to write preferences to %s: %s", file, str(e))

    preferenceChanged = Signal()

    @classmethod
    def getInstance(cls):
        if not cls._instance:
            cls._instance = Preferences()

        return cls._instance

    def _splitKey(self, key):
        group = "general"
        key = key

        if "/" in key:
            parts = key.split("/")
            group = parts[0]
            key = parts[1]

        return (group, key)

    def _findPreference(self, key):
        group, key = self._splitKey(key)

        if group in self._preferences:
            if key in self._preferences[group]:
                return self._preferences[group][key]

        return None

    def _loadFile(self, file):
        if self._file and self._file == file:
            return self._parser
        try:
            self._parser = configparser.ConfigParser(interpolation = None) #pylint: disable=bad-whitespace
            self._parser.read(file)

            if self._parser["general"]["version"] != "2":
                Logger.log("w", "Old config file found, ignoring")
                self._parser = None
                return
        except Exception as e:
            Logger.log("e", "An exception occured while trying to read preferences file: %s", e)
            self._parser = None
            return

        del self._parser["general"]["version"]

    _instance = None

class _Preference:
    def __init__(self, name, default = None, value = None): #pylint: disable=bad-whitespace
        self._name = name
        self._default = default
        self._value = default if value is None else value

    def getName(self):
        return self._name

    def getValue(self):
        return self._value

    def getDefault(self):
        return self._default

    def setDefault(self, default):
        self._default = default

    def setValue(self, value):
        self._value = value
<commit_msg>Removed superfilious debug log<commit_after># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

import configparser

from UM.Signal import Signal, signalemitter
from UM.Logger import Logger

from UM.SaveFile import SaveFile

##      Preferences are application based settings that are saved for future use.
#       Typical preferences would be window size, standard machine, etc.
@signalemitter
class Preferences():
    def __init__(self):
        super().__init__()

        self._file = None
        self._parser = None
        self._preferences = {}

    def addPreference(self, key, default_value):
        preference = self._findPreference(key)
        if preference:
            preference.setDefault(default_value)
            return

        group, key = self._splitKey(key)
        if group not in self._preferences:
            self._preferences[group] = {}

        self._preferences[group][key] = _Preference(key, default_value)

    ##  Changes the default value of a preference.
    #
    #   If the preference is currently set to the old default, the value of the
    #   preference will be set to the new default.
    #
    #   \param key The key of the preference to set the default of.
    #   \param default_value The new default value of the preference.
    def setDefault(self, key, default_value):
        preference = self._findPreference(key)
        if not preference: #Key not found.
            Logger.log("w", "Tried to set the default value of non-existing setting %s.", key)
            return
        if preference.getValue() == preference.getDefault():
            self.setValue(key, default_value)
        preference.setDefault(default_value)

    def setValue(self, key, value):
        preference = self._findPreference(key)

        if preference:
            preference.setValue(value)
            self.preferenceChanged.emit(key)
        else:
            Logger.log("w", "Tried to set the value of non-existing setting %s.", key)

    def getValue(self, key):
        preference = self._findPreference(key)

        if preference:
            value = preference.getValue()
            if value == "True":
                value = True
            elif value == "False":
                value = False
            return value

        Logger.log("w", "Tried to get the value of non-existing setting %s.", key)
        return None

    def resetPreference(self, key):
        preference = self._findPreference(key)

        if preference:
            preference.setValue(preference.getDefault())
            self.preferenceChanged.emit(key)

    def readFromFile(self, file):
        self._loadFile(file)

        if not self._parser:
            return

        for group, group_entries in self._parser.items():
            if group == "DEFAULT":
                continue

            if group not in self._preferences:
                self._preferences[group] = {}

            for key, value in group_entries.items():
                if key not in self._preferences[group]:
                    self._preferences[group][key] = _Preference(key)

                self._preferences[group][key].setValue(value)
                self.preferenceChanged.emit("{0}/{1}".format(group, key))

    def writeToFile(self, file):
        parser = configparser.ConfigParser(interpolation = None) #pylint: disable=bad-whitespace
        for group, group_entries in self._preferences.items():
            parser[group] = {}
            for key, pref in group_entries.items():
                if pref.getValue() != pref.getDefault():
                    parser[group][key] = str(pref.getValue())

        parser["general"]["version"] = "2"

        try:
            with SaveFile(file, "wt") as save_file:
                parser.write(save_file)
        except Exception as e:
            Logger.log("e", "Failed to write preferences to %s: %s", file, str(e))

    preferenceChanged = Signal()

    @classmethod
    def getInstance(cls):
        if not cls._instance:
            cls._instance = Preferences()

        return cls._instance

    def _splitKey(self, key):
        group = "general"
        key = key

        if "/" in key:
            parts = key.split("/")
            group = parts[0]
            key = parts[1]

        return (group, key)

    def _findPreference(self, key):
        group, key = self._splitKey(key)

        if group in self._preferences:
            if key in self._preferences[group]:
                return self._preferences[group][key]

        return None

    def _loadFile(self, file):
        if self._file and self._file == file:
            return self._parser
        try:
            self._parser = configparser.ConfigParser(interpolation = None) #pylint: disable=bad-whitespace
            self._parser.read(file)

            if self._parser["general"]["version"] != "2":
                Logger.log("w", "Old config file found, ignoring")
                self._parser = None
                return
        except Exception as e:
            Logger.log("e", "An exception occured while trying to read preferences file: %s", e)
            self._parser = None
            return

        del self._parser["general"]["version"]

    _instance = None

class _Preference:
    def __init__(self, name, default = None, value = None): #pylint: disable=bad-whitespace
        self._name = name
        self._default = default
        self._value = default if value is None else value

    def getName(self):
        return self._name

    def getValue(self):
        return self._value

    def getDefault(self):
        return self._default

    def setDefault(self, default):
        self._default = default

    def setValue(self, value):
        self._value = value
<|endoftext|>


<commit_before># Developed by Aditya Atluri
# Date: 18 Jan 2014
# Mail: pyurutu@gmail.com
# This file contains the execution of CUDA code using PyCUDA
# Modified: 18 Jan 2014

import numpy

class cu_exe:
	cu_args = []
	args = []
	argl = 0
	retl = 0
	returns = []
	nam_returns = []
	nam_args = []
	id_ret = []
	is_alloc = []
	def exe_cu(self,stringg,func_name,threads,blocks,dyn_p,is_shared):
		try:
			import pycuda.driver as cuda
			import pycuda.autoinit
			from pycuda.compiler import SourceModule
		except:
			return
		if dyn_p is True:
			mod=SourceModule(stringg, options=['-rdc=true'],linkers=['-lcudadevrt'])
		else:
			mod=SourceModule(stringg)
		func=mod.get_function(func_name)
		if is_shared == True:
			func(*self.cu_args,block=(threads[0],threads[1],threads[2]),grid=(blocks[0],blocks[1],blocks[2]),shared=16*1024)
		else:
			func(*self.cu_args,block=(threads[0],threads[1],threads[2]),grid=(blocks[0],blocks[1],blocks[2]))

	def malloc(self,args,arg_nam):
		try:
			import pycuda.driver as cuda
			import pycuda.autoinit
		except:
			return
		self.args = args
		self.nam_args = arg_nam
		self.argl = len(args)
		for i in range(len(args)):
			self.cu_args.append(cuda.mem_alloc(args[i].nbytes))

	def htod(self,arg_nam):
		try:
			import pycuda.driver as cuda
			import pycuda.autoinit
		except:
			return
		index = self.nam_args.index(arg_nam)
		cuda.memcpy_htod(self.cu_args[index],self.args[index])

	def flags(self):
		for i in self.nam_args:
			self.is_alloc.append(False)
			if self.nam_returns.count(i) == 1:
				self.id_ret.append(self.nam_args.index(i))

	def dfree(self):
		import pycuda.driver as cuda
		import pycuda.autoinit
		for arg in self.cu_args:
			arg.free()
			self.cu_args.pop(0)

	def get_cu_args(self):
		return self.cu_args[:]

	def get_returns(self,returns,moved):
		for i in returns:
			if i in moved:
				print i
				self.dtoh(i)
			ret_id = self.nam_args.index(i)
			self.returns.append(self.args[ret_id])
		self.dfree()
		return self.returns

	def dtoh(self,nam):
		try:
			import pycuda.driver as cuda
			import pycuda.autoinit
		except:
			return
		var_id = self.nam_args.index(nam)
		cuda.memcpy_dtoh(self.args[var_id], self.cu_args[var_id]);
#		cuda.memcpy_dtoh(self.returns[i],self.cu_args[self.id_ret[i]])

# End of File
<commit_msg>dynamic parallelism tweak<commit_after># Developed by Aditya Atluri
# Date: 18 Jan 2014
# Mail: pyurutu@gmail.com
# This file contains the execution of CUDA code using PyCUDA
# Modified: 18 Jan 2014

import numpy

class cu_exe:
	cu_args = []
	args = []
	argl = 0
	retl = 0
	returns = []
	nam_returns = []
	nam_args = []
	id_ret = []
	is_alloc = []
	def exe_cu(self,stringg,func_name,threads,blocks,dyn_p,is_shared):
		try:
			import pycuda.driver as cuda
			import pycuda.autoinit
			from pycuda.compiler import SourceModule
		except:
			return
		if dyn_p is True:
			mod=SourceModule(stringg, options=['-rdc=true','-lcudadevrt'])
		else:
			mod=SourceModule(stringg)
		func=mod.get_function(func_name)
		if is_shared == True:
			func(*self.cu_args,block=(threads[0],threads[1],threads[2]),grid=(blocks[0],blocks[1],blocks[2]),shared=16*1024)
		else:
			func(*self.cu_args,block=(threads[0],threads[1],threads[2]),grid=(blocks[0],blocks[1],blocks[2]))

	def malloc(self,args,arg_nam):
		try:
			import pycuda.driver as cuda
			import pycuda.autoinit
		except:
			return
		self.args = args
		self.nam_args = arg_nam
		self.argl = len(args)
		for i in range(len(args)):
			self.cu_args.append(cuda.mem_alloc(args[i].nbytes))

	def htod(self,arg_nam):
		try:
			import pycuda.driver as cuda
			import pycuda.autoinit
		except:
			return
		index = self.nam_args.index(arg_nam)
		cuda.memcpy_htod(self.cu_args[index],self.args[index])

	def flags(self):
		for i in self.nam_args:
			self.is_alloc.append(False)
			if self.nam_returns.count(i) == 1:
				self.id_ret.append(self.nam_args.index(i))

	def dfree(self):
		import pycuda.driver as cuda
		import pycuda.autoinit
		for arg in self.cu_args:
			arg.free()
			self.cu_args.pop(0)

	def get_cu_args(self):
		return self.cu_args[:]

	def get_returns(self,returns,moved):
		for i in returns:
			if i in moved:
				print i
				self.dtoh(i)
			ret_id = self.nam_args.index(i)
			self.returns.append(self.args[ret_id])
		self.dfree()
		return self.returns

	def dtoh(self,nam):
		try:
			import pycuda.driver as cuda
			import pycuda.autoinit
		except:
			return
		var_id = self.nam_args.index(nam)
		cuda.memcpy_dtoh(self.args[var_id], self.cu_args[var_id]);
#		cuda.memcpy_dtoh(self.returns[i],self.cu_args[self.id_ret[i]])

# End of File
<|endoftext|>


<commit_before>##############################################################################
#
# Command line interface manager (CLIManager) for FreeRTOS+CLI
# File: CLIManager.py
# This file contains the code of the CLI manager
#
# This software is released under the MIT licence
#
# Copyright (c) 2015 Jean-Baptiste Quelard
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
###############################################################################
#!/usr/bin/python

import os
import re
import sys
import fileinput
import socket, errno
#import guipy
from gi.repository import Gtk, GObject	#TODO CHECK
from guipy import *

#Default parameters
CONFIG_FILENAME = "CLIManager.conf" #Configuration file
DefaultIP = "127.0.0.1"		  #Default IP address
DefaultPort = "5005"		  #Default port
DefaultType = "UDP"		  #Default protocol for the connection
DefaultColor = "ColorNone"	  #Default color scheme for the CLI
DefaultFont = "Helvetica 14"	  #Default font for the CLI
DefaultSyntaxAssistant = "False"    #Default setting for the syntax assistant option
DefaultEscapeChars = "False"	    #Default setting for the hide escape char option

class ConnectionManagement:

  def __init__(self):
    if os.path.exists(CONFIG_FILENAME) == False:
      #Create default config file
      self.CreateDefaultConfigFile()
    else:
      #Load Connection parameters from existing config file
      self.UDPAddress, self.UDPPort, self.TCPAddress, self.TCPPort = self.GetConnectionsConfigFromFile()
      self.ConnectionType = self.GetConnectionTypeFromFile()

    self.EventHandlerId = None
    self.IsConnected = False
    self.DataHandlerCallback = None

  def Connect(self, callback):
    self.IsConnected = False
    try:
      if self.ConnectionType == "UDP":
	self.socket = socket.socket(socket.AF_INET,
                      socket.SOCK_DGRAM) # UDP
	self.IsConnected = True

      if self.ConnectionType == "TCP":
	self.socket = socket.socket(socket.AF_INET,
                      socket.SOCK_STREAM) # TCP

	self.socket.connect((self.TCPAddress, int(self.TCPPort)))
	self.IsConnected = True

      self.EventHandlerId = GObject.io_add_watch(self.socket, GObject.IO_IN, self.SocketListener)
      self.DataHandlerCallback = callback #Function that will handle datas to display
      return None

    except socket.timeout:
      return "Socket timeout"

    except socket.error, (errno, strerror):
      return("Socket error: " + strerror)

  def Disconnect(self):
    if self.IsConnected:
      self.socket.close()
      self.IsConnected = False

    if self.EventHandlerId != None:
      GObject.source_remove(self.EventHandlerId)
      self.EventHandlerId = None

  def Send(self, command):
    if self.ConnectionType == "UDP":
      self.socket.sendto(command, (self.UDPAddress, int(self.UDPPort)))
    elif self.ConnectionType == "TCP":
      self.socket.sendto(command, (self.TCPAddress, int(self.TCPPort)))

    return(self.socket) #for the socket listener

  def Receive(self):
    Data = self.socket.recv( 1024 )
    return Data

  def SocketListener(self, source, condition):
      Data = self.Receive()
      self.DataHandlerCallback(Data)  #Let the GUI handle the data
      if len(Data) > 0:
	return True
      else:
	return False

  def CreateDefaultConfigFile(self):
    """ Create the default config file """
    with open(CONFIG_FILENAME, 'w') as ConfigFile:
      ConfigFile.write("#Connections\n")
      ConfigFile.write("<UDP:" + DefaultIP + ":" + DefaultPort + "\n" )
      ConfigFile.write("<TCP:" + DefaultIP + ":" + DefaultPort + "\n")
      ConfigFile.write("<Type:" +  DefaultType + "\n")
      ConfigFile.write("#Options\n")
      ConfigFile.write("<Color:" +  DefaultColor + "\n")
      ConfigFile.write("<Font:" + DefaultFont + "\n")
      ConfigFile.write("#Preferences\n")
      ConfigFile.write("<SyntaxAssistant:" +  DefaultSyntaxAssistant + "\n")
      ConfigFile.write("<EscapeChars:" + DefaultEscapeChars + "\n")

    ConfigFile.close()

    #Set default variables
    self.UDPAddress = DefaultIP
    self.UDPPort = DefaultPort
    self.TCPAddress = DefaultIP
    self.TCPPort = DefaultPort
    self.ConnectionType = DefaultType

  def GetConnectionsConfig(self):
    return self.UDPAddress, self.UDPPort, self.TCPAddress, self.TCPPort

  def GetUDPConnectionConfig(self):
    return self.UDPAddress, self.UDPPort

  def GetTCPConnectionConfig(self):
    return self.TCPAddress, self.TCPPort

  def GetConnectionsConfigFromFile(self):

    UDPPattern = re.compile("<UDP:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(.+)$")
    TCPPattern = re.compile("<TCP:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(.+)$")

    #Read the config file and look for the defined patterns
    with open(CONFIG_FILENAME, 'r') as ConfigFile:
      for line in ConfigFile:
	UDPmatch = re.search(UDPPattern,line)
	TCPmatch = re.search(TCPPattern,line)
	if UDPmatch:
	  self.UDPAddress =  UDPmatch.group(1)
	  self.UDPPort = UDPmatch.group(2)
	if TCPmatch:
	  self.TCPAddress =  TCPmatch.group(1)
	  self.TCPPort = TCPmatch.group(2)

      ConfigFile.close()
      return self.UDPAddress, self.UDPPort, self.TCPAddress, self.TCPPort

  def SetConnectionsConfig(self, UDPAddress, UDPPort, TCPAddress, TCPPort):

    UDPPattern = re.compile("<UDP:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(.+)$")
    TCPPattern = re.compile("<TCP:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(.+)$")

    NewConfigUDP = "<UDP:" + UDPAddress + ":" + UDPPort
    NewConfigTCP = "<TCP:" + TCPAddress + ":" + TCPPort

    for line in fileinput.input(CONFIG_FILENAME, inplace=1):
        line = UDPPattern.sub(NewConfigUDP, line)
        line = TCPPattern.sub(NewConfigTCP, line)

	#Avoid having extra blank lines in the config file
	if '\n' in line:
	  line = line.replace("\n", "")
	  print line
	else:
	  print line

    fileinput.close()

    #Update 'local' variables
    self.UDPAddress = UDPAddress
    self.UDPPort = UDPPort
    self.TCPAddress  = TCPAddress
    self.TCPPort = TCPPort

  def SetConnectionType(self, ConnectionType):
    self.ConnectionType = ConnectionType

    TypePattern = re.compile("<Type:(.+)$")

    NewConnectionType = "<Type:" + ConnectionType

    for line in fileinput.input(CONFIG_FILENAME, inplace=1):
      line = TypePattern.sub(NewConnectionType, line)

      #Avoid extra blank lines in the config file
      if '\n' in line:
	line = line.replace("\n", "")
	print line
      else:
	print line

    fileinput.close()

  def GetConnectionType(self):
    return self.ConnectionType

  def GetConnectionTypeFromFile(self):

    TypePattern = re.compile("<Type:(.+)$")

    with open(CONFIG_FILENAME, 'r') as ConfigFile:
      for line in ConfigFile:
	Typematch = re.search(TypePattern,line)
	if Typematch:
	  self.ConnectionType =  Typematch.group(1)

      ConfigFile.close()

    return self.ConnectionType

  def IsConnectionActive(self):
    return self.IsConnected #Tells the GUI if a connection is active

class CLIManager:

  def __init__(self):
    #Create an instance of the connection manager
    self.ConManager = ConnectionManagement()
    self.CommandsSetLoaded = False  #No set loaded
    self.HideEscapeChars = self.GetHideEscapeParamFromFile()    #Get preference
    self.HideSyntaxAssistant = self.GetHideSyntaxAssistantParamFromFile() #Get preference
    self.CLIColor = self.GetCLIColorFromFile()	#Get color scheme from config file
    self.CLIFont = self.GetCLIFontFromFile()  #Get font from the config file

  def IsCommandsSetLoaded(self):
    return self.CommandsSetLoaded #Tells GUI if a set of commands is loaded

  def SetCommandsSetLoaded(self, value):
    self.CommandsSetLoaded = value

  def GenerateCommandsSetFile(self, filename, liststore):
    #Generate the .set file from the list of commands currently loaded
    #(Imported from source file)
    with open(filename,"w") as cfile:
      for row in liststore:
	cfile.write('"' + row[0] + '","' + row[2] + '",' + str(row[1]) + '\n')
    cfile.close()

  def GetCLIColorFromFile(self):
    """ Get the prefered color scheme for the CLI from the config file """
    Pattern = re.compile("<Color:(.+)$")

    with open(CONFIG_FILENAME, 'r') as ConfigFile:
      for line in ConfigFile:
	match = re.search(Pattern,line)
	if match:
	  CLIColor =  match.group(1)

      ConfigFile.close()

    return CLIColor

  def GetCLIColorConfig(self):
    """ Tells the GUI which coor scheme to use """
    return self.CLIColor

  def SetCLIColorConfig(self, value):
    """ Set the prefered color scheme for the CLI """
    self.CLIColor = value

    Pattern = re.compile("<Color:(.+)$")

    Color = "<Color:" + value

    for line in fileinput.input(CONFIG_FILENAME, inplace=1):
      line = Pattern.sub(Color, line)

      if '\n' in line:
	line = line.replace("\n", "")
	print line
      else:
	print line

    fileinput.close()

  def GetCLIFontFromFile(self):
    """ Get the prefered font from the config file """
    CLIFont = DefaultFont
    Pattern = re.compile("<Font:(.+)$")

    with open(CONFIG_FILENAME, 'r') as ConfigFile:
      for line in ConfigFile:
	match = re.search(Pattern,line)
	if match:
	  CLIFont =  match.group(1)

      ConfigFile.close()

    return CLIFont

  def GetCLIFontConfig(self):
    """ Tells the GUI which font to use for the CLI """
    return self.CLIFont

  def SetCLIFontConfig(self, font):
    """ Set the prefered font for the CLI """
    self.CLIFont = font
    Pattern = re.compile("<Font:(.+)$")

    Font = "<Font:" + font

    for line in fileinput.input(CONFIG_FILENAME, inplace=1):
      line = Pattern.sub(Font, line)

      if '\n' in line:
	line = line.replace("\n", "")
	print line
      else:
	print line

    fileinput.close()

  def GetHideEscapeParamFromFile(self):
    """ Get the Hide escape chars option preference from the config file """
    Param = DefaultEscapeChars
    Pattern = re.compile("<EscapeChars:(.+)$")

    with open(CONFIG_FILENAME, 'r') as ConfigFile:
      for line in ConfigFile:
	match = re.search(Pattern,line)
	if match:
	  Param =  match.group(1)

      ConfigFile.close()

    if Param == "True":
      return True
    else:
      return False

  def GetHideEscapeParam(self):
    """ Tells the GUI if the option is enabled """
    return self.HideEscapeChars

  def SetHideEscapeParam(self, value):
    """ Set the option state """
    self.HideEscapeChars = value

    Pattern = re.compile("<EscapeChars:(.+)$")

    Param = "<EscapeChars:" + str(value)

    for line in fileinput.input(CONFIG_FILENAME, inplace=1):
      line = Pattern.sub(Param, line)

      if '\n' in line:
	line = line.replace("\n", "")
	print line
      else:
	print line

    fileinput.close()

  def SetHideEscapeCharColumn(self, liststore):
    """ Remove the escape characters \r and \n from the help string and copy it in the 3rd column """
    for row in liststore:
      row[3] = row[2].replace("\\n", "")
      row[3] = row[3].replace("\\r", "")

  def GetHideSyntaxAssistantParamFromFile(self):
    """ Get the Hide syntax assistant option preference from the config file """
    Param = DefaultSyntaxAssistant
    Pattern = re.compile("<SyntaxAssistant:(.+)$")

    with open(CONFIG_FILENAME, 'r') as ConfigFile:
      for line in ConfigFile:
	match = re.search(Pattern,line)
	if match:
	  Param =  match.group(1)

      ConfigFile.close()

    if Param == "True":
      return True
    else:
      return False

  def GetHideSyntaxAssistantParam(self):
    """ Tells the GUI if the option is enabled """
    return self.HideSyntaxAssistant

  def SetHideSyntaxAssistantParam(self, value):
    """ Set the option state """
    self.HideSyntaxAssistant = value

    Pattern = re.compile("<SyntaxAssistant:(.+)$")

    Param = "<SyntaxAssistant:" + str(value)

    for line in fileinput.input(CONFIG_FILENAME, inplace=1):
      line = Pattern.sub(Param, line)

      if '\n' in line:
	line = line.replace("\n", "")
	print line
      else:
	print line

    fileinput.close()

if __name__ == "__main__":

	app = CLIManager()
	win = MainWindow(app)
	win.connect("delete-event", Gtk.main_quit)
	win.show_all()
	Gtk.main()

<commit_msg>load user preferences at once when the app starts<commit_after>##############################################################################
#
# Command line interface manager (CLIManager) for FreeRTOS+CLI
# File: CLIManager.py
# This file contains the code of the CLI manager
#
# This software is released under the MIT licence
#
# Copyright (c) 2015 Jean-Baptiste Quelard
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
###############################################################################
#!/usr/bin/python

import os
import re
import sys
import fileinput
import socket, errno
#import guipy
from gi.repository import Gtk, GObject	#TODO CHECK
from guipy import *

#Default parameters
CONFIG_FILENAME = "CLIManager.conf" #Configuration file
DefaultIP = "127.0.0.1"		  #Default IP address
DefaultPort = "5005"		  #Default port
DefaultType = "UDP"		  #Default protocol for the connection
DefaultColor = "ColorNone"	  #Default color scheme for the CLI
DefaultFont = "Helvetica 14"	  #Default font for the CLI
DefaultSyntaxAssistant = "False"    #Default setting for the syntax assistant option
DefaultEscapeChars = "False"	    #Default setting for the hide escape char option

class ConnectionManagement:

  def __init__(self):
    if os.path.exists(CONFIG_FILENAME) == False:
      #Create default config file
      self.CreateDefaultConfigFile()
    else:
      #Load Connection parameters from existing config file
      self.UDPAddress, self.UDPPort, self.TCPAddress, self.TCPPort = self.GetConnectionsConfigFromFile()
      self.ConnectionType = self.GetConnectionTypeFromFile()

    self.EventHandlerId = None
    self.IsConnected = False
    self.DataHandlerCallback = None

  def Connect(self, callback):
    self.IsConnected = False
    try:
      if self.ConnectionType == "UDP":
	self.socket = socket.socket(socket.AF_INET,
                      socket.SOCK_DGRAM) # UDP
	self.IsConnected = True

      if self.ConnectionType == "TCP":
	self.socket = socket.socket(socket.AF_INET,
                      socket.SOCK_STREAM) # TCP

	self.socket.connect((self.TCPAddress, int(self.TCPPort)))
	self.IsConnected = True

      self.EventHandlerId = GObject.io_add_watch(self.socket, GObject.IO_IN, self.SocketListener)
      self.DataHandlerCallback = callback #Function that will handle datas to display
      return None

    except socket.timeout:
      return "Socket timeout"

    except socket.error, (errno, strerror):
      return("Socket error: " + strerror)

  def Disconnect(self):
    if self.IsConnected:
      self.socket.close()
      self.IsConnected = False

    if self.EventHandlerId != None:
      GObject.source_remove(self.EventHandlerId)
      self.EventHandlerId = None

  def Send(self, command):
    if self.ConnectionType == "UDP":
      self.socket.sendto(command, (self.UDPAddress, int(self.UDPPort)))
    elif self.ConnectionType == "TCP":
      self.socket.sendto(command, (self.TCPAddress, int(self.TCPPort)))

    return(self.socket) #for the socket listener

  def Receive(self):
    Data = self.socket.recv( 1024 )
    return Data

  def SocketListener(self, source, condition):
      Data = self.Receive()
      self.DataHandlerCallback(Data)  #Let the GUI handle the data
      if len(Data) > 0:
	return True
      else:
	return False

  def CreateDefaultConfigFile(self):
    """ Create the default config file """
    with open(CONFIG_FILENAME, 'w') as ConfigFile:
      ConfigFile.write("#Connections\n")
      ConfigFile.write("<UDP:" + DefaultIP + ":" + DefaultPort + "\n" )
      ConfigFile.write("<TCP:" + DefaultIP + ":" + DefaultPort + "\n")
      ConfigFile.write("<Type:" +  DefaultType + "\n")
      ConfigFile.write("#Options\n")
      ConfigFile.write("<Color:" +  DefaultColor + "\n")
      ConfigFile.write("<Font:" + DefaultFont + "\n")
      ConfigFile.write("#Preferences\n")
      ConfigFile.write("<SyntaxAssistant:" +  DefaultSyntaxAssistant + "\n")
      ConfigFile.write("<EscapeChars:" + DefaultEscapeChars + "\n")

    ConfigFile.close()

    #Set default variables
    self.UDPAddress = DefaultIP
    self.UDPPort = DefaultPort
    self.TCPAddress = DefaultIP
    self.TCPPort = DefaultPort
    self.ConnectionType = DefaultType

  def GetConnectionsConfig(self):
    return self.UDPAddress, self.UDPPort, self.TCPAddress, self.TCPPort

  def GetUDPConnectionConfig(self):
    return self.UDPAddress, self.UDPPort

  def GetTCPConnectionConfig(self):
    return self.TCPAddress, self.TCPPort

  def GetConnectionsConfigFromFile(self):

    UDPPattern = re.compile("<UDP:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(.+)$")
    TCPPattern = re.compile("<TCP:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(.+)$")

    #Read the config file and look for the defined patterns
    with open(CONFIG_FILENAME, 'r') as ConfigFile:
      for line in ConfigFile:
	UDPmatch = re.search(UDPPattern,line)
	TCPmatch = re.search(TCPPattern,line)
	if UDPmatch:
	  self.UDPAddress =  UDPmatch.group(1)
	  self.UDPPort = UDPmatch.group(2)
	if TCPmatch:
	  self.TCPAddress =  TCPmatch.group(1)
	  self.TCPPort = TCPmatch.group(2)

      ConfigFile.close()
      return self.UDPAddress, self.UDPPort, self.TCPAddress, self.TCPPort

  def SetConnectionsConfig(self, UDPAddress, UDPPort, TCPAddress, TCPPort):

    UDPPattern = re.compile("<UDP:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(.+)$")
    TCPPattern = re.compile("<TCP:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(.+)$")

    NewConfigUDP = "<UDP:" + UDPAddress + ":" + UDPPort
    NewConfigTCP = "<TCP:" + TCPAddress + ":" + TCPPort

    for line in fileinput.input(CONFIG_FILENAME, inplace=1):
        line = UDPPattern.sub(NewConfigUDP, line)
        line = TCPPattern.sub(NewConfigTCP, line)

	#Avoid having extra blank lines in the config file
	if '\n' in line:
	  line = line.replace("\n", "")
	  print line
	else:
	  print line

    fileinput.close()

    #Update 'local' variables
    self.UDPAddress = UDPAddress
    self.UDPPort = UDPPort
    self.TCPAddress  = TCPAddress
    self.TCPPort = TCPPort

  def SetConnectionType(self, ConnectionType):
    self.ConnectionType = ConnectionType

    TypePattern = re.compile("<Type:(.+)$")

    NewConnectionType = "<Type:" + ConnectionType

    for line in fileinput.input(CONFIG_FILENAME, inplace=1):
      line = TypePattern.sub(NewConnectionType, line)

      #Avoid extra blank lines in the config file
      if '\n' in line:
	line = line.replace("\n", "")
	print line
      else:
	print line

    fileinput.close()

  def GetConnectionType(self):
    return self.ConnectionType

  def GetConnectionTypeFromFile(self):

    TypePattern = re.compile("<Type:(.+)$")

    with open(CONFIG_FILENAME, 'r') as ConfigFile:
      for line in ConfigFile:
	Typematch = re.search(TypePattern,line)
	if Typematch:
	  self.ConnectionType =  Typematch.group(1)

      ConfigFile.close()

    return self.ConnectionType

  def IsConnectionActive(self):
    return self.IsConnected #Tells the GUI if a connection is active

class CLIManager:

  def __init__(self):
    #Create an instance of the connection manager
    self.ConManager = ConnectionManagement()
    self.CommandsSetLoaded = False  #No set loaded
    self.HideEscapeChars = DefaultEscapeChars
    self.HideSyntaxAssistant = DefaultSyntaxAssistant
    self.CLIColor = DefaultColor
    self.CLIFont = DefaultFont
    self.GetPreferencesFromConfigFile()	# Load user preferences from config file

  def IsCommandsSetLoaded(self):
    return self.CommandsSetLoaded #Tells GUI if a set of commands is loaded

  def SetCommandsSetLoaded(self, value):
    self.CommandsSetLoaded = value

  def GenerateCommandsSetFile(self, filename, liststore):
    #Generate the .set file from the list of commands currently loaded
    #(Imported from source file)
    with open(filename,"w") as cfile:
      for row in liststore:
	cfile.write('"' + row[0] + '","' + row[2] + '",' + str(row[1]) + '\n')
    cfile.close()

  def GetPreferencesFromConfigFile(self):
    """ Retrieve parameters and preferences from the configuration file """

    # Define regex for pattern matching in the config file
    CLIColorPattern = re.compile("<Color:(.+)$")
    CLIFontPattern = re.compile("<Font:(.+)$")
    EscapeCharsPattern = re.compile("<EscapeChars:(.+)$")
    SyntaxAssistantPattern = re.compile("<SyntaxAssistant:(.+)$")

    with open(CONFIG_FILENAME, 'r') as ConfigFile:
      for line in ConfigFile:
	CLIColorMatch = re.search(CLIColorPattern,line)
	CLIFontMatch = re.search(CLIFontPattern,line)
	EscapeCharsMatch = re.search(EscapeCharsPattern,line)
	SyntaxAssistantMatch = re.search(SyntaxAssistantPattern,line)

	if CLIColorMatch:
	  self.CLIColor = CLIColorMatch.group(1)
	elif CLIFontMatch:
	  self.CLIFont = CLIFontMatch.group(1)
	elif EscapeCharsMatch:
	  if EscapeCharsMatch.group(1) == 'True':
	    self.HideEscapeChars = True
	  else:
	    self.HideEscapeChars = False
	elif SyntaxAssistantMatch:
	  if SyntaxAssistantMatch.group(1) == 'True':
	    self.HideSyntaxAssistant = True
	  else:
	    self.HideSyntaxAssistant = False

      ConfigFile.close()

  def GetCLIColorConfig(self):
    """ Tells the GUI which coor scheme to use """
    return self.CLIColor

  def SetCLIColorConfig(self, value):
    """ Set the prefered color scheme for the CLI """
    self.CLIColor = value

    Pattern = re.compile("<Color:(.+)$")

    Color = "<Color:" + value

    for line in fileinput.input(CONFIG_FILENAME, inplace=1):
      line = Pattern.sub(Color, line)

      if '\n' in line:
	line = line.replace("\n", "")
	print line
      else:
	print line

    fileinput.close()

  def GetCLIFontConfig(self):
    """ Tells the GUI which font to use for the CLI """
    return self.CLIFont

  def SetCLIFontConfig(self, font):
    """ Set the prefered font for the CLI """
    self.CLIFont = font
    Pattern = re.compile("<Font:(.+)$")

    Font = "<Font:" + font

    for line in fileinput.input(CONFIG_FILENAME, inplace=1):
      line = Pattern.sub(Font, line)

      if '\n' in line:
	line = line.replace("\n", "")
	print line
      else:
	print line

    fileinput.close()

  def GetHideEscapeParam(self):
    """ Tells the GUI if the option is enabled """
    return self.HideEscapeChars

  def SetHideEscapeParam(self, value):
    """ Set the option state """
    self.HideEscapeChars = value

    Pattern = re.compile("<EscapeChars:(.+)$")

    Param = "<EscapeChars:" + str(value)

    for line in fileinput.input(CONFIG_FILENAME, inplace=1):
      line = Pattern.sub(Param, line)

      if '\n' in line:
	line = line.replace("\n", "")
	print line
      else:
	print line

    fileinput.close()

  def SetHideEscapeCharColumn(self, liststore):
    """ Remove the escape characters \r and \n from the help string and copy it in the 3rd column """
    for row in liststore:
      row[3] = row[2].replace("\\n", "")
      row[3] = row[3].replace("\\r", "")

  def GetHideSyntaxAssistantParam(self):
    """ Tells the GUI if the option is enabled """
    return self.HideSyntaxAssistant

  def SetHideSyntaxAssistantParam(self, value):
    """ Set the option state """
    self.HideSyntaxAssistant = value

    Pattern = re.compile("<SyntaxAssistant:(.+)$")

    Param = "<SyntaxAssistant:" + str(value)

    for line in fileinput.input(CONFIG_FILENAME, inplace=1):
      line = Pattern.sub(Param, line)

      if '\n' in line:
	line = line.replace("\n", "")
	print line
      else:
	print line

    fileinput.close()

if __name__ == "__main__":

	app = CLIManager()
	win = MainWindow(app)
	win.connect("delete-event", Gtk.main_quit)
	win.show_all()
	Gtk.main()

<|endoftext|>


<commit_before>import boto3

from EOSS.aws.utils import dev_client, prod_client
from EOSS.aws.utils import eval_subnet

class Service:

    def __init__(self, cluster_arn, dev=False):
        if dev:
            self.client = dev_client('ecs')
        else:
            self.client = prod_client('ecs')
        self.cluster_arn = str(cluster_arn)

    def build_service(self, problem_id, task_definition_arn):
        print('\n\n---------- BUILDING NEW SERVICE ----------')
        service_arn = self.does_service_exist(problem_id)
        if service_arn is not None:
            print('--> SERVICE ALREADY EXISTS')
            return service_arn
        else:
            print('--> CREATING SERVICE')
            service_name = Service.formulate_service_name(problem_id)
            response = self.client.create_service(
                cluster=self.cluster_arn,
                serviceName=service_name,
                taskDefinition=task_definition_arn,
                desiredCount=1,
                launchType='FARGATE',
                networkConfiguration={
                    'awsvpcConfiguration': {
                        'subnets': [eval_subnet()],
                        'assignPublicIp': 'DISABLED'
                    }
                },
                schedulingStrategy='REPLICA',
                deploymentController={'type': 'ECS'},
                tags=[
                    {'key': 'PROBLEM_ID', 'value': str(problem_id)},
                    {'key': 'TYPE', 'value': 'EVAL'}
                ]
            )
            print('--> CREATE SERVICE RESPONSE', response)
            return response['service']['serviceArn']

    def does_service_exist(self, problem_id):
        service_name = Service.formulate_service_name(problem_id)
        service_arns = self.get_cluster_service_arns()
        response = self.client.describe_services(
            cluster=self.cluster_arn,
            services=service_arns,
        )
        for service in response['services']:
            if service['serviceName'] == service_name:
                return service['serviceArn']
        return None

    def get_cluster_service_arns(self):
        response = self.client.list_services(
            cluster=self.cluster_arn,
            launchType='FARGATE',
        )
        return response['serviceArns']

    @staticmethod
    def formulate_service_name(problem_id):
        return 'evaluator-service-' + str(problem_id)

<commit_msg>enabled task deregistration for testing<commit_after>import boto3

from EOSS.aws.utils import dev_client, prod_client
from EOSS.aws.utils import eval_subnet

class Service:

    def __init__(self, cluster_arn, dev=False):
        if dev:
            self.client = dev_client('ecs')
        else:
            self.client = prod_client('ecs')
        self.cluster_arn = str(cluster_arn)

    def build_service(self, problem_id, task_definition_arn):
        print('\n\n---------- BUILDING NEW SERVICE ----------')
        service_arn = self.does_service_exist(problem_id)
        if service_arn is not None:
            print('--> SERVICE ALREADY EXISTS')
            return service_arn
        else:
            print('--> CREATING SERVICE')
            service_name = Service.formulate_service_name(problem_id)
            response = self.client.create_service(
                cluster=self.cluster_arn,
                serviceName=service_name,
                taskDefinition=task_definition_arn,
                desiredCount=1,
                launchType='FARGATE',
                networkConfiguration={
                    'awsvpcConfiguration': {
                        'subnets': [eval_subnet()],
                        'assignPublicIp': 'DISABLED'
                    }
                },
                schedulingStrategy='REPLICA',
                deploymentController={'type': 'ECS'},
                tags=[
                    {'key': 'PROBLEM_ID', 'value': str(problem_id)},
                    {'key': 'TYPE', 'value': 'EVAL'}
                ]
            )
            print('--> CREATE SERVICE RESPONSE', response)
            return response['service']['serviceArn']

    def does_service_exist(self, problem_id):
        service_name = Service.formulate_service_name(problem_id)
        service_arns = self.get_cluster_service_arns()
        if not service_arns:
            return None
        response = self.client.describe_services(
            cluster=self.cluster_arn,
            services=service_arns,
        )
        for service in response['services']:
            if service['serviceName'] == service_name:
                return service['serviceArn']
        return None

    def get_cluster_service_arns(self):
        response = self.client.list_services(
            cluster=self.cluster_arn,
            launchType='FARGATE',
        )
        return response['serviceArns']

    @staticmethod
    def formulate_service_name(problem_id):
        return 'evaluator-service-' + str(problem_id)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Extend werkzeug request handler to suit our needs."""

import time
from werkzeug.serving import BaseRequestHandler

class ShRequestHandler(BaseRequestHandler):
    """Extend werkzeug request handler to suit our needs."""
    def handle(self):
        self.shRequestStarted = time.time()
        rv = super(ShRequestHandler, self).handle()
        return rv

    def send_response(self, *args, **kw):
        self.shRequestProcessed = time.time()
        super(ShRequestHandler, self).send_response(*args, **kw)

    def log_request(self, code='-', size='-'):
        duration = int((self.shRequestProcessed - self.shRequestStarted) * 1000)
        self.log('info', '"{0}" {1} {2} [{3}ms]'.format(self.requestline, code, size, duration))

<commit_msg>Handle logging when a '%' character is in the URL.<commit_after># -*- coding: utf-8 -*-

"""Extend werkzeug request handler to suit our needs."""

import time
from werkzeug.serving import BaseRequestHandler

class ShRequestHandler(BaseRequestHandler):
    """Extend werkzeug request handler to suit our needs."""
    def handle(self):
        self.shRequestStarted = time.time()
        rv = super(ShRequestHandler, self).handle()
        return rv

    def send_response(self, *args, **kw):
        self.shRequestProcessed = time.time()
        super(ShRequestHandler, self).send_response(*args, **kw)

    def log_request(self, code='-', size='-'):
        duration = int((self.shRequestProcessed - self.shRequestStarted) * 1000)
        self.log('info', u'"{0}" {1} {2} [{3}ms]'.format(self.requestline.replace('%', '%%'), code, size, duration))

<|endoftext|>


<commit_before>import pandas as pd
import numpy as np
import logging

import Pulses
import C_Funct
import Utilities

def candidates(pulses,idL):
  pulses.sort(['Pulse','Sigma'],ascending=[1,0],inplace=True)

  pulses.Candidate[pulses.Pulse==0] = Repeated_candidates_beam(pulses[pulses.Pulse==0],0)

  if pulses.Candidate.unique().size <=12:
    pulses.Candidate[(pulses.Pulse<=1)&(pulses.Candidate<0)] = Repeated_candidates_beam(pulses[(pulses.Pulse<=1)&(pulses.Candidate<0)],1)

  if pulses.Candidate.unique().size <=12:
    pulses.Candidate[pulses.Candidate<0] = Repeated_candidates_beam(pulses[pulses.Candidate<0],2)

  cands_unique = pulses[(pulses.Candidate==-1)&(pulses.Sigma>=10)].groupby(['SAP','BEAM'],sort=False)[['SAP','BEAM']].head(5).astype(np.int32)
  #ATTENZIONE: aumentare head(2) e selezionare head(2) solo dopo aver visto quali non sono in altri candidates
  pulses.Candidate.loc[cands_unique.index.get_level_values('idx')] = np.arange(cands_unique.shape[0]) * 1000 + cands_unique.SAP * 100 + cands_unique.BEAM

  if not pulses[pulses.Candidate>=0].empty:
    cands = candidates_generator(pulses[pulses.Candidate>=0].copy(),idL)
    cands['main_cand'] = 0

    #Unify the same repeated candidates in different beams
    cands.sort(['Rank','Sigma'],ascending=[0,1],inplace=True)
    new_cand = cands.index

    C_Funct.Compare_candidates(cands.DM.astype(np.float32).values,cands.Time.astype(np.float32).values,cands.index.values,cands.main_cand.values)

  else: cands = pd.DataFrame()

  return cands

def Repeated_candidates_beam(pulses,rank):
  gb_puls = pulses.groupby(['SAP','BEAM'],sort=False)
  dirs = [n for n in gb_puls.indices.iterkeys()]
  pulses['cand'] = -1

  for (sap,beam) in dirs:
    puls = pulses[(pulses.SAP==sap)&(pulses.BEAM==beam)].copy()
    puls.DM = 3*(puls.DM.astype(np.float64)/3).round(2)

    span = 0.25

    top_count = puls.groupby('DM')['Sigma'].count()
    top_sum = puls.groupby('DM')['Sigma'].sum()

    top_sum = top_sum[top_count >= 2]
    #top_count = top_count[top_count >= 2]

    i = 1

    while not top_sum[top_sum!=0].empty:
      DM = top_sum.argmax()
      #Sigma = top_sum.loc[DM-span:DM+span].sum()
      #N_puls = top_count.loc[DM-span:DM+span].sum()
      selected_pulses = puls.cand[(puls.DM>=DM-span)&(puls.DM<=DM+span)]
      if selected_pulses.shape[0] > 1:
        pulses.cand.loc[selected_pulses.index] = i * 100000 + sap * 1000 + beam * 10 + rank
      #top_count.loc[DM-span:DM+span] = 0
      top_sum.loc[DM-span:DM+span] = 0
      i += 1

  return pulses.cand

def period(x):
  if x.size<=1: return 0
  else: return Utilities.rrat_period(x)[0]

def period_err(x):
  if x.size<=1: return 0
  else: return Utilities.rrat_period(x)[1]

def candidates_generator(pulses,idL):
  pulses['Period'] = pulses.Time
  pulses['Period_err'] = pulses.Time

  cands = pulses.groupby(['Candidate','SAP','BEAM'],as_index=False,sort=False).agg({'Sigma':np.sum,'N_events':np.size,'DM':np.mean,'Time':np.min,'Period':period,'Period_err':period_err,'Pulse':np.max})

  cands = cands.astype(np.float32)
  cands[['N_events','Pulse','SAP','BEAM']] = cands[['N_events','Pulse','SAP','BEAM']].astype(np.int16)

  cands.index = cands.Candidate.astype(int)
  cands.index.name = 'idx'
  cands.rename(columns={'N_events': 'N_pulses', 'Pulse': 'Rank'}, inplace=True)
  cands = cands.drop('Candidate',axis=1)
  cands.Time[cands.N_pulses>1] = 0
  cands['id'] = idL + '_' + cands.SAP.astype(str) + '_' + cands.BEAM.astype(str) + '_' + cands.index.astype(str)

  cands.sort(['Rank','Sigma'],ascending=[1,0],inplace=True)
  best_cands = cands[cands.N_pulses==1].groupby('SAP').head(4)
  best_cands = best_cands.append(cands[cands.N_pulses>1].groupby('BEAM').head(2).groupby('SAP').head(4))

  return best_cands
<commit_msg>Candidates selection fixed<commit_after>import pandas as pd
import numpy as np
import logging

import Pulses
import C_Funct
import Utilities

def candidates(pulses,idL):
  pulses.sort(['Pulse','Sigma'],ascending=[1,0],inplace=True)

  pulses.Candidate[pulses.Pulse==0] = Repeated_candidates_beam(pulses[pulses.Pulse==0],0)

  if pulses.Candidate.unique().size <=12:
    pulses.Candidate[(pulses.Pulse<=1)&(pulses.Candidate<0)] = Repeated_candidates_beam(pulses[(pulses.Pulse<=1)&(pulses.Candidate<0)],1)

  if pulses.Candidate.unique().size <=12:
    pulses.Candidate[pulses.Candidate<0] = Repeated_candidates_beam(pulses[pulses.Candidate<0],2)

  cands_unique = pulses[(pulses.Candidate==-1)&(pulses.Sigma>=10)].groupby(['SAP','BEAM'],sort=False)[['SAP','BEAM']].head(5).astype(np.int32)
  pulses.Candidate.loc[cands_unique.index.get_level_values('idx')] = np.arange(cands_unique.shape[0]) * 1000 + cands_unique.SAP * 100 + cands_unique.BEAM

  if not pulses[pulses.Candidate>=0].empty:
    cands = candidates_generator(pulses[pulses.Candidate>=0].copy(),idL)
    cands['main_cand'] = 0

    #Unify the same repeated candidates in different beams
    cands.sort(['Rank','Sigma'],ascending=[0,1],inplace=True)
    new_cand = cands.index

    C_Funct.Compare_candidates(cands.DM.astype(np.float32).values,cands.Time.astype(np.float32).values,cands.index.values,cands.main_cand.values)

    #selezionare solo top 2 signle candidates

  else: cands = pd.DataFrame()

  return cands

def Repeated_candidates_beam(pulses,rank):
  gb_puls = pulses.groupby(['SAP','BEAM'],sort=False)
  dirs = [n for n in gb_puls.indices.iterkeys()]
  pulses['cand'] = -1

  for (sap,beam) in dirs:
    puls = pulses[(pulses.SAP==sap)&(pulses.BEAM==beam)].copy()
    puls.DM = 3*(puls.DM.astype(np.float64)/3).round(2)

    span = 0.25

    top_count = puls.groupby('DM')['Sigma'].count()
    top_sum = puls.groupby('DM')['Sigma'].sum()

    top_sum = top_sum[top_count >= 2]
    #top_count = top_count[top_count >= 2]

    i = 1

    while not top_sum[top_sum!=0].empty:
      DM = top_sum.argmax()
      #Sigma = top_sum.loc[DM-span:DM+span].sum()
      #N_puls = top_count.loc[DM-span:DM+span].sum()
      selected_pulses = puls.cand[(puls.DM>=DM-span)&(puls.DM<=DM+span)]
      if selected_pulses.shape[0] > 1:
        pulses.cand.loc[selected_pulses.index] = i * 100000 + sap * 1000 + beam * 10 + rank
      #top_count.loc[DM-span:DM+span] = 0
      top_sum.loc[DM-span:DM+span] = 0
      i += 1

  return pulses.cand

def period(x):
  if x.size<=1: return 0
  else: return Utilities.rrat_period(x)[0]

def period_err(x):
  if x.size<=1: return 0
  else: return Utilities.rrat_period(x)[1]

def candidates_generator(pulses,idL):
  pulses['Period'] = pulses.Time
  pulses['Period_err'] = pulses.Time

  cands = pulses.groupby(['Candidate','SAP','BEAM'],as_index=False,sort=False).agg({'Sigma':np.sum,'N_events':np.size,'DM':np.mean,'Time':np.min,'Period':period,'Period_err':period_err,'Pulse':np.max})

  cands = cands.astype(np.float32)
  cands[['N_events','Pulse','SAP','BEAM']] = cands[['N_events','Pulse','SAP','BEAM']].astype(np.int16)

  cands.index = cands.Candidate.astype(int)
  cands.index.name = 'idx'
  cands.rename(columns={'N_events': 'N_pulses', 'Pulse': 'Rank'}, inplace=True)
  cands = cands.drop('Candidate',axis=1)
  cands.Time[cands.N_pulses>1] = 0
  cands['id'] = idL + '_' + cands.SAP.astype(str) + '_' + cands.BEAM.astype(str) + '_' + cands.index.astype(str)

  cands.sort(['Rank','Sigma'],ascending=[1,0],inplace=True)
  best_cands = cands[cands.N_pulses==1].groupby('SAP').head(4)
  best_cands = best_cands.append(cands[cands.N_pulses>1].groupby('BEAM').head(2).groupby('SAP').head(4))

  return best_cands
<|endoftext|>


<commit_before>from flask.sessions import SessionInterface, SessionMixin
import os

class SessionData(dict, SessionMixin): pass

class Session(SessionInterface):
    session_class = SessionData

    def open_session(self, app, request):
        self.cookie_session_id = request.cookies.get(app.session_cookie_name, None)
        self.session_new = False
        if self.cookie_session_id is None:
            self.cookie_session_id = os.urandom(40).encode('hex')
            self.session_new = True
        self.memcache_session_id = '@'.join(
                    [
                        request.remote_addr,
                        self.cookie_session_id
                    ]
                )
        app.logger.debug('Open session %s', self.memcache_session_id)
        return self.session_class(app.cache.get(self.memcache_session_id) or {})

    def save_session(self, app, session, response):
        expires = self.get_expiration_time(app, session)
        domain = self.get_cookie_domain(app)
        path = self.get_cookie_path(app)
        httponly = self.get_cookie_httponly(app)
        secure = self.get_cookie_secure(app)
        app.cache.set(self.memcache_session_id, session)
        if self.session_new:
            response.set_cookie(app.session_cookie_name, self.cookie_session_id, path=path,
                                expires=expires, httponly=httponly,
                                secure=secure, domain=domain)
            app.logger.debug('Set session %s with %s', self.memcache_session_id, session)
<commit_msg>Now timeout update after fetching<commit_after>from flask.sessions import SessionInterface, SessionMixin
import os

class SessionData(dict, SessionMixin): pass

class Session(SessionInterface):
    session_class = SessionData

    def open_session(self, app, request):
        self.cookie_session_id = request.cookies.get(app.session_cookie_name, None)
        self.session_new = False
        if self.cookie_session_id is None:
            self.cookie_session_id = os.urandom(40).encode('hex')
            self.session_new = True
        self.memcache_session_id = '@'.join(
                    [
                        request.remote_addr,
                        self.cookie_session_id
                    ]
                )
        app.logger.debug('Open session %s', self.memcache_session_id)
        session = app.cache.get(self.memcache_session_id) or {}
        app.cache.set(self.memcache_session_id, session)
        return self.session_class(session)

    def save_session(self, app, session, response):
        expires = self.get_expiration_time(app, session)
        domain = self.get_cookie_domain(app)
        path = self.get_cookie_path(app)
        httponly = self.get_cookie_httponly(app)
        secure = self.get_cookie_secure(app)
        app.cache.set(self.memcache_session_id, session)
        if self.session_new:
            response.set_cookie(app.session_cookie_name, self.cookie_session_id, path=path,
                                expires=expires, httponly=httponly,
                                secure=secure, domain=domain)
            app.logger.debug('Set session %s with %s', self.memcache_session_id, session)
<|endoftext|>


<commit_before>#!/usr/bin/python3

from ScaleManager import ScaleManager
from FileManager import FileManager
#import GUI as UI
from ConfigReader import configReader

class SpudScale() :

    def __init__(self) :
        self.fileName = ''

        configInfo = configReader()
        comPorts = configInfo['ports']
        self.inputTitle = configInfo['input']
        self.orderedNames = configInfo['ordered']
        self.scaleAddressNames = configInfo['scales']

        self.numScales = len(comPorts)
        self.numInputs = len(self.inputTitle)
        self.numOutputs  = self.numInputs + self.numScales
        self.currentValues = [None]*self.numScales
        self.lastFiveRecorded = [[None]*(self.numOutputs)]*5

        #self.ui = UI(inputTitle, self.orderedNames)

        self.scaleManager = ScaleManager(self.scaleAddressNames, comPorts)

        self.fileManager = FileManager('testOutput.csv')
        self.fileManager.record(self.inputTitle + self.orderedNames)

    def record(self, userInput) :
        recordEntry = [userInput] + self.getCurrentValues()
        self.fileManager.record(recordEntry)
        self.updateLastFiveRecorded(recordEntry)

    def getCurrentValues(self) :
        newValues = self.scaleManager.getValues()
        for i in range(0, self.numScales) :
            name = self.orderedNames[i]
            self.currentValues[i] = newValues[name]
        return self.currentValues

    def getLastFiveRecorded(self) :
        return self.lastFiveRecorded

    def setFileName(self, newFileName) :
        self.fileName = newFileName

    def updateLastFiveRecorded(self, recordEntry) :
        i = len(self.lastFiveRecorded) - 1
        while i > 0 :
            self.lastFiveRecorded[i] = self.lastFiveRecorded[i - 1]
            i -= 1
        self.lastFiveRecorded[0] = recordEntry

def main() :
    import time
    spudScale = SpudScale()
    time.sleep(10)
    for i in range(0,100) :
        print(spudScale.getCurrentValues())
        spudScale.record('Line' + str(i))
        if (i%5 == 0) :
            print(spudScale.getLastFiveRecorded())
        time.sleep(.5)
    return 0

if __name__ == '__main__':
    main()

"""
getScaleNames()
getCurrentValues()
recordValues(plotNumber)
getLastFive()
Is there anything else it would need?
.setFileName(...)
"""
<commit_msg>Added argument to UI<commit_after>#!/usr/bin/python3

from ScaleManager import ScaleManager
from FileManager import FileManager
#import GUI as UI
from ConfigReader import configReader

class SpudScale() :

    def __init__(self) :
        self.fileName = ''

        configInfo = configReader()
        comPorts = configInfo['ports']
        self.inputTitle = configInfo['input']
        self.orderedNames = configInfo['ordered']
        self.scaleAddressNames = configInfo['scales']

        self.numScales = len(comPorts)
        self.numInputs = len(self.inputTitle)
        self.numOutputs  = self.numInputs + self.numScales
        self.currentValues = [None]*self.numScales
        self.lastFiveRecorded = [[None]*(self.numOutputs)]*5

        #self.ui = UI(self, inputTitle, self.orderedNames)

        self.scaleManager = ScaleManager(self.scaleAddressNames, comPorts)

        self.fileManager = FileManager('testOutput.csv')
        self.fileManager.record(self.inputTitle + self.orderedNames)

    def record(self, userInput) :
        recordEntry = [userInput] + self.getCurrentValues()
        self.fileManager.record(recordEntry)
        self.updateLastFiveRecorded(recordEntry)

    def getCurrentValues(self) :
        newValues = self.scaleManager.getValues()
        for i in range(0, self.numScales) :
            name = self.orderedNames[i]
            self.currentValues[i] = newValues[name]
        return self.currentValues

    def getLastFiveRecorded(self) :
        return self.lastFiveRecorded

    def setFileName(self, newFileName) :
        self.fileName = newFileName

    def updateLastFiveRecorded(self, recordEntry) :
        i = len(self.lastFiveRecorded) - 1
        while i > 0 :
            self.lastFiveRecorded[i] = self.lastFiveRecorded[i - 1]
            i -= 1
        self.lastFiveRecorded[0] = recordEntry

def main() :
    import time
    spudScale = SpudScale()
    time.sleep(10)
    for i in range(0,100) :
        print(spudScale.getCurrentValues())
        spudScale.record('Line' + str(i))
        if (i%5 == 0) :
            print(spudScale.getLastFiveRecorded())
        time.sleep(.5)
    return 0

if __name__ == '__main__':
    main()

"""
getScaleNames()
getCurrentValues()
recordValues(plotNumber)
getLastFive()
Is there anything else it would need?
.setFileName(...)
"""
<|endoftext|>