<commit_before>"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
     http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

from collections import defaultdict

from django.core.management.base import BaseCommand, CommandError
from optparse import make_option
from portal.models import Rna

import pymysql.cursors

def get_ensembl_connection():
    """
    Connect to the public Ensembl MySQL database.
    """
    connection = pymysql.connect(host='ensembldb.ensembl.org',
                                 user='anonymous',
                                 port=3306,
                                 cursorclass=pymysql.cursors.DictCursor)
    return connection

def get_ensembl_databases(cursor):
    """
    Get a list of all available databases.
    Return a list of the most recent core databases, for example:
        homo_sapiens_core_91_38
        mus_musculus_core_91_38
    """
    databases = defaultdict(list)
    cursor.execute("show databases")
    for result in cursor.fetchall():
        database = result['Database']
        if database.count('_') != 4 or 'mirror' in database:
            continue
        genus, species, database_type, ensembl_release, _ = database.split('_')
        ensembl_release = int(ensembl_release)
        if ensembl_release < 80:
            continue
        if database_type == 'core':
            organism = genus + ' ' + species
            databases[organism].append((ensembl_release, database))

    to_analyse = []
    # get the most recent database
    for organism, dbs in databases.iteritems():
        most_recent = max(dbs,key=lambda item:item[0])
        to_analyse.append(most_recent[1])
    return to_analyse

def get_ensembl_insdc_mapping(cursor, database):
    """
    Get a mapping between chromosome-level INSDC accessions and Ensembl names.
    Example:
        CM001500.1	10
        CM001492.1	2
    """
    mapping = []
    cursor.execute("USE %s" % database)
    sql = """
    SELECT synonym AS insdc, name AS ensembl
    FROM seq_region t1, seq_region_synonym t2, external_db t3
    WHERE
    t1.`seq_region_id` = t2.`seq_region_id`
    AND t2.external_db_id = t3.`external_db_id`
    AND t3.db_name = 'INSDC'
    AND t1.`coord_system_id` in (
        SELECT coord_system_id
        FROM coord_system
        WHERE name = 'chromosome'
        AND attrib = 'default_version'
    )
    """
    cursor.execute(sql)
    for result in cursor.fetchall():
        mapping.append({
            'insdc': result['insdc'],
            'ensembl': result['ensembl'],
        })
    return mapping

def store_mapping(mapping, ensembl_metadata):
    """
    """
    assembly_id = ensembl_metadata['assembly.default']
    for entry in mapping:
        print '"{insdc}","{ensembl}","{assembly_id}"'.format(insdc=entry['insdc'],
            ensembl=entry['ensembl'],
            assembly_id=assembly_id)

def get_ensembl_metadata(cursor, database):
    """
    Get metadata about genomic assemblies used in Ensembl.
    """
    cursor.execute("USE %s" % database)
    sql = """
    SELECT meta_key, meta_value
    FROM meta
    WHERE meta_key IN (
    	'assembly.accession',
    	'assembly.default',
    	'assembly.long_name',
    	'assembly.name',
    	'assembly.ucsc_alias',
    	'species.division',
    	'species.production_name',
    	'species.taxonomy_id',
    	'species.common_name',
    	'species.scientific_name'
    )
    """
    cursor.execute(sql)
    metadata = {}
    for result in cursor.fetchall():
        metadata[result['meta_key']] = result['meta_value']
    return metadata

def store_ensembl_metadata(metadata):
    """
    """
    line = "{assembly_id}\t{assembly_full_name}\t{GCA_accession}\t{assembly_ucsc}\t{common_name}\t{taxid}".format(
        assembly_id=metadata['assembly.default'],
        GCA_accession=metadata['assembly.accession'] if 'assembly.accession' in metadata else '',
        assembly_full_name=metadata['assembly.name'],
        assembly_ucsc=metadata['assembly.ucsc_alias'] if 'assembly.ucsc_alias' in metadata else '',
        division=metadata['species.division'],
        taxid=metadata['species.taxonomy_id'],
        common_name=metadata['species.common_name'],
    )
    print line

class Command(BaseCommand):
    """
    Usage:
    python manage.py update_ensembl_genome_mapping
    """
    def handle(self, *args, **options):
        """
        Main function, called by django.
        """
        connection = get_ensembl_connection()
        try:
            with connection.cursor() as cursor:
                databases = get_ensembl_databases(cursor)
                for database in databases:
                    ensembl_metadata = get_ensembl_metadata(cursor, database)
                    mapping = get_ensembl_insdc_mapping(cursor, database)
                    store_mapping(mapping, ensembl_metadata)
                    # store_ensembl_metadata(ensembl_metadata)
        finally:
            connection.close()
<commit_msg>Populate EnsemblAssembly and EnsemblInsdcMapping tables<commit_after>"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
     http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

from collections import defaultdict

from django.core.management.base import BaseCommand, CommandError
from optparse import make_option
from portal.models import EnsemblAssembly, EnsemblInsdcMapping

import pymysql.cursors

def get_ensembl_connection():
    """
    Connect to the public Ensembl MySQL database.
    """
    connection = pymysql.connect(host='ensembldb.ensembl.org',
                                 user='anonymous',
                                 port=3306,
                                 cursorclass=pymysql.cursors.DictCursor)
    return connection

def get_ensembl_databases(cursor):
    """
    Get a list of all available databases.
    Return a list of the most recent core databases, for example:
        homo_sapiens_core_91_38
        mus_musculus_core_91_38
    """
    databases = defaultdict(list)
    cursor.execute("show databases")
    for result in cursor.fetchall():
        database = result['Database']
        if database.count('_') != 4 or 'mirror' in database:
            continue
        genus, species, database_type, ensembl_release, _ = database.split('_')
        ensembl_release = int(ensembl_release)
        if ensembl_release < 80:
            continue
        if database_type == 'core':
            organism = genus + ' ' + species
            databases[organism].append((ensembl_release, database))

    to_analyse = []
    # get the most recent database
    for organism, dbs in databases.iteritems():
        most_recent = max(dbs,key=lambda item:item[0])
        to_analyse.append(most_recent[1])
    return to_analyse

def get_ensembl_insdc_mapping(cursor, database):
    """
    Get a mapping between chromosome-level INSDC accessions and Ensembl names.
    Example:
        CM001500.1	10
        CM001492.1	2
    """
    mapping = []
    cursor.execute("USE %s" % database)
    sql = """
    SELECT synonym AS insdc, name AS ensembl
    FROM seq_region t1, seq_region_synonym t2, external_db t3
    WHERE
    t1.`seq_region_id` = t2.`seq_region_id`
    AND t2.external_db_id = t3.`external_db_id`
    AND t3.db_name = 'INSDC'
    AND t1.`coord_system_id` in (
        SELECT coord_system_id
        FROM coord_system
        WHERE name = 'chromosome'
        AND attrib = 'default_version'
    )
    """
    cursor.execute(sql)
    for result in cursor.fetchall():
        mapping.append({
            'insdc': result['insdc'],
            'ensembl': result['ensembl'],
        })
    return mapping

def store_mapping(mapping, ensembl_metadata):
    """
    """
    assembly_id = ensembl_metadata['assembly.default']
    EnsemblInsdcMapping.objects.filter(assembly_id=assembly_id).delete()
    data = []
    for entry in mapping:
        print '"{insdc}","{ensembl}","{assembly_id}"'.format(insdc=entry['insdc'],
            ensembl=entry['ensembl'],
            assembly_id=assembly_id)
        data.append(EnsemblInsdcMapping(
            assembly_id_id=assembly_id,
            insdc=entry['insdc'],
            ensembl_name=entry['ensembl']
        ))
    EnsemblInsdcMapping.objects.bulk_create(data)

def get_ensembl_metadata(cursor, database):
    """
    Get metadata about genomic assemblies used in Ensembl.
    """
    cursor.execute("USE %s" % database)
    sql = """
    SELECT meta_key, meta_value
    FROM meta
    WHERE meta_key IN (
    	'assembly.accession',
    	'assembly.default',
    	'assembly.long_name',
    	'assembly.name',
    	'assembly.ucsc_alias',
    	'species.division',
    	'species.production_name',
    	'species.taxonomy_id',
    	'species.common_name',
    	'species.scientific_name'
    )
    """
    cursor.execute(sql)
    metadata = {}
    for result in cursor.fetchall():
        metadata[result['meta_key']] = result['meta_value']
    return metadata

def store_ensembl_metadata(metadata):
    """
    """
    EnsemblAssembly.objects.filter(taxid=metadata['species.taxonomy_id']).delete()
    assembly = EnsemblAssembly(
        assembly_id = metadata['assembly.default'],
        assembly_full_name = metadata['assembly.name'],
        gca_accession = metadata['assembly.accession'] if 'assembly.accession' in metadata else '',
        assembly_ucsc = metadata['assembly.ucsc_alias'] if 'assembly.ucsc_alias' in metadata else '',
        common_name = metadata['species.common_name'],
        taxid = metadata['species.taxonomy_id'],
    )
    assembly.save()

    line = "{assembly_id}\t{assembly_full_name}\t{GCA_accession}\t{assembly_ucsc}\t{common_name}\t{taxid}".format(
        assembly_id=metadata['assembly.default'],
        GCA_accession=metadata['assembly.accession'] if 'assembly.accession' in metadata else '',
        assembly_full_name=metadata['assembly.name'],
        assembly_ucsc=metadata['assembly.ucsc_alias'] if 'assembly.ucsc_alias' in metadata else '',
        division=metadata['species.division'],
        taxid=metadata['species.taxonomy_id'],
        common_name=metadata['species.common_name'],
    )
    print line

class Command(BaseCommand):
    """
    Usage:
    python manage.py update_ensembl_genome_mapping
    """
    def handle(self, *args, **options):
        """
        Main function, called by django.
        """
        connection = get_ensembl_connection()
        try:
            with connection.cursor() as cursor:
                databases = get_ensembl_databases(cursor)
                for database in databases:
                    ensembl_metadata = get_ensembl_metadata(cursor, database)
                    store_ensembl_metadata(ensembl_metadata)
                    mapping = get_ensembl_insdc_mapping(cursor, database)
                    store_mapping(mapping, ensembl_metadata)
        finally:
            connection.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import json
import os
import signal
import socket
import sys
import threading
from types import SimpleNamespace

from twisted.internet import task
from twisted.internet.defer import Deferred, DeferredQueue, inlineCallbacks
from twisted.internet.endpoints import UNIXClientEndpoint, clientFromString
from twisted.internet.protocol import Factory
from twisted.protocols.basic import LineReceiver

import varlink

with open(os.path.join(os.path.dirname(__file__), 'org.varlink.example.more.varlink')) as f:
    INTERFACE_org_varlink_example_more = varlink.Interface(f.read())

class VarlinkReceiver(LineReceiver):
    delimiter = b'\0'
    MAX_LENGTH = 8 * 1024 * 1024

class VarlinkClient(VarlinkReceiver, varlink.ClientInterfaceHandler):
    def _send_message(self, out):
        self.sendLine(out)

    def __init__(self):
        self.whenDisconnected = Deferred()
        super().__init__(INTERFACE_org_varlink_example_more, namespaced=True)
        self._lock = threading.Lock()
        self.queue = DeferredQueue()

    def lineReceived(self, incoming_message):
        self.queue.put(incoming_message)

    def connectionMade(self):
        pass

    def connectionLost(self, _):
        self.whenDisconnected.callback(None)

    @inlineCallbacks
    def close(self):
        yield self.transport.loseConnection()

    @inlineCallbacks
    def _next_message(self):
        self.resumeProducing()
        msg = yield self.queue.get()
        self.pauseProducing()
        return msg

    @inlineCallbacks
    def replyMore(self):
        message = yield self._next_message()
        if self._namespaced:
            message = json.loads(message, object_hook=lambda d: SimpleNamespace(**d))
            if hasattr(message, "error"):
                raise varlink.VarlinkError(message, self._namespaced)
            else:
                return message.parameters, hasattr(message, "continues") and message.continues
        else:
            message = json.loads(message)
            if 'error' in message:
                raise varlink.VarlinkError(message, self._namespaced)
            else:
                return message['parameters'], ('continues' in message) and message['continues']

    @inlineCallbacks
    def reply(self):
        (ret, more) = yield self.replyMore()
        return ret

    def _add_method(self, method):
        @inlineCallbacks
        def _wrapped(*args, **kwargs):
            parameters = self._interface.filter_params(method.in_type, args, kwargs)

            if "_more" in kwargs and kwargs.pop("_more"):
                out = {'method': self._interface.name + "." + method.name, 'more': True, 'parameters': parameters}
            else:
                out = {'method': self._interface.name + "." + method.name, 'parameters': parameters}
            yield self.sendLine(json.dumps(out, cls=varlink.VarlinkEncoder).encode('utf-8'))

        _wrapped.__name__ = method.name
        _wrapped.__doc__ = "Varlink call: " + method.signature
        setattr(self, method.name, _wrapped)

CHILD_PID = 0

def sigterm_handler(signum, _):
    global CHILD_PID
    if signum == signal.SIGTERM and CHILD_PID:
        try:
            os.kill(CHILD_PID, signal.SIGTERM)
        except OSError:
            pass
        os.waitpid(CHILD_PID, 0)
        CHILD_PID = 0

def varlink_to_twisted_endpoint(reactor, address):
    global CHILD_PID

    if address.startswith("unix:"):
        address = address[5:]
        address = address.replace(';mode=', ':mode=')
        address = address.replace('@', '\0', 1)
        # serverFromString() doesn't handle the zero byte
        return UNIXClientEndpoint(reactor, address)
    elif address.startswith("ip:"):
        address = address[3:]
        p = address.rfind(":")
        port = address[p + 1:]
        address = address[:p]
        address = "tcp:%s:interface=%s" % (port, address)
        return clientFromString(address)
    elif address.startswith("exec:"):
        executable = address[5:]
        s = socket.socket(socket.AF_UNIX)
        s.setblocking(0)
        s.bind("")
        s.listen()
        address = s.getsockname().decode('ascii')

        CHILD_PID = os.fork()
        if CHILD_PID == 0:
            # child
            n = s.fileno()
            if n == 3:
                # without dup() the socket is closed with the python destructor
                n = os.dup(3)
                del s
            else:
                try:
                    os.close(3)
                except OSError:
                    pass

            os.dup2(n, 3)
            address = address.replace('\0', '@', 1)
            address = "unix:%s;mode=0600" % address
            os.execlp(executable, executable, address)
            sys.exit(1)
        # parent
        s.close()

        signal.signal(signal.SIGALRM, sigterm_handler)
        return UNIXClientEndpoint(reactor, address)
    else:
        raise Exception("Invalid address '%s'" % address)

@inlineCallbacks
def main(reactor, address):
    factory = Factory.forProtocol(VarlinkClient)
    endpoint1 = varlink_to_twisted_endpoint(reactor, address)
    endpoint2 = varlink_to_twisted_endpoint(reactor, address)

    try:
        con1 = yield endpoint1.connect(factory)
        con2 = yield endpoint2.connect(factory)
        yield con1.TestMore(10, _more=True)
        while True:
            (m, more) = yield con1.replyMore()

            if hasattr(m.state, 'start'):
                if m.state.start:
                    print("--- Start ---", file=sys.stderr)

            if hasattr(m.state, 'progress'):
                print("Progress:", m.state.progress, file=sys.stderr)
                if m.state.progress > 50:
                    yield con2.Ping("Test")
                    ret = yield con2.reply()
                    print("Ping: ", ret.pong)

            if hasattr(m.state, 'end'):
                if m.state.end:
                    print("--- End ---", file=sys.stderr)

            if not more:
                break

        yield con1.close()
        yield con2.close()
        reactor.stop()
    except ConnectionError as e:
        import traceback
        print("ConnectionError", e, file=sys.stderr)
        traceback.print_exception(type(e), e, e.__traceback__)
        raise e
    except varlink.VarlinkError as e:
        print(e, file=sys.stderr)
        print(e.error(), file=sys.stderr)
        print(e.parameters(), file=sys.stderr)
        raise e

if __name__ == '__main__':
    listen_fd = None

    if len(sys.argv) == 2:
        connect_address = sys.argv[1]
    else:
        connect_address = 'exec:./server-twisted.py'

    print('Connecting to %s\n' % connect_address)

    task.react(main, [connect_address])

    if CHILD_PID != 0:
        try:
            os.kill(CHILD_PID, signal.SIGTERM)
        except OSError:
            pass
        os.waitpid(CHILD_PID, 0)
<commit_msg>only start server once for exec:<commit_after>#!/usr/bin/env python3

import json
import os
import signal
import socket
import sys
import threading
from types import SimpleNamespace

from twisted.internet import task
from twisted.internet.defer import Deferred, DeferredQueue, inlineCallbacks
from twisted.internet.endpoints import UNIXClientEndpoint, clientFromString
from twisted.internet.protocol import Factory
from twisted.protocols.basic import LineReceiver

import varlink

with open(os.path.join(os.path.dirname(__file__), 'org.varlink.example.more.varlink')) as f:
    INTERFACE_org_varlink_example_more = varlink.Interface(f.read())

class VarlinkReceiver(LineReceiver):
    delimiter = b'\0'
    MAX_LENGTH = 8 * 1024 * 1024

class VarlinkClient(VarlinkReceiver, varlink.ClientInterfaceHandler):
    def _send_message(self, out):
        self.sendLine(out)

    def __init__(self):
        self.whenDisconnected = Deferred()
        super().__init__(INTERFACE_org_varlink_example_more, namespaced=True)
        self._lock = threading.Lock()
        self.queue = DeferredQueue()

    def lineReceived(self, incoming_message):
        self.queue.put(incoming_message)

    def connectionMade(self):
        pass

    def connectionLost(self, _):
        self.whenDisconnected.callback(None)

    @inlineCallbacks
    def close(self):
        yield self.transport.loseConnection()

    @inlineCallbacks
    def _next_message(self):
        self.resumeProducing()
        msg = yield self.queue.get()
        self.pauseProducing()
        return msg

    @inlineCallbacks
    def replyMore(self):
        message = yield self._next_message()
        if self._namespaced:
            message = json.loads(message, object_hook=lambda d: SimpleNamespace(**d))
            if hasattr(message, "error"):
                raise varlink.VarlinkError(message, self._namespaced)
            else:
                return message.parameters, hasattr(message, "continues") and message.continues
        else:
            message = json.loads(message)
            if 'error' in message:
                raise varlink.VarlinkError(message, self._namespaced)
            else:
                return message['parameters'], ('continues' in message) and message['continues']

    @inlineCallbacks
    def reply(self):
        (ret, more) = yield self.replyMore()
        return ret

    def _add_method(self, method):
        @inlineCallbacks
        def _wrapped(*args, **kwargs):
            parameters = self._interface.filter_params(method.in_type, args, kwargs)

            if "_more" in kwargs and kwargs.pop("_more"):
                out = {'method': self._interface.name + "." + method.name, 'more': True, 'parameters': parameters}
            else:
                out = {'method': self._interface.name + "." + method.name, 'parameters': parameters}
            yield self.sendLine(json.dumps(out, cls=varlink.VarlinkEncoder).encode('utf-8'))

        _wrapped.__name__ = method.name
        _wrapped.__doc__ = "Varlink call: " + method.signature
        setattr(self, method.name, _wrapped)

CHILD_PID = 0

def sigterm_handler(signum, _):
    global CHILD_PID
    if signum == signal.SIGTERM and CHILD_PID:
        try:
            os.kill(CHILD_PID, signal.SIGTERM)
        except OSError:
            pass
        os.waitpid(CHILD_PID, 0)
        CHILD_PID = 0

def varlink_filter_exec_address(address):
    if address.startswith("exec:"):
        executable = address[5:]
        s = socket.socket(socket.AF_UNIX)
        s.setblocking(0)
        s.bind("")
        s.listen()
        address = s.getsockname().decode('ascii')

        CHILD_PID = os.fork()
        if CHILD_PID == 0:
            # child
            n = s.fileno()
            if n == 3:
                # without dup() the socket is closed with the python destructor
                n = os.dup(3)
                del s
            else:
                try:
                    os.close(3)
                except OSError:
                    pass

            os.dup2(n, 3)
            address = address.replace('\0', '@', 1)
            address = "unix:%s;mode=0600" % address
            os.execlp(executable, executable, address)
            sys.exit(1)
        # parent
        s.close()
        signal.signal(signal.SIGALRM, sigterm_handler)
        address = "unix:%s" % address
        address = address.replace('\0', '@', 1)

    return address

def varlink_to_twisted_endpoint(reactor, address):
    global CHILD_PID

    if address.startswith("unix:"):
        address = address[5:]
        address = address.replace(';mode=', ':mode=')
        address = address.replace('@', '\0', 1)
        # serverFromString() doesn't handle the zero byte
        return UNIXClientEndpoint(reactor, address)
    elif address.startswith("ip:"):
        address = address[3:]
        p = address.rfind(":")
        port = address[p + 1:]
        address = address[:p]
        address = "tcp:%s:interface=%s" % (port, address)
        return clientFromString(address)
    elif address.startswith("exec:"):
        executable = address[5:]
        s = socket.socket(socket.AF_UNIX)
        s.setblocking(0)
        s.bind("")
        s.listen()
        address = s.getsockname().decode('ascii')

        CHILD_PID = os.fork()
        if CHILD_PID == 0:
            # child
            n = s.fileno()
            if n == 3:
                # without dup() the socket is closed with the python destructor
                n = os.dup(3)
                del s
            else:
                try:
                    os.close(3)
                except OSError:
                    pass

            os.dup2(n, 3)
            address = address.replace('\0', '@', 1)
            address = "unix:%s;mode=0600" % address
            os.execlp(executable, executable, address)
            sys.exit(1)
        # parent
        s.close()

        signal.signal(signal.SIGALRM, sigterm_handler)
        return UNIXClientEndpoint(reactor, address)
    else:
        raise Exception("Invalid address '%s'" % address)

@inlineCallbacks
def main(reactor, address):
    factory = Factory.forProtocol(VarlinkClient)
    endpoint1 = varlink_to_twisted_endpoint(reactor, address)
    endpoint2 = varlink_to_twisted_endpoint(reactor, address)

    try:
        con1 = yield endpoint1.connect(factory)
        con2 = yield endpoint2.connect(factory)
        yield con1.TestMore(10, _more=True)
        while True:
            (m, more) = yield con1.replyMore()

            if hasattr(m.state, 'start'):
                if m.state.start:
                    print("--- Start ---", file=sys.stderr)

            if hasattr(m.state, 'progress'):
                print("Progress:", m.state.progress, file=sys.stderr)
                if m.state.progress > 50:
                    yield con2.Ping("Test")
                    ret = yield con2.reply()
                    print("Ping: ", ret.pong)

            if hasattr(m.state, 'end'):
                if m.state.end:
                    print("--- End ---", file=sys.stderr)

            if not more:
                break

        yield con1.close()
        yield con2.close()
        reactor.stop()
    except ConnectionError as e:
        import traceback
        print("ConnectionError", e, file=sys.stderr)
        traceback.print_exception(type(e), e, e.__traceback__)
        raise e
    except varlink.VarlinkError as e:
        print(e, file=sys.stderr)
        print(e.error(), file=sys.stderr)
        print(e.parameters(), file=sys.stderr)
        raise e

if __name__ == '__main__':
    listen_fd = None

    if len(sys.argv) == 2:
        connect_address = sys.argv[1]
    else:
        connect_address = 'exec:./server-twisted.py'

    print('Connecting to %s\n' % connect_address)
    connect_address = varlink_filter_exec_address(connect_address)

    task.react(main, [connect_address])

    if CHILD_PID != 0:
        try:
            os.kill(CHILD_PID, signal.SIGTERM)
        except OSError:
            pass
        os.waitpid(CHILD_PID, 0)
<|endoftext|>


<commit_before>import sys
import urllib2
import json
from generateURL import RequestWithMethod

# This is an example of deleting a database

# The URL where server listens
apiURL = 'http://localhost:5000/'

# The ID of the database
db_id = 'metadata'

req = RequestWithMethod(apiURL + 'evarilos/raw_data/v1.0/database/' + db_id, 'DELETE', headers={"Content-Type": "application/json"})
resp = urllib2.urlopen(req)
response = json.loads(resp.read())
print response	<commit_msg>message_types<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""deleteDatabase.py: Delete a database in the R2DM service."""

__author__ = "Filip Lemic"
__copyright__ = "Copyright 2015, EVARILOS Project"

__version__ = "1.0.0"
__maintainer__ = "Filip Lemic"
__email__ = "lemic@tkn.tu-berlin.de"
__status__ = "Development"

import sys
import urllib2
import json
from generateURL import RequestWithMethod

# This is an example of deleting a database

# The URL where server listens
apiURL = 'http://localhost:5000/'

# The ID of the database
db_id = 'test_db'

req = RequestWithMethod(apiURL + 'evarilos/raw_data/v1.0/database/' + db_id, 'DELETE', headers={"Content-Type": "application/json"})
resp = urllib2.urlopen(req)
response = json.loads(resp.read())
print response	<|endoftext|>


<commit_before>###############################################################################
#
# Copyright 2011 Pants (see AUTHORS.txt)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
###############################################################################

###############################################################################
# Imports
###############################################################################

import os
import socket

from pants.channel import Channel
from pants.engine import Engine

###############################################################################
# Logging
###############################################################################

import logging
log = logging.getLogger("pants")

###############################################################################
# Stream Class
###############################################################################

class Stream(Channel):
    """
    A streaming, connection-based :class:`~pants.channel.Channel`.

    ==========  ============
    Arguments   Description
    ==========  ============
    kwargs      Keyword arguments to be passed through to :class:`~pants.channel.Channel`
    ==========  ============
    """
    DATA_STRING = 0
    DATA_FILE = 1

    def __init__(self, **kwargs):
        if kwargs.setdefault("type", socket.SOCK_STREAM) != socket.SOCK_STREAM:
            raise TypeError("Cannot create a %s with a type other than SOCK_STREAM."
                    % self.__class__.__name__)

        Channel.__init__(self, **kwargs)

        # Socket
        self.remote_addr = (None, None)
        self.local_addr = (None, None)

        # I/O attributes
        self.read_delimiter = None
        self._recv_buffer = ""
        self._send_buffer = []

        # Channel state
        self.connected = False
        self.connecting = False
        self.listening = False

    ##### Control Methods #####################################################

    def connect(self, host, port):
        """
        Connect the channel to a remote socket.

        Returns the channel.

        ==========  ============
        Arguments   Description
        ==========  ============
        host        The remote host to connect to.
        port        The port to connect on.
        ==========  ============
        """
        if self.connected or self.listening or self.connecting:
            raise RuntimeError("connect() called on active %s #%d."
                    % (self.__class__.__name__, self.fileno))

        if self._socket is None:
            raise RuntimeError("connect() called on closed %s."
                    % self.__class__.__name__)

        self.connecting = True

        try:
            connected = self._socket_connect((host, port))
        except socket.error, err:
            self.close()
            raise

        if connected:
            self._handle_connect_event()

        return self

    def listen(self, port=8080, host='', backlog=1024):
        """
        Begin listening for connections made to the channel.

        Returns the channel.

        ==========  ============
        Arguments   Description
        ==========  ============
        port        *Optional.* The port to listen for connections on. By default, is 8080.
        host        *Optional.* The local host to bind to. By default, is ''.
        backlog     *Optional.* The size of the connection queue. By default, is 1024.
        ==========  ============
        """
        if self.connected or self.listening or self.connecting:
            raise RuntimeError("listen() called on active %s #%d."
                    % (self.__class__.__name__, self.fileno))

        if self._socket is None:
            raise RuntimeError("listen() called on closed %s."
                    % self.__class__.__name__)

        try:
            self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
        except AttributeError:
            pass

        try:
            self._socket_bind((host, port))
            self._socket_listen(backlog)
        except socket.error, err:
            self.close()
            raise

        self.listening = True
        self._update_addr()
        self._safely_call(self.on_listen)

        return self

    def close(self):
        """
        Close the channel.
        """
        if self._socket is None:
            return

        self.read_delimiter = None
        self._recv_buffer = ""
        self._send_buffer = []

        self.connected = False
        self.connecting = False
        self.listening = False

        self._update_addr()

        Channel.close(self)

    ##### I/O Methods #########################################################

    def write(self, data, buffer_data=False):
        """
        Write data to the channel.

        ============  ============
        Arguments     Description
        ============  ============
        data          A string of data to write to the channel.
        buffer_data   If True, the data will be buffered and written later.
        ============  ============
        """
        if self._socket is None:
            log.warning("Attempted to write to closed %s #%d." %
                    (self.__class__.__name__, self.fileno))
            return

        if not self.connected:
            log.warning("Attempted to write to disconnected %s #%d." %
                    (self.__class__.__name__, self.fileno))
            return

        if self._send_buffer and self._send_buffer[-1][0] == Stream.DATA_STRING:
            data_type, existing_data = self._send_buffer.pop(-1)
            data = existing_data + data

        self._send_buffer.append((Stream.DATA_STRING, data))

        if not buffer_data:
            self._process_send_buffer()
        else:
            self._wait_for_write_event = True

    def write_file(self, sfile, nbytes=0, offset=0, buffer_data=False):
        """
        Write a file to the channel.

        ============  ============
        Arguments     Description
        ============  ============
        sfile         A file object to write to the channel.
        nbytes        The number of bytes of the file to write. If 0, all bytes will be written.
        offset        The number of bytes to offset writing by.
        buffer_data   If True, the file will be buffered and written later.
        ============  ============
        """
        if self._socket is None:
            log.warning("Attempted to write file to closed %s #%d." %
                    (self.__class__.__name__, self.fileno))
            return

        if not self.connected:
            log.warning("Attempted to write file to disconnected %s #%d." %
                    (self.__class__.__name__, self.fileno))
            return

        self._send_buffer.append((Stream.DATA_FILE, (sfile, offset, nbytes)))

        if not buffer_data:
            self._process_send_buffer()
        else:
            self._wait_for_write_event = True

    ##### Internal Methods ####################################################

    def _update_addr(self):
        """
        Update the stream's attr:`~pants.stream.Stream.remote_addr` and
        attr:`~pants.stream.Stream.local_addr` attributes.
        """
        if self.connected:
            self.remote_addr = self._socket.getpeername()
            self.local_addr = self._socket.getsockname()
        elif self.listening:
            self.remote_addr = (None, None)
            self.local_addr = self._socket.getsockname()
        else:
            self.remote_addr = (None, None)
            self.local_addr = (None, None)

    ##### Internal Event Handler Methods ######################################

    def _handle_read_event(self):
        """
        Handle a read event raised on the channel.
        """
        if self.listening:
            self._handle_accept_event()
            return

        while True:
            try:
                data = self._socket_recv()
            except socket.error, err:
                log.exception("Exception raised by recv() on %s #%d." %
                        (self.__class__.__name__, self.fileno))
                # TODO Close this Stream here?
                self.close()
                return

            if data is None:
                self.close()
                return
            elif len(data) == 0:
                break
            else:
                self._recv_buffer += data

        self._process_recv_buffer()

    def _handle_write_event(self):
        """
        Handle a write event raised on the channel.
        """
        if self.listening:
            log.warning("Received write event for listening %s #%d." %
                    (self.__class__.__name__, self.fileno))
            return

        if not self.connected:
            self._handle_connect_event()

        if not self._send_buffer:
            return

        self._process_send_buffer()

    def _handle_accept_event(self):
        """
        Handle an accept event raised on the channel.
        """
        while True:
            try:
                sock, addr = self._socket_accept()
            except socket.error, err:
                log.exception("Exception raised by accept() on %s #%d." %
                        (self.__class__.__name__, self.fileno))
                try:
                    sock.close()
                except socket.error, err:
                    # TODO What do we do here?
                    pass
                # TODO Close this Stream here?
                return

            if sock is None:
                return

            self._safely_call(self.on_accept, sock, addr)

    def _handle_connect_event(self):
        """
        Handle a connect event raised on the channel.
        """
        self.connecting = False
        self._update_addr()
        err, errstr = self._get_socket_error()
        if err == 0:
            self.connected = True
            self._safely_call(self.on_connect)
        else:
            self._safely_call(self.on_connect_error, (err, errstr))

    ##### Internal Processing Methods #########################################

    def _process_recv_buffer(self):
        """
        Process the :attr:`~pants.stream.Stream._recv_buffer`, passing
        chunks of data to :meth:`~pants.stream.Stream.on_read`.
        """
        while self._recv_buffer:
            delimiter = self.read_delimiter

            if delimiter is None:
                data = self._recv_buffer
                self._recv_buffer = ""
                self._safely_call(self.on_read, data)

            elif isinstance(delimiter, (int, long)):
                if len(self._recv_buffer) < delimiter:
                    break
                data = self._recv_buffer[:delimiter]
                self._recv_buffer = self._recv_buffer[delimiter:]
                self._safely_call(self.on_read, data)

            elif isinstance(delimiter, basestring):
                mark = self._recv_buffer.find(delimiter)
                if mark == -1:
                    break
                data = self._recv_buffer[:mark]
                self._recv_buffer = self._recv_buffer[mark+len(delimiter):]
                self._safely_call(self.on_read, data)

            else:
                log.warning("Invalid read_delimiter on %s #%d." %
                        (self.__class__.__name__, self.fileno))
                break

            if self._socket is None or not self.connected:
                break

    def _process_send_buffer(self):
        """
        Process the :attr:`~pants.stream.Stream._send_buffer`, passing
        outgoing data to :meth:`~pants.channel.Channel._socket_send` or
        :meth:`~pants.channel.Channel._socket_sendfile` and calling
        :meth:`~pants.stream.Stream.on_write` when sending has finished.
        """
        while self._send_buffer:
            data_type, data = self._send_buffer.pop(0)

            if data_type == Stream.DATA_STRING:
                bytes_sent = self._process_send_data_string(data)
            elif data_type == Stream.DATA_FILE:
                bytes_sent = self._process_send_data_file(*data)

            if bytes_sent == 0:
                break

        if not self._send_buffer:
            self._safely_call(self.on_write)

    def _process_send_data_string(self, data):
        try:
            bytes_sent = self._socket_send(data)
        except socket.error, err:
            log.exception("Exception raised in send() on %s #%d." %
                    (self.__class__.__name__, self.fileno))
            self.close()
            return 0

        if len(data) > bytes_sent:
            self._send_buffer.insert(0, (Stream.DATA_STRING, data[bytes_sent:]))

        return bytes_sent

    def _process_send_data_file(self, sfile, offset, nbytes):
        try:
            bytes_sent = self._socket_sendfile(sfile, offset, nbytes)
        except socket.error, err:
            log.exception("Exception raised in sendfile() on %s #%d." %
                    (self.__class__.__name__, self.fileno))
            self.close()
            return 0

        offset += bytes_sent

        if nbytes > 0:
            if nbytes - bytes_sent > 0:
                nbytes -= bytes_sent
            else:
                # Reached the end of the segment.
                return bytes_sent

        if os.fstat(sfile.fileno()).st_size - offset <= 0:
            # Reached the end of the file.
            return bytes_sent

        self._send_buffer.insert(0, (Stream.DATA_FILE, (sfile, offset, nbytes)))

        return bytes_sent
<commit_msg>Cleanup pants.stream<commit_after>###############################################################################
#
# Copyright 2011 Pants (see AUTHORS.txt)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
###############################################################################

###############################################################################
# Imports
###############################################################################

import os
import socket

from pants.channel import Channel

###############################################################################
# Logging
###############################################################################

import logging
log = logging.getLogger("pants")

###############################################################################
# Stream Class
###############################################################################

class Stream(Channel):
    """
    A streaming, connection-based :class:`~pants.channel.Channel`.

    ==========  ============
    Arguments   Description
    ==========  ============
    kwargs      Keyword arguments to be passed through to :class:`~pants.channel.Channel`
    ==========  ============
    """
    DATA_STRING = 0
    DATA_FILE = 1

    def __init__(self, **kwargs):
        if kwargs.setdefault("type", socket.SOCK_STREAM) != socket.SOCK_STREAM:
            raise TypeError("Cannot create a %s with a type other than SOCK_STREAM."
                    % self.__class__.__name__)

        Channel.__init__(self, **kwargs)

        # Socket
        self.remote_addr = (None, None)
        self.local_addr = (None, None)

        # I/O attributes
        self.read_delimiter = None
        self._recv_buffer = ""
        self._send_buffer = []

        # Channel state
        self.connected = False
        self.connecting = False
        self.listening = False

    ##### Control Methods #####################################################

    def connect(self, host, port):
        """
        Connect the channel to a remote socket.

        Returns the channel.

        ==========  ============
        Arguments   Description
        ==========  ============
        host        The remote host to connect to.
        port        The port to connect on.
        ==========  ============
        """
        if self.connected or self.listening or self.connecting:
            raise RuntimeError("connect() called on active %s #%d."
                    % (self.__class__.__name__, self.fileno))

        if self._socket is None:
            raise RuntimeError("connect() called on closed %s."
                    % self.__class__.__name__)

        self.connecting = True

        try:
            connected = self._socket_connect((host, port))
        except socket.error:
            self.close()
            raise

        if connected:
            self._handle_connect_event()

        return self

    def listen(self, port=8080, host='', backlog=1024):
        """
        Begin listening for connections made to the channel.

        Returns the channel.

        ==========  ============
        Arguments   Description
        ==========  ============
        port        *Optional.* The port to listen for connections on. By default, is 8080.
        host        *Optional.* The local host to bind to. By default, is ''.
        backlog     *Optional.* The size of the connection queue. By default, is 1024.
        ==========  ============
        """
        if self.connected or self.listening or self.connecting:
            raise RuntimeError("listen() called on active %s #%d."
                    % (self.__class__.__name__, self.fileno))

        if self._socket is None:
            raise RuntimeError("listen() called on closed %s."
                    % self.__class__.__name__)

        try:
            self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
        except AttributeError:
            pass

        try:
            self._socket_bind((host, port))
            self._socket_listen(backlog)
        except socket.error, err:
            self.close()
            raise

        self.listening = True
        self._update_addr()
        self._safely_call(self.on_listen)

        return self

    def close(self):
        """
        Close the channel.
        """
        if self._socket is None:
            return

        self.read_delimiter = None
        self._recv_buffer = ""
        self._send_buffer = []

        self.connected = False
        self.connecting = False
        self.listening = False

        self._update_addr()

        Channel.close(self)

    ##### I/O Methods #########################################################

    def write(self, data, buffer_data=False):
        """
        Write data to the channel.

        ============  ============
        Arguments     Description
        ============  ============
        data          A string of data to write to the channel.
        buffer_data   If True, the data will be buffered and written later.
        ============  ============
        """
        if self._socket is None:
            log.warning("Attempted to write to closed %s #%d." %
                    (self.__class__.__name__, self.fileno))
            return

        if not self.connected:
            log.warning("Attempted to write to disconnected %s #%d." %
                    (self.__class__.__name__, self.fileno))
            return

        if self._send_buffer and self._send_buffer[-1][0] == Stream.DATA_STRING:
            data_type, existing_data = self._send_buffer.pop(-1)
            data = existing_data + data

        self._send_buffer.append((Stream.DATA_STRING, data))

        if not buffer_data:
            self._process_send_buffer()
        else:
            self._wait_for_write_event = True

    def write_file(self, sfile, nbytes=0, offset=0, buffer_data=False):
        """
        Write a file to the channel.

        ============  ============
        Arguments     Description
        ============  ============
        sfile         A file object to write to the channel.
        nbytes        The number of bytes of the file to write. If 0, all bytes will be written.
        offset        The number of bytes to offset writing by.
        buffer_data   If True, the file will be buffered and written later.
        ============  ============
        """
        if self._socket is None:
            log.warning("Attempted to write file to closed %s #%d." %
                    (self.__class__.__name__, self.fileno))
            return

        if not self.connected:
            log.warning("Attempted to write file to disconnected %s #%d." %
                    (self.__class__.__name__, self.fileno))
            return

        self._send_buffer.append((Stream.DATA_FILE, (sfile, offset, nbytes)))

        if not buffer_data:
            self._process_send_buffer()
        else:
            self._wait_for_write_event = True

    ##### Internal Methods ####################################################

    def _update_addr(self):
        """
        Update the stream's attr:`~pants.stream.Stream.remote_addr` and
        attr:`~pants.stream.Stream.local_addr` attributes.
        """
        if self.connected:
            self.remote_addr = self._socket.getpeername()
            self.local_addr = self._socket.getsockname()
        elif self.listening:
            self.remote_addr = (None, None)
            self.local_addr = self._socket.getsockname()
        else:
            self.remote_addr = (None, None)
            self.local_addr = (None, None)

    ##### Internal Event Handler Methods ######################################

    def _handle_read_event(self):
        """
        Handle a read event raised on the channel.
        """
        if self.listening:
            self._handle_accept_event()
            return

        while True:
            try:
                data = self._socket_recv()
            except socket.error:
                log.exception("Exception raised by recv() on %s #%d." %
                        (self.__class__.__name__, self.fileno))
                # TODO Close this Stream here?
                self.close()
                return

            if data is None:
                self.close()
                return
            elif len(data) == 0:
                break
            else:
                self._recv_buffer += data

        self._process_recv_buffer()

    def _handle_write_event(self):
        """
        Handle a write event raised on the channel.
        """
        if self.listening:
            log.warning("Received write event for listening %s #%d." %
                    (self.__class__.__name__, self.fileno))
            return

        if not self.connected:
            self._handle_connect_event()

        if not self._send_buffer:
            return

        self._process_send_buffer()

    def _handle_accept_event(self):
        """
        Handle an accept event raised on the channel.
        """
        while True:
            try:
                sock, addr = self._socket_accept()
            except socket.error:
                log.exception("Exception raised by accept() on %s #%d." %
                        (self.__class__.__name__, self.fileno))
                try:
                    sock.close()
                except socket.error:
                    # TODO What do we do here?
                    pass
                # TODO Close this Stream here?
                return

            if sock is None:
                return

            self._safely_call(self.on_accept, sock, addr)

    def _handle_connect_event(self):
        """
        Handle a connect event raised on the channel.
        """
        self.connecting = False
        self._update_addr()
        err, errstr = self._get_socket_error()
        if err == 0:
            self.connected = True
            self._safely_call(self.on_connect)
        else:
            self._safely_call(self.on_connect_error, (err, errstr))

    ##### Internal Processing Methods #########################################

    def _process_recv_buffer(self):
        """
        Process the :attr:`~pants.stream.Stream._recv_buffer`, passing
        chunks of data to :meth:`~pants.stream.Stream.on_read`.
        """
        while self._recv_buffer:
            delimiter = self.read_delimiter

            if delimiter is None:
                data = self._recv_buffer
                self._recv_buffer = ""
                self._safely_call(self.on_read, data)

            elif isinstance(delimiter, (int, long)):
                if len(self._recv_buffer) < delimiter:
                    break
                data = self._recv_buffer[:delimiter]
                self._recv_buffer = self._recv_buffer[delimiter:]
                self._safely_call(self.on_read, data)

            elif isinstance(delimiter, basestring):
                mark = self._recv_buffer.find(delimiter)
                if mark == -1:
                    break
                data = self._recv_buffer[:mark]
                self._recv_buffer = self._recv_buffer[mark + len(delimiter):]
                self._safely_call(self.on_read, data)

            else:
                log.warning("Invalid read_delimiter on %s #%d." %
                        (self.__class__.__name__, self.fileno))
                break

            if self._socket is None or not self.connected:
                break

    def _process_send_buffer(self):
        """
        Process the :attr:`~pants.stream.Stream._send_buffer`, passing
        outgoing data to :meth:`~pants.channel.Channel._socket_send` or
        :meth:`~pants.channel.Channel._socket_sendfile` and calling
        :meth:`~pants.stream.Stream.on_write` when sending has finished.
        """
        while self._send_buffer:
            data_type, data = self._send_buffer.pop(0)

            if data_type == Stream.DATA_STRING:
                bytes_sent = self._process_send_data_string(data)
            elif data_type == Stream.DATA_FILE:
                bytes_sent = self._process_send_data_file(*data)

            if bytes_sent == 0:
                break

        if not self._send_buffer:
            self._safely_call(self.on_write)

    def _process_send_data_string(self, data):
        try:
            bytes_sent = self._socket_send(data)
        except socket.error:
            log.exception("Exception raised in send() on %s #%d." %
                    (self.__class__.__name__, self.fileno))
            self.close()
            return 0

        if len(data) > bytes_sent:
            self._send_buffer.insert(0, (Stream.DATA_STRING, data[bytes_sent:]))

        return bytes_sent

    def _process_send_data_file(self, sfile, offset, nbytes):
        try:
            bytes_sent = self._socket_sendfile(sfile, offset, nbytes)
        except socket.error:
            log.exception("Exception raised in sendfile() on %s #%d." %
                    (self.__class__.__name__, self.fileno))
            self.close()
            return 0

        offset += bytes_sent

        if nbytes > 0:
            if nbytes - bytes_sent > 0:
                nbytes -= bytes_sent
            else:
                # Reached the end of the segment.
                return bytes_sent

        if os.fstat(sfile.fileno()).st_size - offset <= 0:
            # Reached the end of the file.
            return bytes_sent

        self._send_buffer.insert(0, (Stream.DATA_FILE, (sfile, offset, nbytes)))

        return bytes_sent
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import time
import logging

from datetime import datetime, timedelta

from avery.controller import Controller
from avery.worker import Worker, JobChannel  # noqa

def work_func(channel: JobChannel):
    print('========= GOT JOB =========')
    print('id: ', channel.job.id)
    print('locked_at: ', channel.job.locked_at)
    print('args: ', channel.job.args)
    print('===========================')
    channel.requeue_job(datetime.utcnow() + timedelta(seconds=3))

def setup_backend(db_uri):
    import pymongo
    client = pymongo.MongoClient(db_uri)
    client.drop_database(client.get_default_database().name)

if __name__ == '__main__':
    logging.basicConfig(format='%(asctime)s\t%(levelname)s:\t%(message)s', level=logging.INFO)

    db_uri = 'mongodb://localhost/avery-example'

    setup_backend(db_uri)

    controller = Controller(db_uri)

    worker = Worker('example-worker', ['example-tag'], work_func, controller)
    worker.start()

    job_id = controller.create_job_id()
    controller.create_job(job_id, 'example-tag', {'payload': 42})

    try:
        while worker.is_running:
            time.sleep(1)

    except KeyboardInterrupt:
        worker.request_stop()
        worker.join()
<commit_msg>Use system python<commit_after>#!/usr/bin/env python

import time
import logging

from datetime import datetime, timedelta

from avery.controller import Controller
from avery.worker import Worker, JobChannel  # noqa

def work_func(channel: JobChannel):
    print('========= GOT JOB =========')
    print('id: ', channel.job.id)
    print('locked_at: ', channel.job.locked_at)
    print('args: ', channel.job.args)
    print('===========================')
    channel.requeue_job(datetime.utcnow() + timedelta(seconds=3))

def setup_backend(db_uri):
    import pymongo
    client = pymongo.MongoClient(db_uri)
    client.drop_database(client.get_default_database().name)

if __name__ == '__main__':
    logging.basicConfig(format='%(asctime)s\t%(levelname)s:\t%(message)s', level=logging.INFO)

    db_uri = 'mongodb://localhost/avery-example'

    setup_backend(db_uri)

    controller = Controller(db_uri)

    worker = Worker('example-worker', ['example-tag'], work_func, controller)
    worker.start()

    job_id = controller.create_job_id()
    controller.create_job(job_id, 'example-tag', {'payload': 42})

    try:
        while worker.is_running:
            time.sleep(1)

    except KeyboardInterrupt:
        worker.request_stop()
        worker.join()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- encoding: utf-8 -*-

import pyparsing as pp
import re
import numbers

class semantic :
    def __init__(self, produce=False):
        # Common grammar
        self.sep = pp.Optional(pp.Word(',',exact=1))
        self.town = pp.Optional( pp.Word( pp.alphas ), default="")
        self.cp =  pp.Optional( pp.Word( pp.nums,exact=5) , default="")

        # If this class is used to analyze the data
        if produce==False :
            self.alphas_fr = pp.alphas+'éèëêàâûôÉÈËÊçîù0123456789'.decode('utf-8').encode('latin-1')
        else :
            # ... or to generate the data
            self.alphas_fr = pp.alphas+'éèëêàâûôÉÈËÊçîù0123456789'

    def shortWords(self,sentence) :
        """
        Delete "short words"
        """
        list_words = [ "de",
                        "d",
                        "l",
                        "la",
                        "le",
                        "les",
                        "du",
                        "des",
                        "un"]

        for word in list_words :
            sentence = re.sub(r'\b'+word+'\s+',' ',sentence)

        return sentence

    def abbreviation(self, sentence) :
        """
        Replace abbreviation
        """

        # Replace symbols
        for w in ['-',';',':','\'','.'] :
            sentence = sentence.replace(w,' ')

        # Lower letters
        sentence = sentence.lower()

        # Replace abbreviation
        dico = [ ['/'," sur "],
                [ "bld"," boulevard "],
                [ "bd", " boulevard "],
                [ "av"," avenue "],
                [ "ave"," avenue "],
                [ "fg"," faubourg "],
                [ "fb"," faubourg "],
                [ "fbg", " faubourg "],
                [ "pl", " place "],
                [ "sq", " square "],
                [ "imp", " impasse "],
                [ "ch", " chemin "],
                [ "che", " chemin "],
                [ "all", u' allée '],
                [ "rt", " route "],
                [ "rte", " route "],
                [ "doc", " docteur "],
                [ "dr"," docteur "],
                [ "st", " saint "],
                [ "ste", " sainte "],
                [ "lt", " lieutenant "],
                [ "cpt", " capitaine "],
                [ "cdt", " commandant "],
                [ "gal", u' général '],
                [ "pdt", u' président '],
                [ "mgr", " monseigneur "],
                [ "me", u' maître ' ]
            ]

        for d in dico :
            sentence = re.sub(r'\b'+d[0]+'\s+', d[1],sentence)

        return sentence

    def analyze( self, sentence):
        """
        This module analyze the sentence.
        It returns a dictionnary matching a pattern. (or default)
        """

        sentence = self.abbreviation( sentence )
        sentence = self.shortWords( sentence )

        # Tests
        #-------

        resStation = self.parserStation( sentence )
        if resStation["name"] != "" :
            # It's a station

            # Convert the numbers
            resStation["name"] =self.addressToLetters( resStation["name"] )

            return resStation

        resaddress = self.parserAddress( sentence )
        if resaddress["name"] != "" :
            # It's an  address

            # Convert the numbers
            resaddress["name"] = self.addressToLetters( resaddress["name"] )

            return resaddress

        # Default pattern/type
        resDefault = self.parserDefault(sentence)
        # Convert the numbers
        resDefault["name"] = self.addressToLetters( resDefault["name"] )

        return resDefault

    def addressToLetters(self, address) :
        """
        Convert the numbers contained in an address to words
        """
        n = numbers.numbers()
        for i in range(0,len(address)) :
            if address[i].isdigit()==True :
                address[i] = n.convNumber2letter(int(address[i])).lower()
        return address

    def parserDefault(self,sentence) :
        """
        Default parser.
        Slice the sentence into tokens.
        """
        # Initializing results
        result = {}

        # Grammar
        nom = pp.Group( pp.OneOrMore( pp.Word(self.alphas_fr) ) + pp.Suppress(self.sep))
        defaut = pp.Optional(nom, default="")  + self.cp + self.town

        # Slice the sentence
        jetons = defaut.parseString( sentence )
        result["name"] = jetons[0]
        result["cp"] = jetons[1]
        result["town"] = jetons[2]
        result["type"] = "poi"

        return result

    def parserAddress(self, sentence) :
        """
        Recognizing components of an address
        """
        ways = [
            "place",
            "square",
            "carrefour",
            "cours",
            "mail",
            "allée",
            "promenade",
            "chemin",
            "voie",
            "rue",
            "impasse",
            "ruelle",
            "passage",
            "avenue",
            "boulevard"
            ]

        # Initializing results
        result = {}

        # Grammar of an address
        name = pp.Group( pp.OneOrMore( pp.Word(self.alphas_fr) ) + pp.Suppress(self.sep))
        way = pp.oneOf( ways )
        number =  pp.Group(pp.Word( pp.nums))
        number2 =  pp.Group(pp.Word( pp.nums) + pp.Optional( pp.Word(pp.alphas)) )

        # Using the grammar
        address1 =pp.Suppress(self.sep) +\
            way + pp.Optional(name, default="") + self.cp + self.town
        address = pp.Optional( number, default="##" ) + address1
        address2 = pp.Optional( number2, default="##" ) + address1

        try:
            # Detect if sentence is matching address pattern
            #  (only numbered adress. Ex = 1 )
            tokens = address.parseString( sentence )
            result["number"] = tokens[0]
            result["way"] = tokens[1]
            result["name"] = tokens[2]
            result["cp"] = tokens[3]
            result["town"] = tokens[4]
            result["type"] ="address"
        except :
            try:
                # Detect if sentence is matching address2 pattern
                #  (number followed by a letter. Ex = 1B )
                tokens = address2.parseString( sentence )
                result["number"] = tokens[0]
                result["way"] = tokens[1]
                result["name"] = tokens[2]
                result["cp"] = tokens[3]
                result["town"] = tokens[4]
                result["type"] ="address"
            except :
                # Fill empty fields
                result["number"] = ""
                result["way"] = ""
                result["name"] = ""
                result["cp"] = ""
                result["town"] = ""
                result["type"] = ""
        return result

    def parserStation(self, sentence) :
        """
        Recognizing items of public transports
        """

        # Stop-words for stations
        stations = [
            u'arrêt',
            "arret",
            "gare"
            ]

        # Initializing results
        result = {}

        # Grammar of station
        station_type = pp.oneOf( stations )
        name = pp.Group( pp.OneOrMore( pp.Word( self.alphas_fr ) ) + pp.Suppress(self.sep))

        # Using grammar to slicing
        station = station_type + pp.Optional(name, default="")  + self.cp + self.town

        try:
            # Detect if sentence is matching station pattern
            tokens = station.parseString( sentence )
            test_station = True
            result["station"] = tokens[0]
            result["name"] = tokens[1]
            result["cp"] = tokens[2]
            result["town"] = tokens[3]
            result["type"] ="station"
        except :
            # Fill empty fields
            result["number"] = ""
            result["way"] = ""
            result["name"] = ""
            result["cp"] = ""
            result["town"] = ""
            result["type"] = ""

        return result

if __name__ == '__main__':
    s = semantic()

    print s.analyze( "r" )
    print s.analyze( "rue" )
    print s.analyze( "rue de" )
    print s.analyze( "rue de la " )
    print s.analyze( "rue de la gare" )
    print s.analyze( "4b, rue de la gare" )
    print s.analyze( "56 ter, rue de la gare" )
    print s.analyze( "56 ter, rue de la gare, TOURS" )
    print s.analyze( "56 ter, rue de la gare,37 TOURS" )
    print s.analyze( "57 ter rue de la gare 37000 TOURS" )
    print s.analyze( "Boulevard d'Italie, Paris" )
    print s.analyze( "rue du Dr. faton" )
    print s.analyze( "Av. du Gal De Gaulle" )
    print s.analyze( "gare de tours" )
    print s.analyze( "Rue Désiré Lecomte tours" )
    print s.analyze( "Rue de la Cave Lagas")
    print s.analyze( "Gare Montparnasse")
    print s.analyze( "Gare Montparnasse, Paris")
    print s.analyze( "clos" )
    print s.analyze( "allée hector" )
    print s.analyze( "Jules Verne" )
    print s.analyze( "Jules Verne, nantes" )
    print s.analyze( "Station de captage d'eau potable" )
    print s.analyze( "58 rue colbert" )
    print s.analyze( "58b rue colbert" )
    print s.analyze( "arrêt palais")
    print s.analyze( "la grande bruère")
    print s.analyze( "charles lindbergh")
    print s.analyze( "l'heure tranquille")
    print s.analyze( "rue du 11 novembre")
    print s.analyze( "les 2 lions")

<commit_msg>Prise en charge de letrres accentuée spécifiques. fixes #12<commit_after>#!/usr/bin/python
# -*- encoding: utf-8 -*-

import pyparsing as pp
import re
import numbers

class semantic :
    def __init__(self, produce=False):
        # Common grammar
        self.sep = pp.Optional(pp.Word(',',exact=1))
        self.town = pp.Optional( pp.Word( pp.alphas ), default="")
        self.cp =  pp.Optional( pp.Word( pp.nums,exact=5) , default="")

        # If this class is used to analyze the data
        if produce==False :
            self.alphas_fr = pp.alphas+'éèëêàâûôÉÈËÊçîù0123456789'.decode('utf-8').encode('latin-1')
        else :
            # ... or to generate the data
            self.alphas_fr = pp.alphas+'éèëêàâûôÉÈËÊçîù0123456789'

    def frenchLetters(self,sentence):
        """
        Convert specials french letters to standards letters
        """
        sentence = re.sub(ur'â','a',sentence)
        sentence = re.sub(ur'ê','e',sentence)
        sentence = re.sub(ur'É','e',sentence)
        sentence = re.sub(ur'î','i',sentence)
        sentence = re.sub(ur'Î','i',sentence)
        sentence = re.sub(ur'ù','u',sentence)
        return sentence

    def shortWords(self,sentence) :
        """
        Delete "short words"
        """
        list_words = [ "de",
                        "d",
                        "l",
                        "la",
                        "le",
                        "les",
                        "du",
                        "des",
                        "un"]

        for word in list_words :
            sentence = re.sub(r'\b'+word+'\s+',' ',sentence)

        return sentence

    def abbreviation(self, sentence) :
        """
        Replace abbreviation
        """

        # Replace symbols
        for w in ['-',';',':','\'','.'] :
            sentence = sentence.replace(w,' ')

        # Lower letters
        sentence = sentence.lower()

        # Replace abbreviation
        dico = [ ['/'," sur "],
                [ "bld"," boulevard "],
                [ "bd", " boulevard "],
                [ "av"," avenue "],
                [ "ave"," avenue "],
                [ "fg"," faubourg "],
                [ "fb"," faubourg "],
                [ "fbg", " faubourg "],
                [ "pl", " place "],
                [ "sq", " square "],
                [ "imp", " impasse "],
                [ "ch", " chemin "],
                [ "che", " chemin "],
                [ "all", u' allée '],
                [ "rt", " route "],
                [ "rte", " route "],
                [ "doc", " docteur "],
                [ "dr"," docteur "],
                [ "st", " saint "],
                [ "ste", " sainte "],
                [ "lt", " lieutenant "],
                [ "cpt", " capitaine "],
                [ "cdt", " commandant "],
                [ "gal", u' général '],
                [ "pdt", u' président '],
                [ "mgr", " monseigneur "],
                [ "me", u' maître ' ]
            ]

        for d in dico :
            sentence = re.sub(r'\b'+d[0]+'\s+', d[1],sentence)

        return sentence

    def analyze( self, sentence):
        """
        This module analyze the sentence.
        It returns a dictionnary matching a pattern. (or default)
        """

        sentence = self.abbreviation( sentence )
        sentence = self.frenchLetters( sentence )
        sentence = self.shortWords( sentence )

        # Tests
        #-------

        resStation = self.parserStation( sentence )
        if resStation["name"] != "" :
            # It's a station

            # Convert the numbers
            resStation["name"] =self.addressToLetters( resStation["name"] )

            return resStation

        resaddress = self.parserAddress( sentence )
        if resaddress["name"] != "" :
            # It's an  address

            # Convert the numbers
            resaddress["name"] = self.addressToLetters( resaddress["name"] )

            return resaddress

        # Default pattern/type
        resDefault = self.parserDefault(sentence)
        # Convert the numbers
        resDefault["name"] = self.addressToLetters( resDefault["name"] )

        return resDefault

    def addressToLetters(self, address) :
        """
        Convert the numbers contained in an address to words
        """
        n = numbers.numbers()
        for i in range(0,len(address)) :
            if address[i].isdigit()==True :
                address[i] = n.convNumber2letter(int(address[i])).lower()
        return address

    def parserDefault(self,sentence) :
        """
        Default parser.
        Slice the sentence into tokens.
        """
        # Initializing results
        result = {}

        # Grammar
        nom = pp.Group( pp.OneOrMore( pp.Word(self.alphas_fr) ) + pp.Suppress(self.sep))
        defaut = pp.Optional(nom, default="")  + self.cp + self.town

        # Slice the sentence
        jetons = defaut.parseString( sentence )
        result["name"] = jetons[0]
        result["cp"] = jetons[1]
        result["town"] = jetons[2]
        result["type"] = "poi"

        return result

    def parserAddress(self, sentence) :
        """
        Recognizing components of an address
        """
        ways = [
            "place",
            "square",
            "carrefour",
            "cours",
            "mail",
            "allée",
            "promenade",
            "chemin",
            "voie",
            "rue",
            "impasse",
            "ruelle",
            "passage",
            "avenue",
            "boulevard"
            ]

        # Initializing results
        result = {}

        # Grammar of an address
        name = pp.Group( pp.OneOrMore( pp.Word(self.alphas_fr) ) + pp.Suppress(self.sep))
        way = pp.oneOf( ways )
        number =  pp.Group(pp.Word( pp.nums))
        number2 =  pp.Group(pp.Word( pp.nums) + pp.Optional( pp.Word(pp.alphas)) )

        # Using the grammar
        address1 =pp.Suppress(self.sep) +\
            way + pp.Optional(name, default="") + self.cp + self.town
        address = pp.Optional( number, default="##" ) + address1
        address2 = pp.Optional( number2, default="##" ) + address1

        try:
            # Detect if sentence is matching address pattern
            #  (only numbered adress. Ex = 1 )
            tokens = address.parseString( sentence )
            result["number"] = tokens[0]
            result["way"] = tokens[1]
            result["name"] = tokens[2]
            result["cp"] = tokens[3]
            result["town"] = tokens[4]
            result["type"] ="address"
        except :
            try:
                # Detect if sentence is matching address2 pattern
                #  (number followed by a letter. Ex = 1B )
                tokens = address2.parseString( sentence )
                result["number"] = tokens[0]
                result["way"] = tokens[1]
                result["name"] = tokens[2]
                result["cp"] = tokens[3]
                result["town"] = tokens[4]
                result["type"] ="address"
            except :
                # Fill empty fields
                result["number"] = ""
                result["way"] = ""
                result["name"] = ""
                result["cp"] = ""
                result["town"] = ""
                result["type"] = ""
        return result

    def parserStation(self, sentence) :
        """
        Recognizing items of public transports
        """

        # Stop-words for stations
        stations = [
            u'arrêt',
            "arret",
            "gare"
            ]

        # Initializing results
        result = {}

        # Grammar of station
        station_type = pp.oneOf( stations )
        name = pp.Group( pp.OneOrMore( pp.Word( self.alphas_fr ) ) + pp.Suppress(self.sep))

        # Using grammar to slicing
        station = station_type + pp.Optional(name, default="")  + self.cp + self.town

        try:
            # Detect if sentence is matching station pattern
            tokens = station.parseString( sentence )
            test_station = True
            result["station"] = tokens[0]
            result["name"] = tokens[1]
            result["cp"] = tokens[2]
            result["town"] = tokens[3]
            result["type"] ="station"
        except :
            # Fill empty fields
            result["number"] = ""
            result["way"] = ""
            result["name"] = ""
            result["cp"] = ""
            result["town"] = ""
            result["type"] = ""

        return result

if __name__ == '__main__':
    s = semantic()

    print s.analyze( "r" )
    print s.analyze( "rue" )
    print s.analyze( "rue de" )
    print s.analyze( "rue de la " )
    print s.analyze( "rue de la gare" )
    print s.analyze( "4b, rue de la gare" )
    print s.analyze( "56 ter, rue de la gare" )
    print s.analyze( "56 ter, rue de la gare, TOURS" )
    print s.analyze( "56 ter, rue de la gare,37 TOURS" )
    print s.analyze( "57 ter rue de la gare 37000 TOURS" )
    print s.analyze( "Boulevard d'Italie, Paris" )
    print s.analyze( "rue du Dr. faton" )
    print s.analyze( "Av. du Gal De Gaulle" )
    print s.analyze( "gare de tours" )
    print s.analyze( "Rue Désiré Lecomte tours" )
    print s.analyze( "Rue de la Cave Lagas")
    print s.analyze( "Gare Montparnasse")
    print s.analyze( "Gare Montparnasse, Paris")
    print s.analyze( "clos" )
    print s.analyze( "allée hector" )
    print s.analyze( "Jules Verne" )
    print s.analyze( "Jules Verne, nantes" )
    print s.analyze( "Station de captage d'eau potable" )
    print s.analyze( "58 rue colbert" )
    print s.analyze( "58b rue colbert" )
    print s.analyze( "arrêt palais")
    print s.analyze( "la grande bruère")
    print s.analyze( "charles lindbergh")
    print s.analyze( "l'heure tranquille")
    print s.analyze( "rue du 11 novembre")
    print s.analyze( "les 2 lions")
    print s.analyze( u"île simon")
    print s.analyze( u"ile simon")

<|endoftext|>


<commit_before>"""
utils.py

Useful functions related to the parameters file, eg. create params file from excel table

Contributors: salvadordura@gmail.com
"""
import os, sys
from neuron import h

def getSecName(sec, dirCellSecNames = {}):
	if '>.' in sec.name():
		fullSecName = sec.name().split('>.')[1]
	elif '.' in sec.name():
		fullSecName = sec.name().split('.')[1]
	else:
		fullSecName = sec.name()
	if '[' in fullSecName:  # if section is array element
		secNameTemp = fullSecName.split('[')[0]
		secIndex = int(fullSecName.split('[')[1].split(']')[0])
		secName = secNameTemp+'_'+str(secIndex) #(secNameTemp,secIndex)
	else:
		secName = fullSecName
		secIndex = -1
	if secName in dirCellSecNames:  # get sec names from python
		secName = dirCellSecNames[secName]
	return secName

def importCellParams(fileName, labels, values, key = None):
	params = {}
	if fileName.endswith('.py'):
		try:
	 		filePath,fileNameOnly = os.path.split(fileName)  # split path from filename
	  		if filePath not in sys.path:  # add to path if not there (need to import module)
	 			sys.path.insert(0, filePath)
			moduleName = fileNameOnly.split('.py')[0]  # remove .py to obtain module name
			exec('import '+ moduleName + ' as tempModule') in locals() # import module dynamically
			modulePointer = tempModule
			paramLabels = getattr(modulePointer, labels) # tuple with labels
			paramValues = getattr(modulePointer, values)  # variable with paramValues
			if key:  # if paramValues = dict
				paramValues = paramValues[key]
			params = dict(zip(paramLabels, paramValues))
			sys.path.remove(filePath)
		except:
			print "Error loading cell parameter values from " + fileName
	else:
		print "Trying to import izhi params from a file without the .py extension"
	return params

def mechVarList():
    msname = h.ref('')
    varList = {}
    for i, mechtype in enumerate(['mechs','pointps']):
        mt = h.MechanismType(i)  # either distributed mechs (0) or point process (1)
        varList[mechtype] = {}
        for j in xrange(int(mt.count())):
            mt.select(j)
            mt.selected(msname)
            ms = h.MechanismStandard(msname[0], 1) # PARAMETER (modifiable)
            varList[mechtype][msname[0]] = []
            propName = h.ref('')
            for var in xrange(int(ms.count())):
                k = ms.name(propName, var)
                varList[mechtype][msname[0]].append(propName[0])
    return varList

def importCell(fileName, cellName, cellArgs = {}):
	''' Import cell from HOC template or python file into framework format (dict of sections, with geom, topol, mechs, syns)'''
	if fileName.endswith('.hoc'):
		h.load_file(fileName)
		cell = getattr(h, cellName)(**cellArgs)  # arguments correspond to zloc, type and id -- remove in future (not used internally)
		secList = list(cell.allsec())
		dirCell = dir(cell)
	elif fileName.endswith('.py'):
 		filePath,fileNameOnly = os.path.split(fileName)  # split path from filename
  		if filePath not in sys.path:  # add to path if not there (need to import module)
 			sys.path.insert(0, filePath)
		moduleName = fileNameOnly.split('.py')[0]  # remove .py to obtain module name
		exec('import ' + moduleName + ' as tempModule') in globals(), locals() # import module dynamically
		modulePointer = tempModule
		cell = getattr(modulePointer, cellName)(**cellArgs)  # create cell and pass type as argument
		dirCell = dir(cell)

		if 'all_sec' in dirCell:
			secList = cell.all_sec
		elif 'sec' in dirCell:
			secList = [cell.sec]
		elif 'allsec' in dir(h):
			secList = [sec for sec in h.allsec()]
		elif 'soma' in dirCell:
			secList = [cell.soma]
		else:
			secList = []
		sys.path.remove(filePath)
	else:
		print "File name should be either .hoc or .py file"
		return

	# create dict with hname of each element in dir(cell)
	dirCellHnames = {}
	for dirCellName in dirCell:
		try:
			dirCellHnames.update({cell.__dict__[dirCellName].hname(): dirCellName})
		except:
			pass
	# create dict with dir(cell) name corresponding to each hname
	dirCellSecNames = {}
	for sec in secList:
		dirCellSecNames.update({hname: name for hname,name in dirCellHnames.iteritems() if hname == sec.hname()})

	secDic = {}
	for sec in secList:
		# create new section dict with name of section
		secName = getSecName(sec, dirCellSecNames)

		if len(secList) == 1:
			secName = 'soma' # if just one section rename to 'soma'
		secDic[secName] = {'geom': {}, 'topol': {}, 'mechs': {}, 'syns': {}}  # create dictionary to store sec info

		# store geometry properties
		standardGeomParams = ['L', 'nseg', 'diam', 'Ra', 'cm']
		secDir = dir(sec)
		for geomParam in standardGeomParams:
			#if geomParam in secDir:
			try:
				secDic[secName]['geom'][geomParam] = sec.__getattribute__(geomParam)
			except:
				pass

		# store 3d geometry
		numPoints = int(h.n3d())
		if numPoints:
			points = []
			for ipoint in range(numPoints):
				x = h.x3d(ipoint)
				y = h.y3d(ipoint)
				z = h.z3d(ipoint)
				diam = h.diam3d(ipoint)
				points.append((x, y, z, diam))
			secDic[secName]['geom']['pt3d'] = points

		# store mechanisms
		varList = mechVarList()  # list of properties for all density mechanisms and point processes
		ignoreMechs = ['dist']  # dist only used during cell creation
		mechDic = {}
		for mech in dir(sec(0.5)):
			if h.ismembrane(mech) and mech not in ignoreMechs:  # check if membrane mechanism
				mechDic[mech] = {}  # create dic for mechanism properties
				varNames = [varName.replace('_'+mech, '') for varName in varList['mechs'][mech]]
				varVals = []
				for varName in varNames:
					try:
						varVals = [seg.__getattribute__(mech).__getattribute__(varName) for seg in sec]
						if len(set(varVals)) == 1:
							varVals = varVals[0]
						mechDic[mech][varName] = varVals
					except:
						pass
						#print 'Could not read variable %s from mechanism %s'%(varName,mech)

		secDic[secName]['mechs'] = mechDic

		# add synapses and point neurons
		# for now read fixed params, but need to find way to read only synapse params
		syns = {}
		pointps = {}
		for seg in sec:
			for ipoint,point in enumerate(seg.point_processes()):
				pptype = point.hname().split('[')[0]
				varNames = varList['pointps'][pptype]
				if any([s in pptype.lower() for s in ['syn', 'ampa', 'gaba', 'nmda', 'glu']]):
				#if 'syn' in pptype.lower(): # if syn in name of point process then assume synapse
					synName = pptype + '_' + str(len(syns))
					syns[synName] = {}
					syns[synName]['_type'] = pptype
					syns[synName]['_loc'] = seg.x
					for varName in varNames:
						try:
							syns[synName][varName] = point.__getattribute__(varName)
						except:
							print 'Could not read variable %s from synapse %s'%(varName,synName)

				else: # assume its a non-synapse point process
					pointpName = pptype + '_'+ str(len(pointps))
					pointps[pointpName] = {}
					pointps[pointpName]['_type'] = pptype
					pointps[pointpName]['_loc'] = seg.x
					for varName in varNames:
						try:
							pointps[pointpName][varName] = point.__getattribute__(varName)
							# special condition for Izhi model, to set vinit=vr
							if varName == 'vr': secDic[secName]['vinit'] = point.__getattribute__(varName)
						except:
							print 'Could not read %s variable from point process %s'%(varName,synName)

		if syns: secDic[secName]['syns'] = syns
		if pointps: secDic[secName]['pointps'] = pointps

		# store topology (keep at the end since h.SectionRef messes remaining loop)
		secRef = h.SectionRef(sec=sec)
		if secRef.has_parent():
			secDic[secName]['topol']['parentSec'] = getSecName(secRef.parent().sec, dirCellSecNames)
			secDic[secName]['topol']['parentX'] = h.parent_connection()
			secDic[secName]['topol']['childX'] = h.section_orientation()

	del(cell) # delete cell
	import gc; gc.collect()

	return secDic

def importConnFromExcel(fileName, sheetName):
	''' Import connectivity rules from Excel sheet'''
	import openpyxl as xl

	# set columns
	colPreTags = 0 # 'A'
	colPostTags = 1 # 'B'
	colConnFunc = 2 # 'C'
	colSyn = 3 # 'D'
	colProb = 5 # 'F'
	colWeight = 6 # 'G'
	colAnnot = 8 # 'I'

	outFileName = fileName[:-5]+'_'+sheetName+'.py' # set output file name

	connText = """## Generated using importConnFromExcel() function in params/utils.py \n\nnetParams['connParams'] = [] \n\n"""

	# open excel file and sheet
	wb = xl.load_workbook(fileName)
	sheet = wb.get_sheet_by_name(sheetName)
	numRows = sheet.get_highest_row()

	with open(outFileName, 'w') as f:
		f.write(connText)  # write starting text
		for row in range(1,numRows+1):
			if sheet.cell(row=row, column=colProb).value:  # if not empty row
				print 'Creating conn rule for row ' + str(row)
				# read row values
				pre = sheet.cell(row=row, column=colPreTags).value
				post = sheet.cell(row=row, column=colPostTags).value
				func = sheet.cell(row=row, column=colConnFunc).value
				syn = sheet.cell(row=row, column=colSyn).value
				prob = sheet.cell(row=row, column=colProb).value
				weight = sheet.cell(row=row, column=colWeight).value

				# write preTags
				line = "netParams['connParams'].append({'preTags': {"
				for i,cond in enumerate(pre.split(';')):  # split into different conditions
					if i>0: line = line + ", "
					cond2 = cond.split('=')  # split into key and value
					line = line + "'" + cond2[0].replace(' ','') + "': " + cond2[1].replace(' ','')   # generate line
				line = line + "}" # end of preTags

				# write postTags
				line = line + ",\n'postTags': {"
				for i,cond in enumerate(post.split(';')):  # split into different conditions
					if i>0: line = line + ", "
					cond2 = cond.split('=')  # split into key and value
					line = line + "'" + cond2[0].replace(' ','') + "': " + cond2[1].replace(' ','')   # generate line
				line = line + "}" # end of postTags
				line = line + ",\n'connFunc': '" + func + "'"  # write connFunc
				line = line + ",\n'synReceptor': '" + syn + "'"  # write synReceptor
				line = line + ",\n'probability': " + str(prob)  # write prob
				line = line + ",\n'weight': " + str(weight)  # write prob
				line = line + "})"  # add closing brackets
				line = line + '\n\n' # new line after each conn rule
				f.write(line)  # write to file

<commit_msg>removed importCell setting of vinit=vr, since already working via INITIAL of izhi mod file<commit_after>"""
utils.py

Useful functions related to the parameters file, eg. create params file from excel table

Contributors: salvadordura@gmail.com
"""
import os, sys
from neuron import h

def getSecName(sec, dirCellSecNames = {}):
	if '>.' in sec.name():
		fullSecName = sec.name().split('>.')[1]
	elif '.' in sec.name():
		fullSecName = sec.name().split('.')[1]
	else:
		fullSecName = sec.name()
	if '[' in fullSecName:  # if section is array element
		secNameTemp = fullSecName.split('[')[0]
		secIndex = int(fullSecName.split('[')[1].split(']')[0])
		secName = secNameTemp+'_'+str(secIndex) #(secNameTemp,secIndex)
	else:
		secName = fullSecName
		secIndex = -1
	if secName in dirCellSecNames:  # get sec names from python
		secName = dirCellSecNames[secName]
	return secName

def importCellParams(fileName, labels, values, key = None):
	params = {}
	if fileName.endswith('.py'):
		try:
	 		filePath,fileNameOnly = os.path.split(fileName)  # split path from filename
	  		if filePath not in sys.path:  # add to path if not there (need to import module)
	 			sys.path.insert(0, filePath)
			moduleName = fileNameOnly.split('.py')[0]  # remove .py to obtain module name
			exec('import '+ moduleName + ' as tempModule') in locals() # import module dynamically
			modulePointer = tempModule
			paramLabels = getattr(modulePointer, labels) # tuple with labels
			paramValues = getattr(modulePointer, values)  # variable with paramValues
			if key:  # if paramValues = dict
				paramValues = paramValues[key]
			params = dict(zip(paramLabels, paramValues))
			sys.path.remove(filePath)
		except:
			print "Error loading cell parameter values from " + fileName
	else:
		print "Trying to import izhi params from a file without the .py extension"
	return params

def mechVarList():
    msname = h.ref('')
    varList = {}
    for i, mechtype in enumerate(['mechs','pointps']):
        mt = h.MechanismType(i)  # either distributed mechs (0) or point process (1)
        varList[mechtype] = {}
        for j in xrange(int(mt.count())):
            mt.select(j)
            mt.selected(msname)
            ms = h.MechanismStandard(msname[0], 1) # PARAMETER (modifiable)
            varList[mechtype][msname[0]] = []
            propName = h.ref('')
            for var in xrange(int(ms.count())):
                k = ms.name(propName, var)
                varList[mechtype][msname[0]].append(propName[0])
    return varList

def importCell(fileName, cellName, cellArgs = {}):
	''' Import cell from HOC template or python file into framework format (dict of sections, with geom, topol, mechs, syns)'''
	if fileName.endswith('.hoc'):
		h.load_file(fileName)
		cell = getattr(h, cellName)(**cellArgs)  # arguments correspond to zloc, type and id -- remove in future (not used internally)
		secList = list(cell.allsec())
		dirCell = dir(cell)
	elif fileName.endswith('.py'):
 		filePath,fileNameOnly = os.path.split(fileName)  # split path from filename
  		if filePath not in sys.path:  # add to path if not there (need to import module)
 			sys.path.insert(0, filePath)
		moduleName = fileNameOnly.split('.py')[0]  # remove .py to obtain module name
		exec('import ' + moduleName + ' as tempModule') in globals(), locals() # import module dynamically
		modulePointer = tempModule
		cell = getattr(modulePointer, cellName)(**cellArgs)  # create cell and pass type as argument
		dirCell = dir(cell)

		if 'all_sec' in dirCell:
			secList = cell.all_sec
		elif 'sec' in dirCell:
			secList = [cell.sec]
		elif 'allsec' in dir(h):
			secList = [sec for sec in h.allsec()]
		elif 'soma' in dirCell:
			secList = [cell.soma]
		else:
			secList = []
		sys.path.remove(filePath)
	else:
		print "File name should be either .hoc or .py file"
		return

	# create dict with hname of each element in dir(cell)
	dirCellHnames = {}
	for dirCellName in dirCell:
		try:
			dirCellHnames.update({cell.__dict__[dirCellName].hname(): dirCellName})
		except:
			pass
	# create dict with dir(cell) name corresponding to each hname
	dirCellSecNames = {}
	for sec in secList:
		dirCellSecNames.update({hname: name for hname,name in dirCellHnames.iteritems() if hname == sec.hname()})

	secDic = {}
	for sec in secList:
		# create new section dict with name of section
		secName = getSecName(sec, dirCellSecNames)

		if len(secList) == 1:
			secName = 'soma' # if just one section rename to 'soma'
		secDic[secName] = {'geom': {}, 'topol': {}, 'mechs': {}, 'syns': {}}  # create dictionary to store sec info

		# store geometry properties
		standardGeomParams = ['L', 'nseg', 'diam', 'Ra', 'cm']
		secDir = dir(sec)
		for geomParam in standardGeomParams:
			#if geomParam in secDir:
			try:
				secDic[secName]['geom'][geomParam] = sec.__getattribute__(geomParam)
			except:
				pass

		# store 3d geometry
		numPoints = int(h.n3d())
		if numPoints:
			points = []
			for ipoint in range(numPoints):
				x = h.x3d(ipoint)
				y = h.y3d(ipoint)
				z = h.z3d(ipoint)
				diam = h.diam3d(ipoint)
				points.append((x, y, z, diam))
			secDic[secName]['geom']['pt3d'] = points

		# store mechanisms
		varList = mechVarList()  # list of properties for all density mechanisms and point processes
		ignoreMechs = ['dist']  # dist only used during cell creation
		mechDic = {}
		for mech in dir(sec(0.5)):
			if h.ismembrane(mech) and mech not in ignoreMechs:  # check if membrane mechanism
				mechDic[mech] = {}  # create dic for mechanism properties
				varNames = [varName.replace('_'+mech, '') for varName in varList['mechs'][mech]]
				varVals = []
				for varName in varNames:
					try:
						varVals = [seg.__getattribute__(mech).__getattribute__(varName) for seg in sec]
						if len(set(varVals)) == 1:
							varVals = varVals[0]
						mechDic[mech][varName] = varVals
					except:
						pass
						#print 'Could not read variable %s from mechanism %s'%(varName,mech)

		secDic[secName]['mechs'] = mechDic

		# add synapses and point neurons
		# for now read fixed params, but need to find way to read only synapse params
		syns = {}
		pointps = {}
		for seg in sec:
			for ipoint,point in enumerate(seg.point_processes()):
				pptype = point.hname().split('[')[0]
				varNames = varList['pointps'][pptype]
				if any([s in pptype.lower() for s in ['syn', 'ampa', 'gaba', 'nmda', 'glu']]):
				#if 'syn' in pptype.lower(): # if syn in name of point process then assume synapse
					synName = pptype + '_' + str(len(syns))
					syns[synName] = {}
					syns[synName]['_type'] = pptype
					syns[synName]['_loc'] = seg.x
					for varName in varNames:
						try:
							syns[synName][varName] = point.__getattribute__(varName)
						except:
							print 'Could not read variable %s from synapse %s'%(varName,synName)

				else: # assume its a non-synapse point process
					pointpName = pptype + '_'+ str(len(pointps))
					pointps[pointpName] = {}
					pointps[pointpName]['_type'] = pptype
					pointps[pointpName]['_loc'] = seg.x
					for varName in varNames:
						try:
							pointps[pointpName][varName] = point.__getattribute__(varName)
							# special condition for Izhi model, to set vinit=vr
							# if varName == 'vr': secDic[secName]['vinit'] = point.__getattribute__(varName)
						except:
							print 'Could not read %s variable from point process %s'%(varName,synName)

		if syns: secDic[secName]['syns'] = syns
		if pointps: secDic[secName]['pointps'] = pointps

		# store topology (keep at the end since h.SectionRef messes remaining loop)
		secRef = h.SectionRef(sec=sec)
		if secRef.has_parent():
			secDic[secName]['topol']['parentSec'] = getSecName(secRef.parent().sec, dirCellSecNames)
			secDic[secName]['topol']['parentX'] = h.parent_connection()
			secDic[secName]['topol']['childX'] = h.section_orientation()

	del(cell) # delete cell
	import gc; gc.collect()

	return secDic

def importConnFromExcel(fileName, sheetName):
	''' Import connectivity rules from Excel sheet'''
	import openpyxl as xl

	# set columns
	colPreTags = 0 # 'A'
	colPostTags = 1 # 'B'
	colConnFunc = 2 # 'C'
	colSyn = 3 # 'D'
	colProb = 5 # 'F'
	colWeight = 6 # 'G'
	colAnnot = 8 # 'I'

	outFileName = fileName[:-5]+'_'+sheetName+'.py' # set output file name

	connText = """## Generated using importConnFromExcel() function in params/utils.py \n\nnetParams['connParams'] = [] \n\n"""

	# open excel file and sheet
	wb = xl.load_workbook(fileName)
	sheet = wb.get_sheet_by_name(sheetName)
	numRows = sheet.get_highest_row()

	with open(outFileName, 'w') as f:
		f.write(connText)  # write starting text
		for row in range(1,numRows+1):
			if sheet.cell(row=row, column=colProb).value:  # if not empty row
				print 'Creating conn rule for row ' + str(row)
				# read row values
				pre = sheet.cell(row=row, column=colPreTags).value
				post = sheet.cell(row=row, column=colPostTags).value
				func = sheet.cell(row=row, column=colConnFunc).value
				syn = sheet.cell(row=row, column=colSyn).value
				prob = sheet.cell(row=row, column=colProb).value
				weight = sheet.cell(row=row, column=colWeight).value

				# write preTags
				line = "netParams['connParams'].append({'preTags': {"
				for i,cond in enumerate(pre.split(';')):  # split into different conditions
					if i>0: line = line + ", "
					cond2 = cond.split('=')  # split into key and value
					line = line + "'" + cond2[0].replace(' ','') + "': " + cond2[1].replace(' ','')   # generate line
				line = line + "}" # end of preTags

				# write postTags
				line = line + ",\n'postTags': {"
				for i,cond in enumerate(post.split(';')):  # split into different conditions
					if i>0: line = line + ", "
					cond2 = cond.split('=')  # split into key and value
					line = line + "'" + cond2[0].replace(' ','') + "': " + cond2[1].replace(' ','')   # generate line
				line = line + "}" # end of postTags
				line = line + ",\n'connFunc': '" + func + "'"  # write connFunc
				line = line + ",\n'synReceptor': '" + syn + "'"  # write synReceptor
				line = line + ",\n'probability': " + str(prob)  # write prob
				line = line + ",\n'weight': " + str(weight)  # write prob
				line = line + "})"  # add closing brackets
				line = line + '\n\n' # new line after each conn rule
				f.write(line)  # write to file

<|endoftext|>