<commit_before>##
# Copyright (c) 2008-2011 Sprymix Inc.
# All rights reserved.
#
# See LICENSE for details.
##

import datetime
import uuid
from functools import partial

import postgresql
from postgresql.python import socket as pg_socket
from postgresql.driver import pq3
from postgresql import types as pg_types
from postgresql.types.io import lib as pg_types_io_lib
from postgresql.string import quote_ident

from semantix.caos.objects import numeric, string
from semantix.caos.backends import pool as caos_pool
from semantix.caos import CaosError
from semantix.utils import datetime as sx_datetime

from semantix.caos.backends.pgsql.driver import io as pg_types_io

_GREEN_SOCKET = False
try:
    from semantix.spin.green import socket as green_socket
    _GREEN_SOCKET = True
except ImportError:
    pass

from .. import session as pg_caos_session

oid_to_type = {
    postgresql.types.UUIDOID: uuid.UUID,
    postgresql.types.NUMERICOID: numeric.Decimal,
    postgresql.types.TEXTOID: string.Str
}

def resolve(typid):
    return pg_types_io.resolve(typid)

class Array(pg_types.Array):
    def __sx_serialize__(self):
        return tuple(self)

class TypeIO(pq3.TypeIO):
    def __init__(self, database):
        super().__init__(database)
        self._pool = database._pool

    def identify(self, **identity_mappings):
        id = list(identity_mappings.items())
        ios = [pg_types_io.resolve(x[0]) for x in id]
        oids = list(self.database.sys.regtypes([x[1] for x in id]))

        update = [
            (oid, io if io.__class__ is tuple else io(oid, self))
            for oid, io in zip(oids, ios)
        ]
        self._cache.update(update)

    def resolve(self, typid, from_resolution_of=(), builtins=resolve, quote_ident=quote_ident):
        return super().resolve(typid, from_resolution_of, builtins, quote_ident)

    def get_session(self):
        if self._pool:
            return self._pool.get_holder(self.database)
        else:
            return None

    def array_from_parts(self, parts):
        return super().array_from_parts(parts, ArrayType=Array)

    def array_io_factory(self, pack_element, unpack_element, typoid, hasbin_input, hasbin_output,
                               array_unpack=pg_types_io_lib.array_unpack):
        pack_array, unpack_array, atype = super().array_io_factory(pack_element, unpack_element,
                                                                   typoid,
                                                                   hasbin_input, hasbin_output)

        session = self.get_session()
        if session is not None:
            backend = session.backend
            source = backend.source_name_from_relid(int(typoid))
            if source is not None and hasbin_output:
                # Override unpacking of arrays containing source node records to filter
                # out None values.  This is necessary due to the absense of support for FILTER
                # in aggregates in Postgres.  In absolute majority of calls to array_agg(<concept>),
                # None values in the resulting array are an unndesired side effect of an outer join.
                #
                def unpack_array(data, array_from_parts = self.array_from_parts):
                    flags, typoid, dims, lbs, elements = array_unpack(data)
                    elements = tuple(unpack_element(x) for x in elements if x is not None)
                    dims[0] = len(elements)
                    return array_from_parts((elements, dims, lbs))

        return (pack_array, unpack_array, atype)

    def RowTypeFactory(self, attribute_map={}, _Row=pg_types.Row.from_sequence,
                       composite_relid = None):
        session = self.get_session()
        if session is not None:
            backend = session.backend
            source = backend.source_name_from_relid(composite_relid)
            if source is not None:
                return partial(backend.entity_from_row, session, source, attribute_map)
        return partial(_Row, attribute_map)

class Connection(pq3.Connection, caos_pool.Connection):
    def __init__(self, connector, pool=None):
        caos_pool.Connection.__init__(self, pool)
        pq3.Connection.__init__(self, connector)

    def connect(self):
        super().connect()
        if self._pool:
            self._pool.backend.init_connection(self)

    def reset(self):
        if self.state in ('failed', 'negotiating', 'busy'):
            self.execute('ROLLBACK')
        self.execute("SET SESSION AUTHORIZATION DEFAULT;")
        self.execute("CLOSE ALL;")
        self.execute("RESET ALL;")
        self.execute("UNLISTEN *;")
        self.execute("SELECT pg_advisory_unlock_all();")
        self.execute("DISCARD PLANS;")
        self.execute("DISCARD TEMP;")

class SocketConnector:
    @staticmethod
    def msghook(msg):
        if msg.message == '=> is deprecated as an operator name':
            return True

    def create_socket_factory(self, **params):
        params['socket_extra'] = {'async': self.async}
        return SocketFactory(**params)

    def __init__(self, async=False):
        self.async = async

class IPConnector(pq3.IPConnector, SocketConnector):
    def __init__(self, host, port, ipv, async=False, **kw):
        pq3.IPConnector.__init__(self, host, port, ipv, **kw)
        SocketConnector.__init__(self, async=async)

class IP4(IPConnector, pq3.IP4):
    pass

class IP6(IPConnector, pq3.IP6):
    pass

class Host(SocketConnector, pq3.Host):
    def __init__(self, host=None, port=None, ipv=None, address_family=None, async=False, **kw):
        pq3.Host.__init__(self, host=host, port=port, ipv=ipv, address_family=address_family, **kw)
        SocketConnector.__init__(self, async=async)

class Unix(SocketConnector, pq3.Unix):
    def __init__(self, unix=None, async=False, **kw):
        pq3.Unix.__init__(self, unix=unix, **kw)
        SocketConnector.__init__(self, async=async)

class SocketFactory(pg_socket.SocketFactory):
    def __init__(self, socket_create, socket_connect, socket_secure=None, socket_extra=None):
        super().__init__(socket_create, socket_connect, socket_secure)
        self.async = socket_extra.get('async', False) if socket_extra else False

    def __call__(self, timeout = None):
        if self.async:
            if not _GREEN_SOCKET:
                raise CaosError('missing green socket implementation, '
                                'unable to start async session')

            s = green_socket.Socket(*self.socket_create)

            timeout = float(timeout) if timeout is not None else None
            if timeout != 0:
                s.settimeout(timeout)

            s.connect(self.socket_connect)
            s.settimeout(None)
            return s

        else:
            return super().__call__(timeout)

class Driver(pq3.Driver):
    def __init__(self):
        super().__init__(typio=TypeIO, connection=Connection)

    def ip4(self, **kw):
        return IP4(driver = self, **kw)

    def ip6(self, **kw):
        return IP6(driver = self, **kw)

    def host(self, **kw):
        return Host(driver = self, **kw)

    def unix(self, **kw):
        return Unix(driver = self, **kw)

driver = Driver()

def connector(iri, async=False):
    params = postgresql.iri.parse(iri)
    settings = params.setdefault('settings', {})
    settings['standard_conforming_strings'] = 'on'
    params['async'] = async
    return driver.fit(**params)
<commit_msg>caos.backends.pgsql: Make pgsql.Array behave like a Container<commit_after>##
# Copyright (c) 2008-2011 Sprymix Inc.
# All rights reserved.
#
# See LICENSE for details.
##

import collections
import datetime
import uuid
from functools import partial

import postgresql
from postgresql.python import socket as pg_socket
from postgresql.driver import pq3
from postgresql import types as pg_types
from postgresql.types.io import lib as pg_types_io_lib
from postgresql.string import quote_ident

from semantix.caos.objects import numeric, string
from semantix.caos.backends import pool as caos_pool
from semantix.caos import CaosError
from semantix.utils import datetime as sx_datetime

from semantix.caos.backends.pgsql.driver import io as pg_types_io

_GREEN_SOCKET = False
try:
    from semantix.spin.green import socket as green_socket
    _GREEN_SOCKET = True
except ImportError:
    pass

from .. import session as pg_caos_session

oid_to_type = {
    postgresql.types.UUIDOID: uuid.UUID,
    postgresql.types.NUMERICOID: numeric.Decimal,
    postgresql.types.TEXTOID: string.Str
}

def resolve(typid):
    return pg_types_io.resolve(typid)

class Array(pg_types.Array, collections.Container):
    def __sx_serialize__(self):
        return tuple(self)

    def __contains__(self, element):
        return element in set(self.elements())

    def __bool__(self):
        return bool(list(self))

class TypeIO(pq3.TypeIO):
    def __init__(self, database):
        super().__init__(database)
        self._pool = database._pool

    def identify(self, **identity_mappings):
        id = list(identity_mappings.items())
        ios = [pg_types_io.resolve(x[0]) for x in id]
        oids = list(self.database.sys.regtypes([x[1] for x in id]))

        update = [
            (oid, io if io.__class__ is tuple else io(oid, self))
            for oid, io in zip(oids, ios)
        ]
        self._cache.update(update)

    def resolve(self, typid, from_resolution_of=(), builtins=resolve, quote_ident=quote_ident):
        return super().resolve(typid, from_resolution_of, builtins, quote_ident)

    def get_session(self):
        if self._pool:
            return self._pool.get_holder(self.database)
        else:
            return None

    def array_from_parts(self, parts):
        return super().array_from_parts(parts, ArrayType=Array)

    def array_io_factory(self, pack_element, unpack_element, typoid, hasbin_input, hasbin_output,
                               array_unpack=pg_types_io_lib.array_unpack):
        pack_array, unpack_array, atype = super().array_io_factory(pack_element, unpack_element,
                                                                   typoid,
                                                                   hasbin_input, hasbin_output)

        session = self.get_session()
        if session is not None:
            backend = session.backend
            source = backend.source_name_from_relid(int(typoid))
            if source is not None and hasbin_output:
                # Override unpacking of arrays containing source node records to filter
                # out None values.  This is necessary due to the absense of support for FILTER
                # in aggregates in Postgres.  In absolute majority of calls to array_agg(<concept>),
                # None values in the resulting array are an unndesired side effect of an outer join.
                #
                def unpack_array(data, array_from_parts = self.array_from_parts):
                    flags, typoid, dims, lbs, elements = array_unpack(data)
                    elements = tuple(unpack_element(x) for x in elements if x is not None)
                    dims[0] = len(elements)
                    return array_from_parts((elements, dims, lbs))

        return (pack_array, unpack_array, atype)

    def RowTypeFactory(self, attribute_map={}, _Row=pg_types.Row.from_sequence,
                       composite_relid = None):
        session = self.get_session()
        if session is not None:
            backend = session.backend
            source = backend.source_name_from_relid(composite_relid)
            if source is not None:
                return partial(backend.entity_from_row, session, source, attribute_map)
        return partial(_Row, attribute_map)

class Connection(pq3.Connection, caos_pool.Connection):
    def __init__(self, connector, pool=None):
        caos_pool.Connection.__init__(self, pool)
        pq3.Connection.__init__(self, connector)

    def connect(self):
        super().connect()
        if self._pool:
            self._pool.backend.init_connection(self)

    def reset(self):
        if self.state in ('failed', 'negotiating', 'busy'):
            self.execute('ROLLBACK')
        self.execute("SET SESSION AUTHORIZATION DEFAULT;")
        self.execute("CLOSE ALL;")
        self.execute("RESET ALL;")
        self.execute("UNLISTEN *;")
        self.execute("SELECT pg_advisory_unlock_all();")
        self.execute("DISCARD PLANS;")
        self.execute("DISCARD TEMP;")

class SocketConnector:
    @staticmethod
    def msghook(msg):
        if msg.message == '=> is deprecated as an operator name':
            return True

    def create_socket_factory(self, **params):
        params['socket_extra'] = {'async': self.async}
        return SocketFactory(**params)

    def __init__(self, async=False):
        self.async = async

class IPConnector(pq3.IPConnector, SocketConnector):
    def __init__(self, host, port, ipv, async=False, **kw):
        pq3.IPConnector.__init__(self, host, port, ipv, **kw)
        SocketConnector.__init__(self, async=async)

class IP4(IPConnector, pq3.IP4):
    pass

class IP6(IPConnector, pq3.IP6):
    pass

class Host(SocketConnector, pq3.Host):
    def __init__(self, host=None, port=None, ipv=None, address_family=None, async=False, **kw):
        pq3.Host.__init__(self, host=host, port=port, ipv=ipv, address_family=address_family, **kw)
        SocketConnector.__init__(self, async=async)

class Unix(SocketConnector, pq3.Unix):
    def __init__(self, unix=None, async=False, **kw):
        pq3.Unix.__init__(self, unix=unix, **kw)
        SocketConnector.__init__(self, async=async)

class SocketFactory(pg_socket.SocketFactory):
    def __init__(self, socket_create, socket_connect, socket_secure=None, socket_extra=None):
        super().__init__(socket_create, socket_connect, socket_secure)
        self.async = socket_extra.get('async', False) if socket_extra else False

    def __call__(self, timeout = None):
        if self.async:
            if not _GREEN_SOCKET:
                raise CaosError('missing green socket implementation, '
                                'unable to start async session')

            s = green_socket.Socket(*self.socket_create)

            timeout = float(timeout) if timeout is not None else None
            if timeout != 0:
                s.settimeout(timeout)

            s.connect(self.socket_connect)
            s.settimeout(None)
            return s

        else:
            return super().__call__(timeout)

class Driver(pq3.Driver):
    def __init__(self):
        super().__init__(typio=TypeIO, connection=Connection)

    def ip4(self, **kw):
        return IP4(driver = self, **kw)

    def ip6(self, **kw):
        return IP6(driver = self, **kw)

    def host(self, **kw):
        return Host(driver = self, **kw)

    def unix(self, **kw):
        return Unix(driver = self, **kw)

driver = Driver()

def connector(iri, async=False):
    params = postgresql.iri.parse(iri)
    settings = params.setdefault('settings', {})
    settings['standard_conforming_strings'] = 'on'
    params['async'] = async
    return driver.fit(**params)
<|endoftext|>


<commit_before>"""Django settings for Drastic project.
"""
__copyright__ = "Copyright (C) 2016 University of Maryland"
__license__ = "GNU AFFERO GENERAL PUBLIC LICENSE, Version 3"

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '6-_8dk(b%2#=tk6swe2d(ejqy#xh-lj*c=e$1%sz3togm3270('

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
TEMPLATE_DEBUG = DEBUG
ALLOWED_HOSTS = []

ALLOWED_HOSTS = ["*"]

# Application definition

INSTALLED_APPS = (
    'flat',
    'django.contrib.sites',
    'django.contrib.staticfiles',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.humanize',
    'django_extensions',
    'rest_framework',
    'django_gravatar',

    'drastic_ui',
    'archive',
    'activity',
    'nodes',
    'users',
    'groups',
    #'s3',
    'cdmi',
    'admin'
)

DEFAULT_APP_CONFIG = 'drastic_ui.DrasticAppConfig'

SITE_ID = 1

FILE_UPLOAD_HANDLERS = ("archive.uploader.AgentUploader",)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    #'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'users.middleware.CassandraAuth',
    'cdmi.middleware.CDMIMiddleware'
)

ROOT_URLCONF = 'drastic_ui.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

CDMI_SERVER = {
    "endpoint": "http://127.0.0.1/api/cdmi",
    "username": "",
    "password": "",
}

ADMIN_SERVER = {
    "endpoint": "http://127.0.0.1/api/admin",
}

WSGI_APPLICATION = 'drastic_ui.wsgi.application'

DATABASES = {
#     'test': {
#         'ENGINE': 'django.db.backends.sqlite3',
#         'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
#     },
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    },
}

LOGIN_REDIRECT_URL = '/'
LOGIN_URL = '/users/login'
#INCLUDE_AUTH_URLS = True
#INCLUDE_REGISTER_URL = True

# LDAP server configuration.
AUTH_LDAP_SERVER_URI = os.getenv('AUTH_LDAP_SERVER_URI', None) # ldap://ldap.example.com
AUTH_LDAP_USER_DN_TEMPLATE = os.getenv('AUTH_LDAP_USER_DN_TEMPLATE', None) # "uid=%(user)s,ou=users,dc=example,dc=com"
# NOTE: AUTHENTICATION_BACKENDS NOT USED, see users/views.py

LANGUAGE_CODE = 'en-gb'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'static')

# Try and load a local, machine specific settings if it exists.
try:
    from local_settings import *
except:
    pass

##############################################################################
# Caching
##############################################################################
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
    }
}

COMPRESS_UPLOADS = True

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
        },
    },
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
        },
        'file': {
            'level': 'DEBUG',
            'class': 'logging.FileHandler',
            'filename': os.path.join(BASE_DIR, '../../log/debug.log'),
            'formatter': 'verbose',
        },
    },
    'loggers': {
        'drastic': {
            'handlers': ['console'],
            'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
        },
        'django': {
            'handlers': ['file'],
            'level': 'DEBUG',
        },
    },
}

REST_FRAMEWORK = {
    'DEFAULT_AUTHENTICATION_CLASSES': [
#        'rest_framework.authentication.SessionAuthentication',
        'rest_framework.authentication.BasicAuthentication'
    ],
#     'DEFAULT_PERMISSION_CLASSES': [
#         'rest_framework.permissions.AllowAny'
    #]
}
#
# REST_SESSION_LOGIN = False
<commit_msg>adjusted logging settings<commit_after>"""Django settings for Drastic project.
"""
__copyright__ = "Copyright (C) 2016 University of Maryland"
__license__ = "GNU AFFERO GENERAL PUBLIC LICENSE, Version 3"

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '6-_8dk(b%2#=tk6swe2d(ejqy#xh-lj*c=e$1%sz3togm3270('

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
TEMPLATE_DEBUG = DEBUG
ALLOWED_HOSTS = []

ALLOWED_HOSTS = ["*"]

# Application definition

INSTALLED_APPS = (
    'flat',
    'django.contrib.sites',
    'django.contrib.staticfiles',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.humanize',
    'django_extensions',
    'rest_framework',
    'django_gravatar',

    'drastic_ui',
    'archive',
    'activity',
    'nodes',
    'users',
    'groups',
    #'s3',
    'cdmi',
    'admin'
)

DEFAULT_APP_CONFIG = 'drastic_ui.DrasticAppConfig'

SITE_ID = 1

FILE_UPLOAD_HANDLERS = ("archive.uploader.AgentUploader",)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    #'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'users.middleware.CassandraAuth',
    'cdmi.middleware.CDMIMiddleware'
)

ROOT_URLCONF = 'drastic_ui.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

CDMI_SERVER = {
    "endpoint": "http://127.0.0.1/api/cdmi",
    "username": "",
    "password": "",
}

ADMIN_SERVER = {
    "endpoint": "http://127.0.0.1/api/admin",
}

WSGI_APPLICATION = 'drastic_ui.wsgi.application'

DATABASES = {
#     'test': {
#         'ENGINE': 'django.db.backends.sqlite3',
#         'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
#     },
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    },
}

LOGIN_REDIRECT_URL = '/'
LOGIN_URL = '/users/login'
#INCLUDE_AUTH_URLS = True
#INCLUDE_REGISTER_URL = True

# LDAP server configuration.
AUTH_LDAP_SERVER_URI = os.getenv('AUTH_LDAP_SERVER_URI', None) # ldap://ldap.example.com
AUTH_LDAP_USER_DN_TEMPLATE = os.getenv('AUTH_LDAP_USER_DN_TEMPLATE', None) # "uid=%(user)s,ou=users,dc=example,dc=com"
# NOTE: AUTHENTICATION_BACKENDS NOT USED, see users/views.py

LANGUAGE_CODE = 'en-gb'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'static')

# Try and load a local, machine specific settings if it exists.
try:
    from local_settings import *
except:
    pass

##############################################################################
# Caching
##############################################################################
CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
    }
}

COMPRESS_UPLOADS = True

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
        },
    },
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
        },
        'file': {
            'level': 'DEBUG',
            'class': 'logging.handlers.RotatingFileHandler',
            'maxBytes': 1024*1024*5,  # 5 MB
            'backupCount': 5,
            'filename': os.path.join(BASE_DIR, '../../log/debug.log'),
            'formatter': 'verbose',
        },
    },
    'loggers': {
        '': {
            'handlers': ['file'],
            'level': 'DEBUG',
            'propagate': True
        },
        'django': {
            'handlers': ['file'],
            'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
        },
        'cassandra': {
            'handlers': ['file'],
            'level': 'WARN',
        }
    },
}

REST_FRAMEWORK = {
    'DEFAULT_AUTHENTICATION_CLASSES': [
#        'rest_framework.authentication.SessionAuthentication',
        'rest_framework.authentication.BasicAuthentication'
    ],
#     'DEFAULT_PERMISSION_CLASSES': [
#         'rest_framework.permissions.AllowAny'
    #]
}
#
# REST_SESSION_LOGIN = False
<|endoftext|>


<commit_before>__version__ = '0.1.18'
__author__ = 'benzkji'
<commit_msg>dev again<commit_after>__version__ = '0.1.19DEV'
__author__ = 'benzkji'
<|endoftext|>


<commit_before>##
# Copyright (c) 2008-2012 Sprymix Inc.
# All rights reserved.
#
# See LICENSE for details.
##

import collections
import datetime
import uuid
from functools import partial

import postgresql
from postgresql.python import socket as pg_socket
from postgresql.python.functools import process_tuple
from postgresql.driver import pq3
from postgresql import types as pg_types
from postgresql.types.io import lib as pg_types_io_lib
from postgresql.string import quote_ident
from postgresql.protocol import element3 as element

from metamagic.caos.objects import numeric, string
from metamagic.caos.backends import pool as caos_pool
from metamagic.caos import CaosError
from metamagic.utils import datetime as sx_datetime
from metamagic.utils.datastructures import Void, xvalue

from metamagic.caos.backends.pgsql import exceptions as backend_exc
from metamagic.caos.backends.pgsql.driver import io as pg_types_io

_GREEN_SOCKET = False
try:
    from metamagic.spin.green import socket as green_socket
    _GREEN_SOCKET = True
except ImportError:
    pass

from .. import session as pg_caos_session

oid_to_type = {
    postgresql.types.UUIDOID: uuid.UUID,
    postgresql.types.NUMERICOID: numeric.Decimal,
    postgresql.types.TEXTOID: string.Str
}

def resolve(typid):
    return pg_types_io.resolve(typid)

class Array(pg_types.Array, collections.Container):
    def __mm_serialize__(self):
        return tuple(self)

    def __contains__(self, element):
        return element in set(self.elements())

    def __bool__(self):
        return bool(list(self))

JSON_OUTPUT_FORMAT = ('pgjson.caos', 1)
FREEFORM_RECORD_ID = '6e51108d-7440-47f7-8c65-dc4d43fd90d2'

class Json(bytes):
    def __new__(cls, value, *, format_info=JSON_OUTPUT_FORMAT):
        result = super().__new__(cls, value)
        result._fmt = format_info
        return result

    def __mm_json__(self):
        fmt = '["{}",{}]'.format(*self._fmt).encode('ascii')
        return b'{"$sxjson$":{"format":' + fmt  + b',"data":[' + self + b']}}'

class TypeIO(pq3.TypeIO):
    RECORD_OID = 2249

    def __init__(self, database):
        super().__init__(database)
        self._pool = database._pool

    def identify(self, **identity_mappings):
        id = list(identity_mappings.items())
        ios = [pg_types_io.resolve(x[0]) for x in id]
        oids = list(self.database.sys.regtypes([x[1] for x in id]))

        update = [
            (oid, io if io.__class__ is tuple else io(oid, self))
            for oid, io in zip(oids, ios)
        ]
        self._cache.update(update)

    def resolve(self, typid, from_resolution_of=(), builtins=resolve, quote_ident=quote_ident):
        if typid == pg_types.JSONOID:
            return self.json_io_factory()
        else:
            return super().resolve(typid, from_resolution_of, builtins, quote_ident)

    def get_session(self):
        if self._pool:
            return self._pool.get_holder(self.database)
        else:
            return None

    def array_from_parts(self, parts):
        return super().array_from_parts(parts, ArrayType=Array)

    def json_io_factory(self):
        def _unpack_json(data):
            return Json(data)
        return (None, _unpack_json, Json)

    def array_io_factory(self, pack_element, unpack_element, typoid, hasbin_input, hasbin_output,
                               array_unpack=pg_types_io_lib.array_unpack):
        pack_array, unpack_array, atype = super().array_io_factory(pack_element, unpack_element,
                                                                   typoid,
                                                                   hasbin_input, hasbin_output)

        if typoid == self.RECORD_OID:
            def unpack_array(data, array_from_parts = self.array_from_parts):
                # Override unpacking of arrays containing source node records to filter
                # out None values.  This is necessary due to the absense of support for FILTER
                # in aggregates in Postgres.  In absolute majority of calls to array_agg(<concept>),
                # None values in the resulting array are an unndesired side effect of an outer join.
                #
                flags, typoid, dims, lbs, elements = array_unpack(data)
                elements = tuple(unpack_element(x) for x in elements if x is not None)
                dims[0] = len(elements)
                return array_from_parts((elements, dims, lbs))

        return (pack_array, unpack_array, atype)

    def anon_record_io_factory(self):
        def raise_unpack_tuple_error(cause, procs, tup, itemnum):
            data = repr(tup[itemnum])
            if len(data) > 80:
                # Be sure not to fill screen with noise.
                data = data[:75] + ' ...'
            self.raise_client_error(element.ClientError((
                (b'C', '--cIO',),
                (b'S', 'ERROR',),
                (b'M', 'Could not unpack element {} from anonymous record'.format(itemnum)),
                (b'W', data,),
                (b'P', str(itemnum),)
            )), cause = cause)

        def _unpack_record(data, unpack = pg_types_io_lib.record_unpack,
                                 process_tuple = process_tuple,
                                 _Row=pg_types.Row.from_sequence):
            record = list(unpack(data))
            coloids = tuple(x[0] for x in record)

            record_info = None

            try:
                marker_oid = self.database._sx_known_record_marker_oid_
            except AttributeError:
                pass
            else:
                if record and record[0][0] == marker_oid:
                    session = self.get_session()
                    if session is not None:
                        recid = record[0][1].decode('ascii')
                        record_info = session.backend._get_record_info_by_id(recid)

            colio = map(self.resolve, coloids)
            column_unpack = tuple(c[1] or self.decode for c in colio)

            data = tuple(x[1] for x in record)
            data = process_tuple(column_unpack, data, raise_unpack_tuple_error)

            if record_info is not None:
                data = dict(zip(record_info.attribute_map, data[1:]))

                if record_info.is_xvalue:
                    assert data['value'] is not None
                    data = xvalue(data['value'], **data['attrs'])

                elif record_info.proto_class == 'metamagic.caos.proto.Concept':
                    data = session.backend.entity_from_row(session, record_info, data)

            return data

        return (None, _unpack_record)

    def RowTypeFactory(self, attribute_map={}, _Row=pg_types.Row.from_sequence,
                       composite_relid = None):
        session = self.get_session()
        if session is not None:
            backend = session.backend
            source = backend.source_name_from_relid(composite_relid)
            if source is not None:
                return partial(backend.entity_from_row_compat, session, source, attribute_map)
        return partial(_Row, attribute_map)

class Connection(pq3.Connection, caos_pool.Connection):
    def __init__(self, connector, pool=None):
        caos_pool.Connection.__init__(self, pool)
        pq3.Connection.__init__(self, connector)

        self._prepared_statements = {}

    def get_prepared_statement(self, query, raw=True):
        try:
            statement = self._prepared_statements[query, raw]
        except KeyError:
            try:
                if raw:
                    stmt_id = 'sx_{:x}'.format(hash(query)).replace('-', '_')
                    prefix = 'PREPARE {} AS '.format(stmt_id)
                    self.execute('{}{}'.format(prefix, query))
                    statement = self.statement_from_id(stmt_id)
                else:
                    prefix = ''
                    statement = self.prepare(query)

            except postgresql.exceptions.Error as e:
                e.__suppress_context__ = True
                raise backend_exc.QueryError(driver_err=e, query_text=query,
                                             query_offset=len(prefix)) from e

            except Exception as e:
                raise e from None

            else:
                self._prepared_statements[query, raw] = statement

        return statement

    def connect(self):
        super().connect()
        if self._pool:
            self._pool.backend.init_connection(self)

    def reset(self, hard=False):
        if self.state in ('failed', 'negotiating', 'busy'):
            self.execute('ROLLBACK')

        if hard:
            self.execute('DISCARD ALL')
            self._prepared_statements.clear()
        else:
            self.execute("SET SESSION AUTHORIZATION DEFAULT;")
            self.execute("RESET ALL;")
            self.execute("CLOSE ALL;")
            self.execute("UNLISTEN *;")
            self.execute("SELECT pg_advisory_unlock_all();")

    def close(self):
        self._prepared_statements.clear()
        super().close()

class SocketConnector:
    @staticmethod
    def msghook(msg):
        if msg.message == '=> is deprecated as an operator name':
            return True

    def create_socket_factory(self, **params):
        params['socket_extra'] = {'async': self.async}
        return SocketFactory(**params)

    def __init__(self, async=False):
        self.async = async

class IPConnector(pq3.IPConnector, SocketConnector):
    def __init__(self, host, port, ipv, async=False, **kw):
        pq3.IPConnector.__init__(self, host, port, ipv, **kw)
        SocketConnector.__init__(self, async=async)

class IP4(IPConnector, pq3.IP4):
    pass

class IP6(IPConnector, pq3.IP6):
    pass

class Host(SocketConnector, pq3.Host):
    def __init__(self, host=None, port=None, ipv=None, address_family=None, async=False, **kw):
        pq3.Host.__init__(self, host=host, port=port, ipv=ipv, address_family=address_family, **kw)
        SocketConnector.__init__(self, async=async)

class Unix(SocketConnector, pq3.Unix):
    def __init__(self, unix=None, async=False, **kw):
        pq3.Unix.__init__(self, unix=unix, **kw)
        SocketConnector.__init__(self, async=async)

class SocketFactory(pg_socket.SocketFactory):
    def __init__(self, socket_create, socket_connect, socket_secure=None, socket_extra=None):
        super().__init__(socket_create, socket_connect, socket_secure)
        self.async = socket_extra.get('async', False) if socket_extra else False

    def __call__(self, timeout = None):
        if self.async:
            if not _GREEN_SOCKET:
                raise CaosError('missing green socket implementation, '
                                'unable to start async session')

            s = green_socket.Socket(*self.socket_create)

            timeout = float(timeout) if timeout is not None else None
            if timeout != 0:
                s.settimeout(timeout)

            s.connect(self.socket_connect)
            s.settimeout(None)
            return s

        else:
            return super().__call__(timeout)

class Driver(pq3.Driver):
    def __init__(self):
        super().__init__(typio=TypeIO, connection=Connection)

    def ip4(self, **kw):
        return IP4(driver = self, **kw)

    def ip6(self, **kw):
        return IP6(driver = self, **kw)

    def host(self, **kw):
        return Host(driver = self, **kw)

    def unix(self, **kw):
        return Unix(driver = self, **kw)

driver = Driver()

def connector(iri, async=False):
    params = postgresql.iri.parse(iri)
    settings = params.setdefault('settings', {})
    settings['standard_conforming_strings'] = 'on'
    params['async'] = async
    return driver.fit(**params)
<commit_msg>caos.backends, spin.green: Split green pq connector code to spin.green<commit_after>##
# Copyright (c) 2008-2012 Sprymix Inc.
# All rights reserved.
#
# See LICENSE for details.
##

import collections
import datetime
import uuid
from functools import partial

import postgresql
from postgresql.python.functools import process_tuple
from postgresql.driver import pq3
from postgresql import types as pg_types
from postgresql.types.io import lib as pg_types_io_lib
from postgresql.string import quote_ident
from postgresql.protocol import element3 as element

from metamagic.caos.objects import numeric, string
from metamagic.caos.backends import pool as caos_pool
from metamagic.caos import CaosError
from metamagic.utils import datetime as sx_datetime
from metamagic.utils.datastructures import Void, xvalue

from metamagic.caos.backends.pgsql import exceptions as backend_exc
from metamagic.caos.backends.pgsql.driver import io as pg_types_io

from metamagic.spin.green import postgres as green_postgres

from .. import session as pg_caos_session

oid_to_type = {
    postgresql.types.UUIDOID: uuid.UUID,
    postgresql.types.NUMERICOID: numeric.Decimal,
    postgresql.types.TEXTOID: string.Str
}

def resolve(typid):
    return pg_types_io.resolve(typid)

class Array(pg_types.Array, collections.Container):
    def __mm_serialize__(self):
        return tuple(self)

    def __contains__(self, element):
        return element in set(self.elements())

    def __bool__(self):
        return bool(list(self))

JSON_OUTPUT_FORMAT = ('pgjson.caos', 1)
FREEFORM_RECORD_ID = '6e51108d-7440-47f7-8c65-dc4d43fd90d2'

class Json(bytes):
    def __new__(cls, value, *, format_info=JSON_OUTPUT_FORMAT):
        result = super().__new__(cls, value)
        result._fmt = format_info
        return result

    def __mm_json__(self):
        fmt = '["{}",{}]'.format(*self._fmt).encode('ascii')
        return b'{"$sxjson$":{"format":' + fmt  + b',"data":[' + self + b']}}'

class TypeIO(pq3.TypeIO):
    RECORD_OID = 2249

    def __init__(self, database):
        super().__init__(database)
        self._pool = database._pool

    def identify(self, **identity_mappings):
        id = list(identity_mappings.items())
        ios = [pg_types_io.resolve(x[0]) for x in id]
        oids = list(self.database.sys.regtypes([x[1] for x in id]))

        update = [
            (oid, io if io.__class__ is tuple else io(oid, self))
            for oid, io in zip(oids, ios)
        ]
        self._cache.update(update)

    def resolve(self, typid, from_resolution_of=(), builtins=resolve, quote_ident=quote_ident):
        if typid == pg_types.JSONOID:
            return self.json_io_factory()
        else:
            return super().resolve(typid, from_resolution_of, builtins, quote_ident)

    def get_session(self):
        if self._pool:
            return self._pool.get_holder(self.database)
        else:
            return None

    def array_from_parts(self, parts):
        return super().array_from_parts(parts, ArrayType=Array)

    def json_io_factory(self):
        def _unpack_json(data):
            return Json(data)
        return (None, _unpack_json, Json)

    def array_io_factory(self, pack_element, unpack_element, typoid, hasbin_input, hasbin_output,
                               array_unpack=pg_types_io_lib.array_unpack):
        pack_array, unpack_array, atype = super().array_io_factory(pack_element, unpack_element,
                                                                   typoid,
                                                                   hasbin_input, hasbin_output)

        if typoid == self.RECORD_OID:
            def unpack_array(data, array_from_parts = self.array_from_parts):
                # Override unpacking of arrays containing source node records to filter
                # out None values.  This is necessary due to the absense of support for FILTER
                # in aggregates in Postgres.  In absolute majority of calls to array_agg(<concept>),
                # None values in the resulting array are an unndesired side effect of an outer join.
                #
                flags, typoid, dims, lbs, elements = array_unpack(data)
                elements = tuple(unpack_element(x) for x in elements if x is not None)
                dims[0] = len(elements)
                return array_from_parts((elements, dims, lbs))

        return (pack_array, unpack_array, atype)

    def anon_record_io_factory(self):
        def raise_unpack_tuple_error(cause, procs, tup, itemnum):
            data = repr(tup[itemnum])
            if len(data) > 80:
                # Be sure not to fill screen with noise.
                data = data[:75] + ' ...'
            self.raise_client_error(element.ClientError((
                (b'C', '--cIO',),
                (b'S', 'ERROR',),
                (b'M', 'Could not unpack element {} from anonymous record'.format(itemnum)),
                (b'W', data,),
                (b'P', str(itemnum),)
            )), cause = cause)

        def _unpack_record(data, unpack = pg_types_io_lib.record_unpack,
                                 process_tuple = process_tuple,
                                 _Row=pg_types.Row.from_sequence):
            record = list(unpack(data))
            coloids = tuple(x[0] for x in record)

            record_info = None

            try:
                marker_oid = self.database._sx_known_record_marker_oid_
            except AttributeError:
                pass
            else:
                if record and record[0][0] == marker_oid:
                    session = self.get_session()
                    if session is not None:
                        recid = record[0][1].decode('ascii')
                        record_info = session.backend._get_record_info_by_id(recid)

            colio = map(self.resolve, coloids)
            column_unpack = tuple(c[1] or self.decode for c in colio)

            data = tuple(x[1] for x in record)
            data = process_tuple(column_unpack, data, raise_unpack_tuple_error)

            if record_info is not None:
                data = dict(zip(record_info.attribute_map, data[1:]))

                if record_info.is_xvalue:
                    assert data['value'] is not None
                    data = xvalue(data['value'], **data['attrs'])

                elif record_info.proto_class == 'metamagic.caos.proto.Concept':
                    data = session.backend.entity_from_row(session, record_info, data)

            return data

        return (None, _unpack_record)

    def RowTypeFactory(self, attribute_map={}, _Row=pg_types.Row.from_sequence,
                       composite_relid = None):
        session = self.get_session()
        if session is not None:
            backend = session.backend
            source = backend.source_name_from_relid(composite_relid)
            if source is not None:
                return partial(backend.entity_from_row_compat, session, source, attribute_map)
        return partial(_Row, attribute_map)

class Connection(pq3.Connection, caos_pool.Connection):
    def __init__(self, connector, pool=None):
        caos_pool.Connection.__init__(self, pool)
        pq3.Connection.__init__(self, connector)

        self._prepared_statements = {}

    def get_prepared_statement(self, query, raw=True):
        try:
            statement = self._prepared_statements[query, raw]
        except KeyError:
            try:
                if raw:
                    stmt_id = 'sx_{:x}'.format(hash(query)).replace('-', '_')
                    prefix = 'PREPARE {} AS '.format(stmt_id)
                    self.execute('{}{}'.format(prefix, query))
                    statement = self.statement_from_id(stmt_id)
                else:
                    prefix = ''
                    statement = self.prepare(query)

            except postgresql.exceptions.Error as e:
                e.__suppress_context__ = True
                raise backend_exc.QueryError(driver_err=e, query_text=query,
                                             query_offset=len(prefix)) from e

            except Exception as e:
                raise e from None

            else:
                self._prepared_statements[query, raw] = statement

        return statement

    def connect(self):
        super().connect()
        if self._pool:
            self._pool.backend.init_connection(self)

    def reset(self, hard=False):
        if self.state in ('failed', 'negotiating', 'busy'):
            self.execute('ROLLBACK')

        if hard:
            self.execute('DISCARD ALL')
            self._prepared_statements.clear()
        else:
            self.execute("SET SESSION AUTHORIZATION DEFAULT;")
            self.execute("RESET ALL;")
            self.execute("CLOSE ALL;")
            self.execute("UNLISTEN *;")
            self.execute("SELECT pg_advisory_unlock_all();")

    def close(self):
        self._prepared_statements.clear()
        super().close()

class Driver(green_postgres.Driver):
    def __init__(self):
        super().__init__(typio=TypeIO, connection=Connection)

driver = Driver()

def connector(iri, async=False):
    return green_postgres.connector(iri, async=async, driver=driver)
<|endoftext|>


<commit_before>#!/usr/bin/env python
from concurrent import futures
from logging import getLogger, StreamHandler, FileHandler, Formatter, INFO, DEBUG
import multiprocessing
import os
from shutil import copy2, rmtree
from subprocess import Popen, PIPE
import pandas as pd
from make_wgs_aux import main as mkwgsaux

# initialize logger
logger = getLogger(__name__)
shandler = StreamHandler()
shandler.setFormatter(Formatter("%(name)s %(levelname)s: %(message)s"))
fhandler = FileHandler(filename="batch_analysis.log")
fhandler.setFormatter(Formatter("%(name)s %(levelname)s: %(message)s"))
shandler.setLevel(INFO)
fhandler.setLevel(INFO)
logger.setLevel(INFO)
logger.addHandler(shandler)
logger.addHandler(fhandler)

def run_panin(directory, aux):
    process = Popen("./panin", stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=directory)
    aux = str.encode(aux)
    stdout, stderr = process.communicate(aux)
    if stderr:
        logger.error("\n".join((directory, str(stderr))))
    else:
        logger.debug("\n".join((directory, str(stdout))))

def run_panair(directory):
    process = Popen("./panair", stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=directory)
    stdout, stderr = process.communicate(b"a502.in")
    if stderr:
        logger.error("\n".join((directory, str(stderr))))
    else:
        logger.debug("\n".join((directory, str(stdout))))
    try:
        with open(os.path.join(directory, "panair.err")) as f:
            if "PanAir is stopping." in f.read():
                logger.critical("\n".join((directory, "fatal error with PanAir")))
    except FileNotFoundError:
        pass

def safe_makedirs(path):
    try:
        os.makedirs(path)
    except OSError:
        if not os.path.isdir(path):
            logger.error("could not create directory at {}".format(path))

def copy2dir(files, olddir, newdir):
    safe_makedirs(newdir)
    if type(files) is str:
        files = (files, )
    for f in files:
        try:
            copy2("{0}/{1}".format(olddir, f), "{0}/{1}".format(newdir, f))
        except FileNotFoundError as e:
            logger.error(e)

def run_analysis(casenum, aux, analysis_dir, params):
    # create directory to run panin and panair
    procid = int(multiprocessing.current_process().pid)
    logger.info("calculating case{} with procid {}".format(casenum, procid))
    target_dir = os.path.join("panair{}".format(procid))
    safe_makedirs(target_dir)

    # run panin and panair
    mkwgsaux(target_dir=target_dir, **params)
    run_panin(target_dir, aux)
    run_panair(target_dir)

    # save results
    files_to_save = ("panair.out", "ffmf", "agps", "panair.err", "panin.dbg", "a502.in")
    newdir = os.path.join("results", "case{}".format(casenum))
    copy2dir(files_to_save, target_dir, newdir)

    # delete intermediate files
    rmtree(target_dir)

if __name__ == '__main__':
    # set variables
    N_PROCS = 3
    AUXNAME = "ADODG_case3.aux"
    ANALYSIS_DIR = "" # directory to run analysis (intermediate files will be stored here)

    logger.info("start batch analysis")

    # read caselist
    caselist = pd.read_csv("caselist.csv")
    with futures.ProcessPoolExecutor(max_workers=N_PROCS) as executor:
        # submit jobs
        fs = list()
        for _, case in caselist.iterrows():
            casenum = int(case["casenum"])
            params = case[2:].to_dict()
            if case["run"] == 1:
                fs.append(executor.submit(run_analysis, casenum, AUXNAME, ANALYSIS_DIR, params))
            else:
                logger.debug("skipping case{}".format(casenum))
                continue

    # run analysis
    futures.wait(fs, return_when=futures.ALL_COMPLETED)

    logger.info("finish batch analysis")<commit_msg>Fix batch analysis<commit_after>#!/usr/bin/env python
from concurrent import futures
from logging import getLogger, StreamHandler, FileHandler, Formatter, INFO, DEBUG
import multiprocessing
import os
from shutil import copy2, rmtree
from subprocess import Popen, PIPE
import pandas as pd
from make_wgs_aux import main as mkwgsaux

# initialize logger
logger = getLogger(__name__)
shandler = StreamHandler()
shandler.setFormatter(Formatter("%(name)s %(levelname)s: %(message)s"))
fhandler = FileHandler(filename="batch_analysis.log")
fhandler.setFormatter(Formatter("%(name)s %(levelname)s: %(message)s"))
shandler.setLevel(INFO)
fhandler.setLevel(INFO)
logger.setLevel(INFO)
logger.addHandler(shandler)
logger.addHandler(fhandler)

def run_panin(directory, aux):
    process = Popen("./panin", stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=directory)
    aux = str.encode(aux)
    stdout, stderr = process.communicate(aux)
    if stderr:
        logger.error("\n".join((directory, str(stderr))))
    else:
        logger.debug("\n".join((directory, str(stdout))))

def run_panair(directory):
    process = Popen("./panair", stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=directory)
    stdout, stderr = process.communicate(b"a502.in")
    if stderr:
        logger.error("\n".join((directory, str(stderr))))
    else:
        logger.debug("\n".join((directory, str(stdout))))
    try:
        with open(os.path.join(directory, "panair.err")) as f:
            if "PanAir is stopping." in f.read():
                logger.critical("\n".join((directory, "fatal error with PanAir")))
    except FileNotFoundError:
        pass

def safe_makedirs(path):
    try:
        os.makedirs(path)
    except OSError:
        if not os.path.isdir(path):
            logger.error("could not create directory at {}".format(path))

def copy2dir(files, olddir, newdir):
    safe_makedirs(newdir)
    if type(files) is str:
        files = (files, )
    for f in files:
        try:
            copy2("{0}/{1}".format(olddir, f), "{0}/{1}".format(newdir, f))
        except FileNotFoundError as e:
            logger.error(e)

def run_analysis(casenum, aux, analysis_dir, params):
    # create directory to run panin and panair
    procid = int(multiprocessing.current_process().pid)
    logger.info("calculating case{} with procid {}".format(casenum, procid))
    target_dir = os.path.join(analysis_dir, "panair{}".format(procid))
    safe_makedirs(target_dir)

    # run panin and panair
    mkwgsaux(target_dir=target_dir, **params)
    run_panin(target_dir, aux)
    run_panair(target_dir)

    # save results
    files_to_save = ("panair.out", "ffmf", "agps", "panair.err", "panin.dbg", "a502.in")
    newdir = os.path.join("results", "case{}".format(casenum))
    copy2dir(files_to_save, target_dir, newdir)

    # delete intermediate files
    rmtree(target_dir)

if __name__ == '__main__':
    # set variables
    N_PROCS = 3
    AUXNAME = "ADODG_case3.aux"
    ANALYSIS_DIR = ""  # directory to run analysis (intermediate files will be stored here)

    logger.info("start batch analysis")

    # read caselist
    caselist = pd.read_csv("caselist.csv")
    with futures.ProcessPoolExecutor(max_workers=N_PROCS) as executor:
        # submit jobs
        fs = list()
        for _, case in caselist.iterrows():
            casenum = int(case["casenum"])
            params = case[2:].to_dict()
            if case["run"] == 1:
                fs.append(executor.submit(run_analysis, casenum, AUXNAME, ANALYSIS_DIR, params))
            else:
                logger.debug("skipping case{}".format(casenum))
                continue

    # run analysis
    futures.wait(fs, return_when=futures.ALL_COMPLETED)

    logger.info("finish batch analysis")
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
#vim:set et sts=4 sw=4:
#
# Flies Python Client
#
# Copyright (c) 2010 Jian Ni <jni@gmail.com>
# Copyright (c) 2010 Red Hat, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330,
# Boston, MA  02111-1307  USA

import getopt, sys
import json
import os.path
from parseconfig import FliesConfig
from flieslib.client import FliesClient
from flieslib.client import NoSuchFileException
from flieslib.client import NoSuchProjectException
from flieslib.client import UnAuthorizedException

sub_command = {
                'help':[],
                'list':[],
                'status':[],
                'project':['info','create', 'remove'],
                'iteration':['info', 'create', 'remove'],
                'publican':['push', 'pull']
                }

class FliesConsole:

    def __init__(self):
        config = FliesConfig()
    	server = config.get_config_value("server")
    	project_id = config.get_config_value("project.id")
    	iteration_id = config.get_config_value("project.iteration.id")
    	user = config.get_config_value("user")
    	apikey = config.get_config_value("apikey")
        self.options = {
                        'server' : server,
                        'project_id':project_id,
                        'iteration_id':iteration_id,
                        'user':user,
                        'apikey':apikey
                        }

    def _PrintUsage(self):
        print ('\nClient for talking to a Flies Server\n\n'
               'basic commands:\n\n'
               'list             List all available projects\n'
               'projectinfo      Retrieve a project\n'
               'iterationinfo    Retrieve a iteration\n\n'
               "Use 'flies help' for the full list of commands")

    def _print_help_info(self, args):
        if not args:
            print ('Client for talking to a Flies Server:\n\n'
                  'list of commands:\n\n'
                  ' list                List all available projects\n'
                  ' projectinfo         Retrieve a project\n'
                  ' iterationinfo       Retrieve a iteration\n'
                  ' create project      Create a project\n'
                  ' create iteration    Create a iteration of a project\n'
                  ' publican pull       Pull the content of publican file\n'
                  ' publican push       Push the content of publican to Flies Server\n')
        else:
            if args[0] == 'list':
                self._print_list_help()

    def _print_list_help(self):
       	print ('flies list [OPTIONS]\n\n'
               'list all available projects\n\n'
               'options:\n\n'
               ' --server url address of the Flies server')

    def _PrintProjectInfoHelp(self):
	    print ('flies project info [OPTIONS]')

    def _PrintProjectCreateHelp(self):
        print ('flies project create [PROJECT_ID] [OPTIONS]')

    def _PrintIterationInfoHelp(self):
	    print ('flies iterationinfo [OPTIONS]')

    def _PrintIterationCreateHelp(self):
        print ('flies create iteration [ITERATION_ID] [OPTIONS]')

    def _list_projects(self):
        if not self.options['server']:
            print "Please provide valid server url by fliesrc or by '--server' option"
            sys.exit()

        flies = FliesClient(self.options['server'])
        res, content = flies.list_projects()
        print 'Status: '+res['status']
        if res.get('status') == '200':
            projects = json.loads(content)
            for project in projects:
                print "*"*40
                print project
        else:
            print 'Flies REST service not available at %s' % self.server

    def _get_project(self):
        if not self.server:
            print "Please provide valid server url by fliesrc or by '--server' option"
            sys.exit()

        if not self.project_id:
            print 'Please use flies project info --project=project_id to retrieve the project info'
            sys.exit()

        flies = FliesClient(self.server)
        res, content = flies.get_project_info(project_id)
        print 'Status: '+res['status']
        print content

    def _get_iteration(self):
        if not self.server:
            print "Please provide valid server url by fliesrc or by '--server' option"
            sys.exit()

        if not self.iteration_id or not self.project_id:
            print 'Please use flies iterationinfo|info --project=project_id --iteration=iteration_id to retrieve the iteration'

        flies = FliesClient(self.server)
        res, content = flies.get_iteration_info(self.project_id, self.iteration_id)
        print 'Status: '+res['status']
        print content

    def _create_project(self, args):
        if not self.server:
            print "Please provide valid server url by fliesrc or by '--server' option"
            sys.exit()

        if self.user and self.apikey:
            flies = FliesClient(self.server, self.user, self.apikey)
        else:
            print "Please provide username and apikey in .fliesrc"
            sys.exit()

        try:
            result = flies.create_project(project_id, self.name, self.desc)
        except NoSuchProjectException as e:
            print "No Such Project on the server"
        except UnAuthorizedException as e:
            print "Unauthorized Operation"

    def _create_iteration(self, args):
        if not self.server:
            print "Please provide valid server url by fliesrc or by '--server' option"
            sys.exit()

        if self.user and self.apikey:
            flies = FliesClient(self.server, self.user, self.apikey)
        else:
            print "Please provide username and apikey in .fliesrc"
            sys.exit()

        if not self.project_id:
            print "Please provide PROJECT_ID for creating iteration"
            sys.exit()

        try:
            result = flies.create_iteration(self.project_id, self.args[0], self.name, self.desc)
        except NoSuchProjectException as e:
            print "No Such Project on the server"

    def _push_publican(self, filename):
        if not self.server:
            print "Please provide valid server url by fliesrc or by '--server' option"
            sys.exit()

        if self.user and self.apikey:
            flies = FliesClient(self.server, self.user, self.apikey)
        else:
            print "Please provide username and apikey in .fliesrc"
            sys.exit()

        if not self.project_id:
            print "Please provide PROJECT_ID for creating iteration"
            sys.exit()

        try:
       	    result = flies.push_publican(filename, self.project_id, self.iteration_id)
        except NoSuchFileException as e:
       	    print "Can not find file"
        except NoSuchProjectException as e:
            print "No Such Project on the server"

    def _pull_publican(self):
        pass

    def _remove_project(self):
        pass

    def _remove_iteration(self):
        pass

    def _project_status(self):
        pass

    def _process_command_line(self):
        try:
            opts, args = getopt.gnu_getopt(sys.argv[1:], "v", ["server=", "project=", "iteration=", "name=", "description="])
        except getopt.GetoptError, err:
            print str(err)
            sys.exit(2)

        if args:
            command = args[0]
            sub = args[1:]
            if sub_command[command]:
                if sub[0]:
                    if sub[0] in sub_command[command]:
                        command = command+'_'+sub[0]
                        command_args = sub[2:]
                    else:
                        print "Can not find such command"
                else:
                    print "Please complete the command!"
            else:
                command_args = sub
        else:
            self._PrintUsage()
            sys.exit(2)

        if opts:
            for o, a in opts:
                if o in ("--server"):
                    self.options[server] = a
                elif o in ("--name"):
                    self.options[name] = a
                elif o in ("--description"):
                    self.options[desc] = a
                elif o in ("--project"):
                    self.options[project_id] = a
                elif o in ("--iteration"):
                    self.options[iteration_id] = a

        return command, command_args

    def run(self):
        command, command_args = self._process_command_line()

        if command == 'help':
            self._print_help_info(command_args)
        elif command == 'list':
            self._list_projects()
        elif command == 'status':
            self._poject_status()
        elif command == 'project_info':
            self._GetProject()
        elif command == 'project_create':
            self._create_project(command_args)
        elif command == 'project_remove':
            self._remove_project(command_args)
        elif command == 'iteration_info':
            self._GetIteration(options)
        elif command == 'iteration_create':
            self._create_iteration(command_args)
        elif command == 'iteration_remove':
            self._remove_iteration(command_args)
        elif command == 'publican_push':
            self._push_publican(comand_args)
        elif command == 'publican_pull':
            self._push_publican(comand_args)
        else:
            self._PrintUsage()
            sys.exit(2)

def main():
    client = FliesConsole()
    client.run()

if __name__ == "__main__":
    main()
<commit_msg>Modify the function name<commit_after>#!/usr/bin/env python
#
#vim:set et sts=4 sw=4:
#
# Flies Python Client
#
# Copyright (c) 2010 Jian Ni <jni@gmail.com>
# Copyright (c) 2010 Red Hat, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330,
# Boston, MA  02111-1307  USA

import getopt, sys
import json
import os.path
from parseconfig import FliesConfig
from flieslib.client import FliesClient
from flieslib.client import NoSuchFileException
from flieslib.client import NoSuchProjectException
from flieslib.client import UnAuthorizedException

sub_command = {
                'help':[],
                'list':[],
                'status':[],
                'project':['info','create', 'remove'],
                'iteration':['info', 'create', 'remove'],
                'publican':['push', 'pull']
                }

class FliesConsole:

    def __init__(self):
        config = FliesConfig()
    	server = config.get_config_value("server")
    	project_id = config.get_config_value("project.id")
    	iteration_id = config.get_config_value("project.iteration.id")
    	user = config.get_config_value("user")
    	apikey = config.get_config_value("apikey")
        self.options = {
                        'server' : server,
                        'project_id':project_id,
                        'iteration_id':iteration_id,
                        'user':user,
                        'apikey':apikey
                        }

    def _PrintUsage(self):
        print ('\nClient for talking to a Flies Server\n\n'
               'basic commands:\n\n'
               'list             List all available projects\n'
               'projectinfo      Retrieve a project\n'
               'iterationinfo    Retrieve a iteration\n\n'
               "Use 'flies help' for the full list of commands")

    def _print_help_info(self, args):
        if not args:
            print ('Client for talking to a Flies Server:\n\n'
                  'list of commands:\n\n'
                  ' list                List all available projects\n'
                  ' projectinfo         Retrieve a project\n'
                  ' iterationinfo       Retrieve a iteration\n'
                  ' create project      Create a project\n'
                  ' create iteration    Create a iteration of a project\n'
                  ' publican pull       Pull the content of publican file\n'
                  ' publican push       Push the content of publican to Flies Server\n')
        else:
            if args[0] == 'list':
                self._list_help()

    def _list_help(self):
       	print ('flies list [OPTIONS]\n\n'
               'list all available projects\n\n'
               'options:\n\n'
               ' --server url address of the Flies server')

    def _projec_info_help(self):
	    print ('flies project info [OPTIONS]')

    def _project_create_help(self):
        print ('flies project create [PROJECT_ID] [OPTIONS]')

    def _iteration_info_help(self):
	    print ('flies iterationinfo [OPTIONS]')

    def _iteration_create_help(self):
        print ('flies create iteration [ITERATION_ID] [OPTIONS]')

    def _list_projects(self):
        if not self.options['server']:
            print "Please provide valid server url by fliesrc or by '--server' option"
            sys.exit()

        flies = FliesClient(self.options['server'])
        res, content = flies.list_projects()
        print 'Status: '+res['status']
        if res.get('status') == '200':
            projects = json.loads(content)
            for project in projects:
                print "*"*40
                print project
        else:
            print 'Flies REST service not available at %s' % self.server

    def _get_project(self):
        if not self.server:
            print "Please provide valid server url by fliesrc or by '--server' option"
            sys.exit()

        if not self.project_id:
            print 'Please use flies project info --project=project_id to retrieve the project info'
            sys.exit()

        flies = FliesClient(self.server)
        res, content = flies.get_project_info(project_id)
        print 'Status: '+res['status']
        print content

    def _get_iteration(self):
        if not self.server:
            print "Please provide valid server url by fliesrc or by '--server' option"
            sys.exit()

        if not self.iteration_id or not self.project_id:
            print 'Please use flies iterationinfo|info --project=project_id --iteration=iteration_id to retrieve the iteration'

        flies = FliesClient(self.server)
        res, content = flies.get_iteration_info(self.project_id, self.iteration_id)
        print 'Status: '+res['status']
        print content

    def _create_project(self, args):
        if not self.server:
            print "Please provide valid server url by fliesrc or by '--server' option"
            sys.exit()

        if self.user and self.apikey:
            flies = FliesClient(self.server, self.user, self.apikey)
        else:
            print "Please provide username and apikey in .fliesrc"
            sys.exit()

        try:
            result = flies.create_project(project_id, self.name, self.desc)
        except NoSuchProjectException as e:
            print "No Such Project on the server"
        except UnAuthorizedException as e:
            print "Unauthorized Operation"

    def _create_iteration(self, args):
        if not self.server:
            print "Please provide valid server url by fliesrc or by '--server' option"
            sys.exit()

        if self.user and self.apikey:
            flies = FliesClient(self.server, self.user, self.apikey)
        else:
            print "Please provide username and apikey in .fliesrc"
            sys.exit()

        if not self.project_id:
            print "Please provide PROJECT_ID for creating iteration"
            sys.exit()

        try:
            result = flies.create_iteration(self.project_id, self.args[0], self.name, self.desc)
        except NoSuchProjectException as e:
            print "No Such Project on the server"

    def _push_publican(self, filename):
        if not self.server:
            print "Please provide valid server url by fliesrc or by '--server' option"
            sys.exit()

        if self.user and self.apikey:
            flies = FliesClient(self.server, self.user, self.apikey)
        else:
            print "Please provide username and apikey in .fliesrc"
            sys.exit()

        if not self.project_id:
            print "Please provide PROJECT_ID for creating iteration"
            sys.exit()

        try:
       	    result = flies.push_publican(filename, self.project_id, self.iteration_id)
        except NoSuchFileException as e:
       	    print "Can not find file"
        except NoSuchProjectException as e:
            print "No Such Project on the server"

    def _pull_publican(self):
        pass

    def _remove_project(self):
        pass

    def _remove_iteration(self):
        pass

    def _project_status(self):
        pass

    def _process_command_line(self):
        try:
            opts, args = getopt.gnu_getopt(sys.argv[1:], "v", ["server=", "project=", "iteration=", "name=", "description="])
        except getopt.GetoptError, err:
            print str(err)
            sys.exit(2)

        if args:
            command = args[0]
            sub = args[1:]
            if sub_command[command]:
                if sub[0]:
                    if sub[0] in sub_command[command]:
                        command = command+'_'+sub[0]
                        command_args = sub[2:]
                    else:
                        print "Can not find such command"
                else:
                    print "Please complete the command!"
            else:
                command_args = sub
        else:
            self._PrintUsage()
            sys.exit(2)

        if opts:
            for o, a in opts:
                if o in ("--server"):
                    self.options[server] = a
                elif o in ("--name"):
                    self.options[name] = a
                elif o in ("--description"):
                    self.options[desc] = a
                elif o in ("--project"):
                    self.options[project_id] = a
                elif o in ("--iteration"):
                    self.options[iteration_id] = a

        return command, command_args

    def run(self):
        command, command_args = self._process_command_line()

        if command == 'help':
            self._print_help_info(command_args)
        elif command == 'list':
            self._list_projects()
        elif command == 'status':
            self._poject_status()
        elif command == 'project_info':
            self._GetProject()
        elif command == 'project_create':
            self._create_project(command_args)
        elif command == 'project_remove':
            self._remove_project(command_args)
        elif command == 'iteration_info':
            self._GetIteration(options)
        elif command == 'iteration_create':
            self._create_iteration(command_args)
        elif command == 'iteration_remove':
            self._remove_iteration(command_args)
        elif command == 'publican_push':
            self._push_publican(comand_args)
        elif command == 'publican_pull':
            self._push_publican(comand_args)
        else:
            self._PrintUsage()
            sys.exit(2)

def main():
    client = FliesConsole()
    client.run()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#  This file is part of the myhdl library, a Python package for using
#  Python as a Hardware Description Language.
#
#  Copyright (C) 2003-2016 Jan Decaluwe
#
#  The myhdl library is free software; you can redistribute it and/or
#  modify it under the terms of the GNU Lesser General Public License as
#  published by the Free Software Foundation; either version 2.1 of the
#  License, or (at your option) any later version.
#
#  This library is distributed in the hope that it will be useful, but
#  WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  Lesser General Public License for more details.

#  You should have received a copy of the GNU Lesser General Public
#  License along with this library; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

""" Block with the @block decorator function. """
from __future__ import absolute_import

import inspect

from functools import wraps

import myhdl
from myhdl import BlockError, BlockInstanceError, Cosimulation
from myhdl._instance import _Instantiator
from myhdl._util import _flatten
from myhdl._extractHierarchy import (_makeMemInfo,
                                     _UserVerilogCode, _UserVhdlCode)
from myhdl._Signal import _Signal, _isListOfSigs

class _error:
    pass
_error.ArgType = "A block should return block or instantiator objects"
_error.InstanceError = "%s: subblock %s should be encapsulated in a block decorator"

class _CallInfo(object):

    def __init__(self, name, modctxt, symdict):
        self.name = name
        self.modctxt = modctxt
        self.symdict = symdict

def _getCallInfo():
    """Get info on the caller of a BlockInstance.

    A BlockInstance should be used in a block context.
    This function gets the required info from the caller
    It uses the frame stack:
    0: this function
    1: block instance constructor
    2: the decorator function call
    3: the function that defines instances
    4: the caller of the block function, e.g. a BlockInstance.

    """

    stack = inspect.stack()
    # caller may be undefined if instantiation from a Python module
    callerrec = None
    funcrec = stack[3]
    if len(stack) > 4:
        callerrec = stack[4]

    name = funcrec[3]
    frame = funcrec[0]
    symdict = dict(frame.f_globals)
    symdict.update(frame.f_locals)
    modctxt = False
    if callerrec is not None:
        f_locals = callerrec[0].f_locals
        if 'self' in f_locals:
            modctxt = isinstance(f_locals['self'], _Block)
    return _CallInfo(name, modctxt, symdict)

def block(func):
    srcfile = inspect.getsourcefile(func)
    srcline = inspect.getsourcelines(func)[0]

    @wraps(func)
    def deco(*args, **kwargs):
        deco.calls += 1
        return _Block(func, deco, srcfile, srcline, *args, **kwargs)
    deco.calls = 0
    return deco

class _Block(object):

    def __init__(self, func, deco, srcfile, srcline, *args, **kwargs):
        calls = deco.calls
        self.func = func
        self.args = args
        self.kwargs = kwargs
        self.__doc__ = func.__doc__
        callinfo = _getCallInfo()
        self.callinfo = callinfo
        self.modctxt = callinfo.modctxt
        self.callername = callinfo.name
        self.symdict = None
        self.sigdict = {}
        self.memdict = {}
        # flatten, but keep BlockInstance objects
        self.subs = _flatten(func(*args, **kwargs))
        self._verifySubs()
        self._updateNamespaces()
        self.name = self.__name__ = func.__name__ + '_' + str(calls - 1)
        self.verilog_code = self.vhdl_code = None
        self.sim = None
        if hasattr(deco, 'verilog_code'):
            self.verilog_code = _UserVerilogCode(deco.verilog_code, self.symdict, func.__name__,
                                                 func, srcfile, srcline)
        if hasattr(deco, 'vhdl_code'):
            self.vhdl_code = _UserVhdlCode(deco.vhdl_code, self.symdict, func.__name__,
                                           func, srcfile, srcline)
        self._config_sim = {'trace': False}

    def _verifySubs(self):
        for inst in self.subs:
            if not isinstance(inst, (_Block, _Instantiator, Cosimulation)):
                raise BlockError(_error.ArgType)
            if isinstance(inst, (_Block, _Instantiator)):
                if not inst.modctxt:
                    raise BlockError(_error.InstanceError % (self.name, inst.callername))

    def _updateNamespaces(self):
        # dicts to keep track of objects used in Instantiator objects
        usedsigdict = {}
        usedlosdict = {}
        for inst in self.subs:
            # the symdict of a block instance is defined by
            # the call context of its instantiations
            if isinstance(inst, Cosimulation):
                continue  # ignore
            if self.symdict is None:
                self.symdict = inst.callinfo.symdict
            if isinstance(inst, _Instantiator):
                usedsigdict.update(inst.sigdict)
                usedlosdict.update(inst.losdict)
        if self.symdict is None:
            self.symdict = {}
        # Special case: due to attribute reference transformation, the
        # sigdict and losdict from Instantiator objects may contain new
        # references. Therefore, update the symdict with them.
        # To be revisited.
        self.symdict.update(usedsigdict)
        self.symdict.update(usedlosdict)
        # Infer sigdict and memdict, with compatibility patches from _extractHierarchy
        for n, v in self.symdict.items():
            if isinstance(v, _Signal):
                self.sigdict[n] = v
                if n in usedsigdict:
                    v._markUsed()
            if _isListOfSigs(v):
                m = _makeMemInfo(v)
                self.memdict[n] = m
                if n in usedlosdict:
                    m._used = True

    def _inferInterface(self):
        from myhdl.conversion._analyze import _analyzeTopFunc
        intf = _analyzeTopFunc(self.func, *self.args, **self.kwargs)
        self.argnames = intf.argnames
        self.argdict = intf.argdict

    # Public methods
    # The puropse now is to define the API, optimizations later

    def verify_convert(self):
        return myhdl.conversion.verify(self)

    def analyze_convert(self):
        return myhdl.conversion.analyze(self)

    def convert(self, hdl='Verilog', **kwargs):
        """Converts this BlockInstance to another HDL

        Args:
            hdl (Optional[str]): Target HDL. Defaults to Verilog
            path (Optional[str]): Destination folder. Defaults to current
                working dir.
            name (Optional[str]): Module and output file name. Defaults to
                `self.mod.__name__`
            trace(Optional[bool]): Verilog only. Whether the testbench should
                dump all signal waveforms. Defaults to False.
            tb (Optional[bool]): Verilog only. Specifies whether a testbench
                should be created. Defaults to True.
            timescale(Optional[str]): Verilog only. Defaults to '1ns/10ps'
        """
        if hdl.lower() == 'vhdl':
            converter = myhdl.conversion._toVHDL.toVHDL
        elif hdl.lower() == 'verilog':
            converter = myhdl.conversion._toVerilog.toVerilog
        else:
            raise BlockInstanceError('unknown hdl %s' % hdl)

        conv_attrs = {}
        if 'name' in kwargs:
            conv_attrs['name'] = kwargs.pop('name')
        conv_attrs['directory'] = kwargs.pop('path', '')
        if hdl.lower() == 'verilog':
            conv_attrs['no_testbench'] = not kwargs.pop('tb', True)
            conv_attrs['timescale'] = kwargs.pop('timescale', '1ns/10ps')
            conv_attrs['trace'] = kwargs.pop('trace', False)
        conv_attrs.update(kwargs)
        for k, v in conv_attrs.items():
            setattr(converter, k, v)
        return converter(self)

    def config_sim(self, trace=False):
        self._config_sim['trace'] = trace

    def run_sim(self, duration=None, quiet=0):
        if self.sim is None:
            sim = self
            if self._config_sim['trace']:
                sim = myhdl.traceSignals(self)
            self.sim = myhdl._Simulation.Simulation(sim)
        self.sim.run(duration, quiet)

    def quit_sim(self):
        if self.sim is not None:
            self.sim.quit()
<commit_msg>Moved assignment of name attribute in _Block to before where it might be needed by an error message.<commit_after>#  This file is part of the myhdl library, a Python package for using
#  Python as a Hardware Description Language.
#
#  Copyright (C) 2003-2016 Jan Decaluwe
#
#  The myhdl library is free software; you can redistribute it and/or
#  modify it under the terms of the GNU Lesser General Public License as
#  published by the Free Software Foundation; either version 2.1 of the
#  License, or (at your option) any later version.
#
#  This library is distributed in the hope that it will be useful, but
#  WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  Lesser General Public License for more details.

#  You should have received a copy of the GNU Lesser General Public
#  License along with this library; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

""" Block with the @block decorator function. """
from __future__ import absolute_import

import inspect

from functools import wraps

import myhdl
from myhdl import BlockError, BlockInstanceError, Cosimulation
from myhdl._instance import _Instantiator
from myhdl._util import _flatten
from myhdl._extractHierarchy import (_makeMemInfo,
                                     _UserVerilogCode, _UserVhdlCode)
from myhdl._Signal import _Signal, _isListOfSigs

class _error:
    pass
_error.ArgType = "A block should return block or instantiator objects"
_error.InstanceError = "%s: subblock %s should be encapsulated in a block decorator"

class _CallInfo(object):

    def __init__(self, name, modctxt, symdict):
        self.name = name
        self.modctxt = modctxt
        self.symdict = symdict

def _getCallInfo():
    """Get info on the caller of a BlockInstance.

    A BlockInstance should be used in a block context.
    This function gets the required info from the caller
    It uses the frame stack:
    0: this function
    1: block instance constructor
    2: the decorator function call
    3: the function that defines instances
    4: the caller of the block function, e.g. a BlockInstance.

    """

    stack = inspect.stack()
    # caller may be undefined if instantiation from a Python module
    callerrec = None
    funcrec = stack[3]
    if len(stack) > 4:
        callerrec = stack[4]

    name = funcrec[3]
    frame = funcrec[0]
    symdict = dict(frame.f_globals)
    symdict.update(frame.f_locals)
    modctxt = False
    if callerrec is not None:
        f_locals = callerrec[0].f_locals
        if 'self' in f_locals:
            modctxt = isinstance(f_locals['self'], _Block)
    return _CallInfo(name, modctxt, symdict)

def block(func):
    srcfile = inspect.getsourcefile(func)
    srcline = inspect.getsourcelines(func)[0]

    @wraps(func)
    def deco(*args, **kwargs):
        deco.calls += 1
        return _Block(func, deco, srcfile, srcline, *args, **kwargs)
    deco.calls = 0
    return deco

class _Block(object):

    def __init__(self, func, deco, srcfile, srcline, *args, **kwargs):
        calls = deco.calls
        self.func = func
        self.args = args
        self.kwargs = kwargs
        self.__doc__ = func.__doc__
        callinfo = _getCallInfo()
        self.callinfo = callinfo
        self.modctxt = callinfo.modctxt
        self.callername = callinfo.name
        self.symdict = None
        self.sigdict = {}
        self.memdict = {}
        self.name = self.__name__ = func.__name__ + '_' + str(calls - 1)

        # flatten, but keep BlockInstance objects
        self.subs = _flatten(func(*args, **kwargs))
        self._verifySubs()
        self._updateNamespaces()
        self.verilog_code = self.vhdl_code = None
        self.sim = None
        if hasattr(deco, 'verilog_code'):
            self.verilog_code = _UserVerilogCode(deco.verilog_code, self.symdict, func.__name__,
                                                 func, srcfile, srcline)
        if hasattr(deco, 'vhdl_code'):
            self.vhdl_code = _UserVhdlCode(deco.vhdl_code, self.symdict, func.__name__,
                                           func, srcfile, srcline)
        self._config_sim = {'trace': False}

    def _verifySubs(self):
        for inst in self.subs:
            if not isinstance(inst, (_Block, _Instantiator, Cosimulation)):
                raise BlockError(_error.ArgType)
            if isinstance(inst, (_Block, _Instantiator)):
                if not inst.modctxt:
                    raise BlockError(_error.InstanceError % (self.name, inst.callername))

    def _updateNamespaces(self):
        # dicts to keep track of objects used in Instantiator objects
        usedsigdict = {}
        usedlosdict = {}
        for inst in self.subs:
            # the symdict of a block instance is defined by
            # the call context of its instantiations
            if isinstance(inst, Cosimulation):
                continue  # ignore
            if self.symdict is None:
                self.symdict = inst.callinfo.symdict
            if isinstance(inst, _Instantiator):
                usedsigdict.update(inst.sigdict)
                usedlosdict.update(inst.losdict)
        if self.symdict is None:
            self.symdict = {}
        # Special case: due to attribute reference transformation, the
        # sigdict and losdict from Instantiator objects may contain new
        # references. Therefore, update the symdict with them.
        # To be revisited.
        self.symdict.update(usedsigdict)
        self.symdict.update(usedlosdict)
        # Infer sigdict and memdict, with compatibility patches from _extractHierarchy
        for n, v in self.symdict.items():
            if isinstance(v, _Signal):
                self.sigdict[n] = v
                if n in usedsigdict:
                    v._markUsed()
            if _isListOfSigs(v):
                m = _makeMemInfo(v)
                self.memdict[n] = m
                if n in usedlosdict:
                    m._used = True

    def _inferInterface(self):
        from myhdl.conversion._analyze import _analyzeTopFunc
        intf = _analyzeTopFunc(self.func, *self.args, **self.kwargs)
        self.argnames = intf.argnames
        self.argdict = intf.argdict

    # Public methods
    # The puropse now is to define the API, optimizations later

    def verify_convert(self):
        return myhdl.conversion.verify(self)

    def analyze_convert(self):
        return myhdl.conversion.analyze(self)

    def convert(self, hdl='Verilog', **kwargs):
        """Converts this BlockInstance to another HDL

        Args:
            hdl (Optional[str]): Target HDL. Defaults to Verilog
            path (Optional[str]): Destination folder. Defaults to current
                working dir.
            name (Optional[str]): Module and output file name. Defaults to
                `self.mod.__name__`
            trace(Optional[bool]): Verilog only. Whether the testbench should
                dump all signal waveforms. Defaults to False.
            tb (Optional[bool]): Verilog only. Specifies whether a testbench
                should be created. Defaults to True.
            timescale(Optional[str]): Verilog only. Defaults to '1ns/10ps'
        """
        if hdl.lower() == 'vhdl':
            converter = myhdl.conversion._toVHDL.toVHDL
        elif hdl.lower() == 'verilog':
            converter = myhdl.conversion._toVerilog.toVerilog
        else:
            raise BlockInstanceError('unknown hdl %s' % hdl)

        conv_attrs = {}
        if 'name' in kwargs:
            conv_attrs['name'] = kwargs.pop('name')
        conv_attrs['directory'] = kwargs.pop('path', '')
        if hdl.lower() == 'verilog':
            conv_attrs['no_testbench'] = not kwargs.pop('tb', True)
            conv_attrs['timescale'] = kwargs.pop('timescale', '1ns/10ps')
            conv_attrs['trace'] = kwargs.pop('trace', False)
        conv_attrs.update(kwargs)
        for k, v in conv_attrs.items():
            setattr(converter, k, v)
        return converter(self)

    def config_sim(self, trace=False):
        self._config_sim['trace'] = trace

    def run_sim(self, duration=None, quiet=0):
        if self.sim is None:
            sim = self
            if self._config_sim['trace']:
                sim = myhdl.traceSignals(self)
            self.sim = myhdl._Simulation.Simulation(sim)
        self.sim.run(duration, quiet)

    def quit_sim(self):
        if self.sim is not None:
            self.sim.quit()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import threading

from django.conf import settings
from djmail.backends import base
from djmail import core

class EmailBackend(base.BaseEmailBackend):
    def _load_settings(self):
        self._send_async = getattr(settings, "DJMAIL_SEND_ASYNC", True)

    def _send_messages(self, email_messages):
        if len(email_messages) == 0:
            return 0

        self._load_settings()

        if self._send_async:
            @core._close_connection_on_finish
            def _send(messages):
                return core._send_messages(email_messages)

            thread = threading.Thread(target=_send, args=[email_messages])
            thread.start()
            return thread

        return core._send_messages(email_messages)
<commit_msg>Update default.py<commit_after># -*- coding: utf-8 -*-

import threading

from django.conf import settings
from djmail.backends import base
from djmail import core

class EmailBackend(base.BaseEmailBackend):
    def _load_settings(self):
        self._send_async = getattr(settings, "DJMAIL_SEND_ASYNC", False)

    def _send_messages(self, email_messages):
        if len(email_messages) == 0:
            return 0

        self._load_settings()

        if self._send_async:
            @core._close_connection_on_finish
            def _send(messages):
                return core._send_messages(email_messages)

            thread = threading.Thread(target=_send, args=[email_messages])
            thread.start()
            return thread

        return core._send_messages(email_messages)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# function
def sum(op1, op2):
    return op1 + op2

my_sum = sum
print my_sum(1, 2)
print my_sum("I am ", "zzz");

# pass
print "press ctrl + c to continue"

while True:
    pass
<commit_msg>Test default value of a function<commit_after>#!/usr/bin/env python
# function
def sum(op1, op2):
    return op1 + op2

my_sum = sum
print my_sum(1, 2)
print my_sum("I am ", "zzz");

# Default value in a fuction
init = 12
def accumulate(val = init):
    val += val
    return val

my_accu = accumulate
init = 11
print my_accu() # is 12 + 12 rather than 11 + 11

# pass
print "press ctrl + c to continue"

while True:
    pass
<|endoftext|>