<commit_before># Copyright (c) 2012 iQIYI Inc.
# Copyright (c) 2013 Tencent Inc.
# All rights reserved.
#
# Author: Jingxu Chen <chenjingxu@qiyi.com>
#         Feng Chen <chen3feng@gmail.com>
# Date:   October 13, 2012

"""
 A helper class to get the files generated from thrift IDL files.

"""

import os

import blade
import configparse
import console

import build_rules
import java_jar_target
import py_targets

from blade_util import var_to_list
from cc_targets import CcTarget
from thrift_helper import ThriftHelper

class ThriftLibrary(CcTarget):
    """A scons thrift library target subclass.

    This class is derived from CcTarget.

    """
    def __init__(self,
                 name,
                 srcs,
                 deps,
                 optimize,
                 deprecated,
                 blade,
                 kwargs):
        """Init method.

        Init the thrift target.

        """
        srcs = var_to_list(srcs)
        self._check_thrift_srcs_name(srcs)
        CcTarget.__init__(self,
                          name,
                          'thrift_library',
                          srcs,
                          deps,
                          '',
                          [], [], [], optimize, [], [],
                          blade,
                          kwargs)
        self.data['python_vars'] = []
        self.data['python_sources'] = []

        thrift_config = configparse.blade_config.get_config('thrift_config')
        thrift_lib = var_to_list(thrift_config['thrift_libs'])
        thrift_bin = thrift_config['thrift']
        if thrift_bin.startswith("//"):
            self._add_hardcode_library(var_to_list(thrift_bin))

        # Hardcode deps rule to thrift libraries.
        self._add_hardcode_library(thrift_lib)

        # Link all the symbols by default
        self.data['link_all_symbols'] = True
        self.data['deprecated'] = deprecated
        self.data['java_sources_explict_dependency'] = []

        # For each thrift file initialize a ThriftHelper, which will be used
        # to get the source files generated from thrift file.
        self.thrift_helpers = {}
        for src in srcs:
            self.thrift_helpers[src] = ThriftHelper(
                    os.path.join(self.path, src))

    def _check_thrift_srcs_name(self, srcs):
        """_check_thrift_srcs_name.

        Checks whether the thrift file's name ends with 'thrift'.

        """
        error = 0
        for src in srcs:
            base_name = os.path.basename(src)
            pos = base_name.rfind('.')
            if pos == -1:
                console.error('invalid thrift file name %s' % src)
                error += 1
            file_suffix = base_name[pos + 1:]
            if file_suffix != 'thrift':
                console.error('invalid thrift file name %s' % src)
                error += 1
        if error > 0:
            console.error_exit('invalid thrift file names found.')

    def _thrift_gen_cpp_files(self, path, src):
        """_thrift_gen_cpp_files.

        Get the c++ files generated from thrift file.

        """

        return [self._target_file_path(path, f)
                for f in self.thrift_helpers[src].get_generated_cpp_files()]

    def _thrift_gen_py_files(self, path, src):
        """_thrift_gen_py_files.

        Get the python files generated from thrift file.

        """

        return [self._target_file_path(path, f)
                for f in self.thrift_helpers[src].get_generated_py_files()]

    def _thrift_gen_java_files(self, path, src):
        """_thrift_gen_java_files.

        Get the java files generated from thrift file.

        """

        return [self._target_file_path(path, f)
                for f in self.thrift_helpers[src].get_generated_java_files()]

    def _thrift_java_rules(self):
        """_thrift_java_rules.

        Generate scons rules for the java files from thrift file.

        """

        for src in self.srcs:
            src_path = os.path.join(self.path, src)
            thrift_java_src_files = self._thrift_gen_java_files(self.path,
                                                                src)

            self._write_rule('%s.ThriftJava(%s, "%s")' % (
                    self._env_name(),
                    str(thrift_java_src_files),
                    src_path))

            self.data['java_sources'] = (
                     os.path.dirname(thrift_java_src_files[0]),
                     os.path.join(self.build_path, self.path),
                     self.name)

            self.data['java_sources_explict_dependency'] += thrift_java_src_files

    def _thrift_python_rules(self):
        """_thrift_python_rules.

        Generate python files.

        """
        for src in self.srcs:
            src_path = os.path.join(self.path, src)
            thrift_py_src_files = self._thrift_gen_py_files(self.path, src)
            py_cmd_var = '%s_python' % self._generate_variable_name(
                    self.path, self.name)
            self._write_rule('%s = %s.ThriftPython(%s, "%s")' % (
                    py_cmd_var,
                    self._env_name(),
                    str(thrift_py_src_files),
                    src_path))
            self.data['python_vars'].append(py_cmd_var)
            self.data['python_sources'] += thrift_py_src_files

    def scons_rules(self):
        """scons_rules.

        It outputs the scons rules according to user options.

        """
        self._prepare_to_generate_rule()

        # Build java source according to its option
        env_name = self._env_name()

        self.options = self.blade.get_options()
        self.direct_targets = self.blade.get_direct_targets()

        if (getattr(self.options, 'generate_java', False) or
            self.data.get('generate_java') or
            self.key in self.direct_targets):
            self._thrift_java_rules()

        if (getattr(self.options, 'generate_python', False) or
            self.data.get('generate_python') or
            self.key in self.direct_targets):
            self._thrift_python_rules()

        self._setup_cc_flags()

        sources = []
        obj_names = []
        for src in self.srcs:
            thrift_cpp_files = self._thrift_gen_cpp_files(self.path, src)
            thrift_cpp_src_files = [f for f in thrift_cpp_files if f.endswith('.cpp')]

            self._write_rule('%s.Thrift(%s, "%s")' % (
                    env_name,
                    str(thrift_cpp_files),
                    os.path.join(self.path, src)))

            for thrift_cpp_src in thrift_cpp_src_files:
                obj_name = '%s_object' % self._generate_variable_name(
                    self.path, thrift_cpp_src)
                obj_names.append(obj_name)
                self._write_rule(
                    '%s = %s.SharedObject(target="%s" + top_env["OBJSUFFIX"], '
                    'source="%s")' % (obj_name,
                                      env_name,
                                      thrift_cpp_src,
                                      thrift_cpp_src))
                sources.append(thrift_cpp_src)

        self._write_rule('%s = [%s]' % (self._objs_name(), ','.join(obj_names)))
        self._write_rule('%s.Depends(%s, %s)' % (
                         env_name, self._objs_name(), sources))

        self._cc_library()
        options = self.blade.get_options()
        if (getattr(options, 'generate_dynamic', False) or
            self.data.get('build_dynamic')):
            self._dynamic_cc_library()

def thrift_library(name,
                   srcs=[],
                   deps=[],
                   optimize=[],
                   deprecated=False,
                   **kwargs):
    """thrift_library target. """
    thrift_library_target = ThriftLibrary(name,
                                          srcs,
                                          deps,
                                          optimize,
                                          deprecated,
                                          blade.blade,
                                          kwargs)
    blade.blade.register_target(thrift_library_target)

build_rules.register_function(thrift_library)
<commit_msg>code review follow up: support setting a cc_binary target as the thrift compiler<commit_after># Copyright (c) 2012 iQIYI Inc.
# Copyright (c) 2013 Tencent Inc.
# All rights reserved.
#
# Author: Jingxu Chen <chenjingxu@qiyi.com>
#         Feng Chen <chen3feng@gmail.com>
# Date:   October 13, 2012

"""
 A helper class to get the files generated from thrift IDL files.

"""

import os

import blade
import configparse
import console

import build_rules
import java_jar_target
import py_targets

from blade_util import var_to_list
from cc_targets import CcTarget
from thrift_helper import ThriftHelper

class ThriftLibrary(CcTarget):
    """A scons thrift library target subclass.

    This class is derived from CcTarget.

    """
    def __init__(self,
                 name,
                 srcs,
                 deps,
                 optimize,
                 deprecated,
                 blade,
                 kwargs):
        """Init method.

        Init the thrift target.

        """
        srcs = var_to_list(srcs)
        self._check_thrift_srcs_name(srcs)
        CcTarget.__init__(self,
                          name,
                          'thrift_library',
                          srcs,
                          deps,
                          '',
                          [], [], [], optimize, [], [],
                          blade,
                          kwargs)
        self.data['python_vars'] = []
        self.data['python_sources'] = []

        thrift_config = configparse.blade_config.get_config('thrift_config')
        thrift_lib = var_to_list(thrift_config['thrift_libs'])
        thrift_bin = thrift_config['thrift']
        if thrift_bin.startswith("//"):
            dkey = self._convert_string_to_target_helper(thrift_bin)
            if dkey not in self.expanded_deps:
                self.expanded_deps.append(dkey)
            if dkey not in self.deps:
                self.deps.append(dkey)

        # Hardcode deps rule to thrift libraries.
        self._add_hardcode_library(thrift_lib)

        # Link all the symbols by default
        self.data['link_all_symbols'] = True
        self.data['deprecated'] = deprecated
        self.data['java_sources_explict_dependency'] = []

        # For each thrift file initialize a ThriftHelper, which will be used
        # to get the source files generated from thrift file.
        self.thrift_helpers = {}
        for src in srcs:
            self.thrift_helpers[src] = ThriftHelper(
                    os.path.join(self.path, src))

    def _check_thrift_srcs_name(self, srcs):
        """_check_thrift_srcs_name.

        Checks whether the thrift file's name ends with 'thrift'.

        """
        error = 0
        for src in srcs:
            base_name = os.path.basename(src)
            pos = base_name.rfind('.')
            if pos == -1:
                console.error('invalid thrift file name %s' % src)
                error += 1
            file_suffix = base_name[pos + 1:]
            if file_suffix != 'thrift':
                console.error('invalid thrift file name %s' % src)
                error += 1
        if error > 0:
            console.error_exit('invalid thrift file names found.')

    def _thrift_gen_cpp_files(self, path, src):
        """_thrift_gen_cpp_files.

        Get the c++ files generated from thrift file.

        """

        return [self._target_file_path(path, f)
                for f in self.thrift_helpers[src].get_generated_cpp_files()]

    def _thrift_gen_py_files(self, path, src):
        """_thrift_gen_py_files.

        Get the python files generated from thrift file.

        """

        return [self._target_file_path(path, f)
                for f in self.thrift_helpers[src].get_generated_py_files()]

    def _thrift_gen_java_files(self, path, src):
        """_thrift_gen_java_files.

        Get the java files generated from thrift file.

        """

        return [self._target_file_path(path, f)
                for f in self.thrift_helpers[src].get_generated_java_files()]

    def _thrift_java_rules(self):
        """_thrift_java_rules.

        Generate scons rules for the java files from thrift file.

        """

        for src in self.srcs:
            src_path = os.path.join(self.path, src)
            thrift_java_src_files = self._thrift_gen_java_files(self.path,
                                                                src)

            self._write_rule('%s.ThriftJava(%s, "%s")' % (
                    self._env_name(),
                    str(thrift_java_src_files),
                    src_path))

            self.data['java_sources'] = (
                     os.path.dirname(thrift_java_src_files[0]),
                     os.path.join(self.build_path, self.path),
                     self.name)

            self.data['java_sources_explict_dependency'] += thrift_java_src_files

    def _thrift_python_rules(self):
        """_thrift_python_rules.

        Generate python files.

        """
        for src in self.srcs:
            src_path = os.path.join(self.path, src)
            thrift_py_src_files = self._thrift_gen_py_files(self.path, src)
            py_cmd_var = '%s_python' % self._generate_variable_name(
                    self.path, self.name)
            self._write_rule('%s = %s.ThriftPython(%s, "%s")' % (
                    py_cmd_var,
                    self._env_name(),
                    str(thrift_py_src_files),
                    src_path))
            self.data['python_vars'].append(py_cmd_var)
            self.data['python_sources'] += thrift_py_src_files

    def scons_rules(self):
        """scons_rules.

        It outputs the scons rules according to user options.

        """
        self._prepare_to_generate_rule()

        # Build java source according to its option
        env_name = self._env_name()

        self.options = self.blade.get_options()
        self.direct_targets = self.blade.get_direct_targets()

        if (getattr(self.options, 'generate_java', False) or
            self.data.get('generate_java') or
            self.key in self.direct_targets):
            self._thrift_java_rules()

        if (getattr(self.options, 'generate_python', False) or
            self.data.get('generate_python') or
            self.key in self.direct_targets):
            self._thrift_python_rules()

        self._setup_cc_flags()

        sources = []
        obj_names = []
        for src in self.srcs:
            thrift_cpp_files = self._thrift_gen_cpp_files(self.path, src)
            thrift_cpp_src_files = [f for f in thrift_cpp_files if f.endswith('.cpp')]

            self._write_rule('%s.Thrift(%s, "%s")' % (
                    env_name,
                    str(thrift_cpp_files),
                    os.path.join(self.path, src)))

            for thrift_cpp_src in thrift_cpp_src_files:
                obj_name = '%s_object' % self._generate_variable_name(
                    self.path, thrift_cpp_src)
                obj_names.append(obj_name)
                self._write_rule(
                    '%s = %s.SharedObject(target="%s" + top_env["OBJSUFFIX"], '
                    'source="%s")' % (obj_name,
                                      env_name,
                                      thrift_cpp_src,
                                      thrift_cpp_src))
                sources.append(thrift_cpp_src)

        self._write_rule('%s = [%s]' % (self._objs_name(), ','.join(obj_names)))
        self._write_rule('%s.Depends(%s, %s)' % (
                         env_name, self._objs_name(), sources))

        self._cc_library()
        options = self.blade.get_options()
        if (getattr(options, 'generate_dynamic', False) or
            self.data.get('build_dynamic')):
            self._dynamic_cc_library()

def thrift_library(name,
                   srcs=[],
                   deps=[],
                   optimize=[],
                   deprecated=False,
                   **kwargs):
    """thrift_library target. """
    thrift_library_target = ThriftLibrary(name,
                                          srcs,
                                          deps,
                                          optimize,
                                          deprecated,
                                          blade.blade,
                                          kwargs)
    blade.blade.register_target(thrift_library_target)

build_rules.register_function(thrift_library)
<|endoftext|>


<commit_before>#  BSD 3-Clause License
#
#  Copyright (c) 2012, the Sentry Team, see AUTHORS for more details
#  Copyright (c) 2019, Elasticsearch BV
#  All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are met:
#
#  * Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
#
#  * Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
#  * Neither the name of the copyright holder nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
#  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
#  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
#  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
#  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
#  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
#  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

from __future__ import absolute_import

import asyncio
import functools
from typing import Dict, Optional

import starlette
from starlette.requests import Request
from starlette.routing import Match, Mount
from starlette.types import ASGIApp, Message

import elasticapm
import elasticapm.instrumentation.control
from elasticapm.base import Client
from elasticapm.conf import constants
from elasticapm.contrib.asyncio.traces import set_context
from elasticapm.contrib.starlette.utils import get_body, get_data_from_request, get_data_from_response
from elasticapm.utils.disttracing import TraceParent
from elasticapm.utils.logging import get_logger

logger = get_logger("elasticapm.errors.client")

def make_apm_client(config: Optional[Dict] = None, client_cls=Client, **defaults) -> Client:
    """Builds ElasticAPM client.

    Args:
        config (dict): Dictionary of Client configuration. All keys must be uppercase. See `elasticapm.conf.Config`.
        client_cls (Client): Must be Client or its child.
        **defaults: Additional parameters for Client. See `elasticapm.base.Client`

    Returns:
        Client
    """
    if "framework_name" not in defaults:
        defaults["framework_name"] = "starlette"
        defaults["framework_version"] = starlette.__version__

    return client_cls(config, **defaults)

class ElasticAPM:
    """
    Starlette / FastAPI middleware for Elastic APM capturing.

    >>> apm = make_apm_client({
        >>> 'SERVICE_NAME': 'myapp',
        >>> 'DEBUG': True,
        >>> 'SERVER_URL': 'http://localhost:8200',
        >>> 'CAPTURE_HEADERS': True,
        >>> 'CAPTURE_BODY': 'all'
    >>> })

    >>> app.add_middleware(ElasticAPM, client=apm)

    Pass an arbitrary APP_NAME and SECRET_TOKEN::

    >>> elasticapm = ElasticAPM(app, service_name='myapp', secret_token='asdasdasd')

    Pass an explicit client (don't pass in additional options in this case)::

    >>> elasticapm = ElasticAPM(app, client=client)

    Capture an exception::

    >>> try:
    >>>     1 / 0
    >>> except ZeroDivisionError:
    >>>     elasticapm.capture_exception()

    Capture a message::

    >>> elasticapm.capture_message('hello, world!')
    """

    def __init__(self, app: ASGIApp, client: Optional[Client], **kwargs):
        """

        Args:
            app (ASGIApp): Starlette app
            client (Client): ElasticAPM Client
        """
        if client:
            self.client = client
        else:
            self.client = make_apm_client(**kwargs)

        if self.client.config.instrument and self.client.config.enabled:
            elasticapm.instrumentation.control.instrument()

        # If we ever make this a general-use ASGI middleware we should use
        # `asgiref.conpatibility.guarantee_single_callable(app)` here
        self.app = app

    async def __call__(self, scope, receive, send):
        """
        Args:
            scope: ASGI scope dictionary
            receive: receive awaitable callable
            send: send awaitable callable
        """

        @functools.wraps(send)
        async def wrapped_send(message):
            if message.get("type") == "http.response.start":
                await set_context(
                    lambda: get_data_from_response(message, self.client.config, constants.TRANSACTION), "response"
                )
                result = "HTTP {}xx".format(message["status"] // 100)
                elasticapm.set_transaction_result(result, override=False)
            await send(message)

        # When we consume the body from receive, we replace the streaming
        # mechanism with a mocked version -- this workaround came from
        # https://github.com/encode/starlette/issues/495#issuecomment-513138055
        body = b""
        while True:
            message = await receive()
            if not message:
                break
            if message["type"] == "http.request":
                b = message.get("body", b"")
                if b:
                    body += b
                if not message.get("more_body", False):
                    break
            if message["type"] == "http.disconnect":
                break

        async def _receive() -> Message:
            await asyncio.sleep(0)
            return {"type": "http.request", "body": body}

        request = Request(scope, receive=_receive)
        await self._request_started(request)

        try:
            await self.app(scope, _receive, wrapped_send)
            elasticapm.set_transaction_outcome(constants.OUTCOME.SUCCESS, override=False)
        except Exception:
            await self.capture_exception(
                context={"request": await get_data_from_request(request, self.client.config, constants.ERROR)}
            )
            elasticapm.set_transaction_result("HTTP 5xx", override=False)
            elasticapm.set_transaction_outcome(constants.OUTCOME.FAILURE, override=False)
            elasticapm.set_context({"status_code": 500}, "response")

            raise
        finally:
            self.client.end_transaction()

    async def capture_exception(self, *args, **kwargs):
        """Captures your exception.

        Args:
            *args:
            **kwargs:
        """
        self.client.capture_exception(*args, **kwargs)

    async def capture_message(self, *args, **kwargs):
        """Captures your message.

        Args:
            *args: Whatever
            **kwargs: Whatever
        """
        self.client.capture_message(*args, **kwargs)

    async def _request_started(self, request: Request):
        """Captures the begin of the request processing to APM.

        Args:
            request (Request)
        """
        # When we consume the body, we replace the streaming mechanism with
        # a mocked version -- this workaround came from
        # https://github.com/encode/starlette/issues/495#issuecomment-513138055
        # and we call the workaround here to make sure that regardless of
        # `capture_body` settings, we will have access to the body if we need it.
        if self.client.config.capture_body != "off":
            await get_body(request)

        if not self.client.should_ignore_url(request.url.path):
            trace_parent = TraceParent.from_headers(dict(request.headers))
            self.client.begin_transaction("request", trace_parent=trace_parent)

            await set_context(
                lambda: get_data_from_request(request, self.client.config, constants.TRANSACTION), "request"
            )
            transaction_name = self.get_route_name(request) or request.url.path
            elasticapm.set_transaction_name("{} {}".format(request.method, transaction_name), override=False)

    def get_route_name(self, request: Request) -> str:
        app = request.app
        scope = request.scope
        routes = app.routes
        route_name = self._get_route_name(scope, routes)

        # Starlette magically redirects requests if the path matches a route name with a trailing slash
        # appended or removed. To not spam the transaction names list, we do the same here and put these
        # redirects all in the same "redirect trailing slashes" transaction name
        if not route_name and app.router.redirect_slashes and scope["path"] != "/":
            redirect_scope = dict(scope)
            if scope["path"].endswith("/"):
                redirect_scope["path"] = scope["path"][:-1]
                trim = True
            else:
                redirect_scope["path"] = scope["path"] + "/"
                trim = False

            route_name = self._get_route_name(redirect_scope, routes)
            if route_name is not None:
                route_name = route_name + "/" if trim else route_name[:-1]
        return route_name

    def _get_route_name(self, scope, routes, route_name=None):
        for route in routes:
            match, child_scope = route.matches(scope)
            if match == Match.FULL:
                route_name = route.path
                child_scope = {**scope, **child_scope}
                if isinstance(route, Mount) and route.routes:
                    child_route_name = self._get_route_name(child_scope, route.routes, route_name)
                    if child_route_name is None:
                        route_name = None
                    else:
                        route_name += child_route_name
                return route_name
            elif match == Match.PARTIAL and route_name is None:
                route_name = route.path
<commit_msg>raise an exception for non-http scopes in the Starlette middleware (#1185)<commit_after>#  BSD 3-Clause License
#
#  Copyright (c) 2012, the Sentry Team, see AUTHORS for more details
#  Copyright (c) 2019, Elasticsearch BV
#  All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are met:
#
#  * Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
#
#  * Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
#  * Neither the name of the copyright holder nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
#  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
#  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
#  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
#  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
#  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
#  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

from __future__ import absolute_import

import asyncio
import functools
from typing import Dict, Optional

import starlette
from starlette.requests import Request
from starlette.routing import Match, Mount
from starlette.types import ASGIApp, Message

import elasticapm
import elasticapm.instrumentation.control
from elasticapm.base import Client
from elasticapm.conf import constants
from elasticapm.contrib.asyncio.traces import set_context
from elasticapm.contrib.starlette.utils import get_body, get_data_from_request, get_data_from_response
from elasticapm.utils.disttracing import TraceParent
from elasticapm.utils.logging import get_logger

logger = get_logger("elasticapm.errors.client")

def make_apm_client(config: Optional[Dict] = None, client_cls=Client, **defaults) -> Client:
    """Builds ElasticAPM client.

    Args:
        config (dict): Dictionary of Client configuration. All keys must be uppercase. See `elasticapm.conf.Config`.
        client_cls (Client): Must be Client or its child.
        **defaults: Additional parameters for Client. See `elasticapm.base.Client`

    Returns:
        Client
    """
    if "framework_name" not in defaults:
        defaults["framework_name"] = "starlette"
        defaults["framework_version"] = starlette.__version__

    return client_cls(config, **defaults)

class ElasticAPM:
    """
    Starlette / FastAPI middleware for Elastic APM capturing.

    >>> apm = make_apm_client({
        >>> 'SERVICE_NAME': 'myapp',
        >>> 'DEBUG': True,
        >>> 'SERVER_URL': 'http://localhost:8200',
        >>> 'CAPTURE_HEADERS': True,
        >>> 'CAPTURE_BODY': 'all'
    >>> })

    >>> app.add_middleware(ElasticAPM, client=apm)

    Pass an arbitrary APP_NAME and SECRET_TOKEN::

    >>> elasticapm = ElasticAPM(app, service_name='myapp', secret_token='asdasdasd')

    Pass an explicit client (don't pass in additional options in this case)::

    >>> elasticapm = ElasticAPM(app, client=client)

    Capture an exception::

    >>> try:
    >>>     1 / 0
    >>> except ZeroDivisionError:
    >>>     elasticapm.capture_exception()

    Capture a message::

    >>> elasticapm.capture_message('hello, world!')
    """

    def __init__(self, app: ASGIApp, client: Optional[Client], **kwargs):
        """

        Args:
            app (ASGIApp): Starlette app
            client (Client): ElasticAPM Client
        """
        if client:
            self.client = client
        else:
            self.client = make_apm_client(**kwargs)

        if self.client.config.instrument and self.client.config.enabled:
            elasticapm.instrumentation.control.instrument()

        # If we ever make this a general-use ASGI middleware we should use
        # `asgiref.conpatibility.guarantee_single_callable(app)` here
        self.app = app

    async def __call__(self, scope, receive, send):
        """
        Args:
            scope: ASGI scope dictionary
            receive: receive awaitable callable
            send: send awaitable callable
        """
        # we only handle the http scope, raise an exception for anything else
        # see https://www.uvicorn.org/#the-asgi-interface
        assert scope["type"] == "http"

        @functools.wraps(send)
        async def wrapped_send(message):
            if message.get("type") == "http.response.start":
                await set_context(
                    lambda: get_data_from_response(message, self.client.config, constants.TRANSACTION), "response"
                )
                result = "HTTP {}xx".format(message["status"] // 100)
                elasticapm.set_transaction_result(result, override=False)
            await send(message)

        # When we consume the body from receive, we replace the streaming
        # mechanism with a mocked version -- this workaround came from
        # https://github.com/encode/starlette/issues/495#issuecomment-513138055
        body = b""
        while True:
            message = await receive()
            if not message:
                break
            if message["type"] == "http.request":
                b = message.get("body", b"")
                if b:
                    body += b
                if not message.get("more_body", False):
                    break
            if message["type"] == "http.disconnect":
                break

        async def _receive() -> Message:
            await asyncio.sleep(0)
            return {"type": "http.request", "body": body}

        request = Request(scope, receive=_receive)
        await self._request_started(request)

        try:
            await self.app(scope, _receive, wrapped_send)
            elasticapm.set_transaction_outcome(constants.OUTCOME.SUCCESS, override=False)
        except Exception:
            await self.capture_exception(
                context={"request": await get_data_from_request(request, self.client.config, constants.ERROR)}
            )
            elasticapm.set_transaction_result("HTTP 5xx", override=False)
            elasticapm.set_transaction_outcome(constants.OUTCOME.FAILURE, override=False)
            elasticapm.set_context({"status_code": 500}, "response")

            raise
        finally:
            self.client.end_transaction()

    async def capture_exception(self, *args, **kwargs):
        """Captures your exception.

        Args:
            *args:
            **kwargs:
        """
        self.client.capture_exception(*args, **kwargs)

    async def capture_message(self, *args, **kwargs):
        """Captures your message.

        Args:
            *args: Whatever
            **kwargs: Whatever
        """
        self.client.capture_message(*args, **kwargs)

    async def _request_started(self, request: Request):
        """Captures the begin of the request processing to APM.

        Args:
            request (Request)
        """
        # When we consume the body, we replace the streaming mechanism with
        # a mocked version -- this workaround came from
        # https://github.com/encode/starlette/issues/495#issuecomment-513138055
        # and we call the workaround here to make sure that regardless of
        # `capture_body` settings, we will have access to the body if we need it.
        if self.client.config.capture_body != "off":
            await get_body(request)

        if not self.client.should_ignore_url(request.url.path):
            trace_parent = TraceParent.from_headers(dict(request.headers))
            self.client.begin_transaction("request", trace_parent=trace_parent)

            await set_context(
                lambda: get_data_from_request(request, self.client.config, constants.TRANSACTION), "request"
            )
            transaction_name = self.get_route_name(request) or request.url.path
            elasticapm.set_transaction_name("{} {}".format(request.method, transaction_name), override=False)

    def get_route_name(self, request: Request) -> str:
        app = request.app
        scope = request.scope
        routes = app.routes
        route_name = self._get_route_name(scope, routes)

        # Starlette magically redirects requests if the path matches a route name with a trailing slash
        # appended or removed. To not spam the transaction names list, we do the same here and put these
        # redirects all in the same "redirect trailing slashes" transaction name
        if not route_name and app.router.redirect_slashes and scope["path"] != "/":
            redirect_scope = dict(scope)
            if scope["path"].endswith("/"):
                redirect_scope["path"] = scope["path"][:-1]
                trim = True
            else:
                redirect_scope["path"] = scope["path"] + "/"
                trim = False

            route_name = self._get_route_name(redirect_scope, routes)
            if route_name is not None:
                route_name = route_name + "/" if trim else route_name[:-1]
        return route_name

    def _get_route_name(self, scope, routes, route_name=None):
        for route in routes:
            match, child_scope = route.matches(scope)
            if match == Match.FULL:
                route_name = route.path
                child_scope = {**scope, **child_scope}
                if isinstance(route, Mount) and route.routes:
                    child_route_name = self._get_route_name(child_scope, route.routes, route_name)
                    if child_route_name is None:
                        route_name = None
                    else:
                        route_name += child_route_name
                return route_name
            elif match == Match.PARTIAL and route_name is None:
                route_name = route.path
<|endoftext|>


<commit_before>import time
import urllib3

from .base import Connection
from ..exceptions import ConnectionError
from ..compat import urlencode

class Urllib3HttpConnection(Connection):
    """
    Default connection class using the `urllib3` library and the http protocol.

    :arg http_auth: optional http auth information as either ':' separated
        string or a tuple
    :arg use_ssl: use ssl for the connection if `True`
    :arg maxsize: the maximum number of connections which will be kept open to
        this host.
    """
    def __init__(self, host='localhost', port=9200, http_auth=None, use_ssl=False, maxsize=10, **kwargs):
        super(Urllib3HttpConnection, self).__init__(host=host, port=port, **kwargs)
        self.headers = {}
        if http_auth is not None:
            if isinstance(http_auth, (tuple, list)):
                http_auth = ':'.join(http_auth)
            self.headers = urllib3.make_headers(basic_auth=http_auth)

        pool_class = urllib3.HTTPConnectionPool
        if use_ssl:
            pool_class = urllib3.HTTPSConnectionPool

        self.pool = pool_class(host, port=port, timeout=self.timeout, maxsize=maxsize)

    def perform_request(self, method, url, params=None, body=None, timeout=None, ignore=()):
        url = self.url_prefix + url
        if params:
            url = '%s?%s' % (url, urlencode(params or {}))
        full_url = self.host + url

        start = time.time()
        try:
            kw = {}
            if timeout:
                kw['timeout'] = timeout
            response = self.pool.urlopen(method, url, body, retries=False, headers=self.headers, **kw)
            duration = time.time() - start
            raw_data = response.data.decode('utf-8')
        except Exception as e:
            self.log_request_fail(method, full_url, body, time.time() - start, exception=e)
            raise ConnectionError('N/A', str(e), e)

        if not (200 <= response.status < 300) and response.status not in ignore:
            self.log_request_fail(method, url, body, duration, response.status)
            self._raise_error(response.status, raw_data)

        self.log_request_success(method, full_url, url, body, response.status,
            raw_data, duration)

        return response.status, response.getheaders(), raw_data

<commit_msg>Make sure url is a string, not unicode in python 2<commit_after>import time
import urllib3

from .base import Connection
from ..exceptions import ConnectionError
from ..compat import urlencode

class Urllib3HttpConnection(Connection):
    """
    Default connection class using the `urllib3` library and the http protocol.

    :arg http_auth: optional http auth information as either ':' separated
        string or a tuple
    :arg use_ssl: use ssl for the connection if `True`
    :arg maxsize: the maximum number of connections which will be kept open to
        this host.
    """
    def __init__(self, host='localhost', port=9200, http_auth=None, use_ssl=False, maxsize=10, **kwargs):
        super(Urllib3HttpConnection, self).__init__(host=host, port=port, **kwargs)
        self.headers = {}
        if http_auth is not None:
            if isinstance(http_auth, (tuple, list)):
                http_auth = ':'.join(http_auth)
            self.headers = urllib3.make_headers(basic_auth=http_auth)

        pool_class = urllib3.HTTPConnectionPool
        if use_ssl:
            pool_class = urllib3.HTTPSConnectionPool

        self.pool = pool_class(host, port=port, timeout=self.timeout, maxsize=maxsize)

    def perform_request(self, method, url, params=None, body=None, timeout=None, ignore=()):
        url = self.url_prefix + url
        if params:
            url = '%s?%s' % (url, urlencode(params or {}))
        full_url = self.host + url

        start = time.time()
        try:
            kw = {}
            if timeout:
                kw['timeout'] = timeout
            if not isinstance(url, str):
                url = url.encode('utf-8')
            response = self.pool.urlopen(method, url, body, retries=False, headers=self.headers, **kw)
            duration = time.time() - start
            raw_data = response.data.decode('utf-8')
        except Exception as e:
            self.log_request_fail(method, full_url, body, time.time() - start, exception=e)
            raise ConnectionError('N/A', str(e), e)

        if not (200 <= response.status < 300) and response.status not in ignore:
            self.log_request_fail(method, url, body, duration, response.status)
            self._raise_error(response.status, raw_data)

        self.log_request_success(method, full_url, url, body, response.status,
            raw_data, duration)

        return response.status, response.getheaders(), raw_data

<|endoftext|>


<commit_before>'''
Created on 29.11.2013

@author: marscher, noe
'''

import numpy as np

import scipy
from scipy.sparse import issparse
from scipy.sparse.sputils import isdense

def time_correlation_direct(P, pi, obs1, obs2=None, time=1):
    r"""Compute time-correlation of obs1, or time-cross-correlation with obs2.

    The time-correlation at time=k is computed by the matrix-vector expression:
    cor(k) = obs1' diag(pi) P^k obs2

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
        Transition matrix
    obs1 : ndarray, shape=(n)
        Vector representing observable 1 on discrete states
    obs2 : ndarray, shape=(n)
        Vector representing observable 2 on discrete states. If not given,
        the autocorrelation of obs1 will be computed
    pi : ndarray, shape=(n)
        stationary distribution vector. Will be computed if not given
    time : int or array like
        time point(s) at which the (auto)correlation will be evaluated

    Returns
    -------

    """
    if not (type( time ) == int):
        raise TypeError("time is not an integer: "+str(time))
    # multiply element-wise obs1 and pi. this is obs1' diag(pi)
    l = np.multiply(obs1, pi)
    # raise transition matrix to power of time
    Pk = np.linalg.matrix_power(P, time)
    # compute product P^k obs2
    r = np.dot(Pk, obs2)
    # return result
    return np.dot(l,r)

def time_correlations_direct(P, pi, obs1, obs2=None, times=[1]):
    r"""Compute time-correlation of obs1, or time-cross-correlation with obs2.

    The time-correlation at time=k is computed by the matrix-vector expression:
    cor(k) = obs1' diag(pi) P^k obs2

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
        Transition matrix
    obs1 : ndarray, shape=(n)
        Vector representing observable 1 on discrete states
    obs2 : ndarray, shape=(n)
        Vector representing observable 2 on discrete states. If not given,
        the autocorrelation of obs1 will be computed
    pi : ndarray, shape=(n)
        stationary distribution vector. Will be computed if not given
    times : array-like, shape(n_t)
        Vector of time points at which the (auto)correlation will be evaluated

    Returns
    -------

    """
    n_t = len(times)
    f = np.zeros((n_t))
    for i in range(n_t):
        f[i] = time_correlation_direct(P, pi, obs1, obs2, times[i])
    return f

def time_relaxation_direct(P, p0, obs, time=1):
    r"""Compute time-correlation of obs1, or time-cross-correlation with obs2.

    The time-correlation at time=k is computed by the matrix-vector expression:
    cor(k) = obs1' diag(pi) P^k obs2

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
        Transition matrix
    p0 : ndarray, shape=(n)
        initial distribution
    obs : ndarray, shape=(n)
        Vector representing observable on discrete states.
    time : int or array like
        time point(s) at which the (auto)correlation will be evaluated

    Returns
    -------

    """
    if not (type( time ) == int):
        raise TypeError("time is not an integer: "+str(time))
    # raise transition matrix to power of time
    # TODO: cache this some how if called from time_correlations_direct
    Pk = np.linalg.matrix_power(P, time)
    # propagate time
    pk = np.dot(p0, Pk)
    # return result
    return np.dot(pk,obs)

def time_relaxations_direct(P, p0, obs, times):
    r"""Compute time-correlation of obs1, or time-cross-correlation with obs2.

    The time-correlation at time=k is computed by the matrix-vector expression:
    cor(k) = obs1' diag(pi) P^k obs2

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
        Transition matrix
    p0 : ndarray, shape=(n)
        initial distribution
    obs : ndarray, shape=(n)
        Vector representing observable on discrete states.
    times : array-like, shape(n_t)
        Vector of time points at which the (auto)correlation will be evaluated

    Returns
    -------

    """
    n_t = len(times)
    f = np.zeros((n_t))
    for i in range(n_t):
        f[i] = time_relaxation_direct(P, p0, obs, times[i])
    return f<commit_msg>added todo<commit_after>'''
Created on 29.11.2013

@author: marscher, noe
'''

import numpy as np

import scipy
from scipy.sparse import issparse
from scipy.sparse.sputils import isdense

def time_correlation_direct(P, pi, obs1, obs2=None, time=1):
    r"""Compute time-correlation of obs1, or time-cross-correlation with obs2.

    The time-correlation at time=k is computed by the matrix-vector expression:
    cor(k) = obs1' diag(pi) P^k obs2

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
        Transition matrix
    obs1 : ndarray, shape=(n)
        Vector representing observable 1 on discrete states
    obs2 : ndarray, shape=(n)
        Vector representing observable 2 on discrete states. If not given,
        the autocorrelation of obs1 will be computed
    pi : ndarray, shape=(n)
        stationary distribution vector. Will be computed if not given
    time : int or array like
        time point(s) at which the (auto)correlation will be evaluated

    Returns
    -------

    """
    if not (type( time ) == int):
        raise TypeError("time is not an integer: "+str(time))
    # multiply element-wise obs1 and pi. this is obs1' diag(pi)
    l = np.multiply(obs1, pi)
    # raise transition matrix to power of time
    Pk = np.linalg.matrix_power(P, time)
    # compute product P^k obs2
    r = np.dot(Pk, obs2)
    # return result
    return np.dot(l,r)

# TODO: efficient impl (avoid P^k via matrix vector product)
def time_correlations_direct(P, pi, obs1, obs2=None, times=[1]):
    r"""Compute time-correlation of obs1, or time-cross-correlation with obs2.

    The time-correlation at time=k is computed by the matrix-vector expression:
    cor(k) = obs1' diag(pi) P^k obs2

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
        Transition matrix
    obs1 : ndarray, shape=(n)
        Vector representing observable 1 on discrete states
    obs2 : ndarray, shape=(n)
        Vector representing observable 2 on discrete states. If not given,
        the autocorrelation of obs1 will be computed
    pi : ndarray, shape=(n)
        stationary distribution vector. Will be computed if not given
    times : array-like, shape(n_t)
        Vector of time points at which the (auto)correlation will be evaluated

    Returns
    -------

    """
    n_t = len(times)
    f = np.zeros((n_t))
    for i in range(n_t):
        f[i] = time_correlation_direct(P, pi, obs1, obs2, times[i])
    return f

def time_relaxation_direct(P, p0, obs, time=1):
    r"""Compute time-correlation of obs1, or time-cross-correlation with obs2.

    The time-correlation at time=k is computed by the matrix-vector expression:
    cor(k) = obs1' diag(pi) P^k obs2

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
        Transition matrix
    p0 : ndarray, shape=(n)
        initial distribution
    obs : ndarray, shape=(n)
        Vector representing observable on discrete states.
    time : int or array like
        time point(s) at which the (auto)correlation will be evaluated

    Returns
    -------

    """
    if not (type( time ) == int):
        raise TypeError("time is not an integer: "+str(time))
    # raise transition matrix to power of time
    # TODO: cache this some how if called from time_correlations_direct
    Pk = np.linalg.matrix_power(P, time)
    # propagate time
    pk = np.dot(p0, Pk)
    # return result
    return np.dot(pk,obs)

def time_relaxations_direct(P, p0, obs, times):
    r"""Compute time-correlation of obs1, or time-cross-correlation with obs2.

    The time-correlation at time=k is computed by the matrix-vector expression:
    cor(k) = obs1' diag(pi) P^k obs2

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
        Transition matrix
    p0 : ndarray, shape=(n)
        initial distribution
    obs : ndarray, shape=(n)
        Vector representing observable on discrete states.
    times : array-like, shape(n_t)
        Vector of time points at which the (auto)correlation will be evaluated

    Returns
    -------

    """
    n_t = len(times)
    f = np.zeros((n_t))
    for i in range(n_t):
        f[i] = time_relaxation_direct(P, p0, obs, times[i])
    return f<|endoftext|>


<commit_before>"""
Script to optimize Basket cell current injection response
"""

from optimalneuron import optimizers
from optimalneuron import evaluators
from optimalneuron import controllers

#first off we need to make an evaluator,

parameters = ['axon_gbar_na','axon_gbar_kv','axon_gbar_kv3','soma_gbar_na','soma_gbar_kv','soma_gbar_kv3']

#manual_vals=[50,50,2000,70,70,5,0.1,28.0,49.0,-73.0,23.0] # EXAMPLE - how to set a seed
min_constraints = [0,0,0,0,0,0]
max_constraints = [0,0,0,0,0,0]

analysis_var={'peak_delta':0,'baseline':0,'dvdt_threshold':2}

weights={'average_minimum': 5, 'spike_frequency_adaptation': 0, 'trough_phase_adaptation': 0, 'mean_spike_frequency': 0, 'average_maximum': 0, 'trough_decay_exponent': 5, 'interspike_time_covar': 0, 'min_peak_no': 1, 'spike_width_adaptation': 0, 'max_peak_no': 5, 'first_spike_time': 0, 'peak_decay_exponent': 0,'spike_broadening':5,'pprd_error':5}

targets={'average_minimum': -38.839498793604541, 'spike_frequency_adaptation': 0.019619800882894008, 'trough_phase_adaptation': 0.005225712358530369, 'mean_spike_frequency': 47.353760445682454, 'average_maximum': 29.320249266525668, 'trough_decay_exponent': 0.11282542321257279, 'interspike_time_covar': 0.042610190921388166, 'min_peak_no': 34, 'spike_broadening': 0.81838856772318913, 'spike_width_adaptation': 0.0095057081186080035, 'max_peak_no': 35, 'first_spike_time': 164.0, 'peak_decay_exponent': -0.04596529555434687,'pptd_error':0}

#using automatic target evaluation:
#what we should do next is separate out the controller and pass it as an object to the evaluator-
#we really need to think about this deeply, separating the nrnproject logic from the neuronoptimizer
#may be quite hard

#remember, under my new design ideas evaluator is decoupled from the implementation as this is a job for controller
import os
project_directory = os.path.pardir
database_directory = os.path.join(os.path.pardir,'sims/output.sqlite')

my_controller=controllers.NrnProject(project_directory,database_directory)

my_evaluator=evaluators.IClampEvaluator(controller=my_controller,
                                        analysis_start_time=1,
                                        analysis_end_time=4000,
                                        target_data_path='../experimental_data/100pA_1.csv',
                                        parameters=parameters,
                                        analysis_var=analysis_var,
                                        weights=weights,
                                        targets=targets,
                                        automatic=True)

my_optimizer=optimizers.CustomOptimizerA(max_constraints,min_constraints,my_evaluator,
                                  population_size=1000,
                                  max_evaluations=10000,
                                  num_selected=20,
                                  num_offspring=100,
                                  num_elites=1,
                                  seeds=None)
my_optimizer.optimize()
<commit_msg>small changes to Optimization script<commit_after>"""
Script to optimize Basket cell current injection response
"""

from optimalneuron import optimizers
from optimalneuron import evaluators
from optimalneuron import controllers

#first off we need to make an evaluator,

parameters = ['axon_gbar_na','axon_gbar_kv','axon_gbar_kv3','soma_gbar_na','soma_gbar_kv','soma_gbar_kv3']

#manual_vals=[50,50,2000,70,70,5,0.1,28.0,49.0,-73.0,23.0] # EXAMPLE - how to set a seed
min_constraints = [0,0,0,0,0,0]
max_constraints = [0,0,0,0,0,0]

analysis_var={'peak_delta':0,'baseline':0,'dvdt_threshold':2}

weights={'average_minimum': 5, 'spike_frequency_adaptation': 0, 'trough_phase_adaptation': 0, 'mean_spike_frequency': 0, 'average_maximum': 0, 'trough_decay_exponent': 5, 'interspike_time_covar': 0, 'min_peak_no': 1, 'spike_width_adaptation': 0, 'max_peak_no': 5, 'first_spike_time': 0, 'peak_decay_exponent': 0,'spike_broadening':5,'pprd_error':5}

targets={'average_minimum': -38.839498793604541, 'spike_frequency_adaptation': 0.019619800882894008, 'trough_phase_adaptation': 0.005225712358530369, 'mean_spike_frequency': 47.353760445682454, 'average_maximum': 29.320249266525668, 'trough_decay_exponent': 0.11282542321257279, 'interspike_time_covar': 0.042610190921388166, 'min_peak_no': 34, 'spike_broadening': 0.81838856772318913, 'spike_width_adaptation': 0.0095057081186080035, 'max_peak_no': 35, 'first_spike_time': 164.0, 'peak_decay_exponent': -0.04596529555434687,'pptd_error':0}

#using automatic target evaluation:
#what we should do next is separate out the controller and pass it as an object to the evaluator-
#we really need to think about this deeply, separating the nrnproject logic from the neuronoptimizer
#may be quite hard

#remember, under my new design ideas evaluator is decoupled from the implementation as this is a job for controller
import os
project_directory = os.path.pardir
database_directory = os.path.join(os.path.pardir,'sims/output.sqlite')

my_controller=controllers.NrnProject(project_directory,database_directory)

my_evaluator=evaluators.IClampEvaluator(controller=my_controller,
                                        analysis_start_time=1,
                                        analysis_end_time=4000,
                                        target_data_path='../experimental_data/100pA_1.csv',
                                        parameters=parameters,
                                        analysis_var=analysis_var,
                                        weights=weights,
                                        targets=targets,
                                        automatic=True)

my_optimizer=optimizers.CustomOptimizerA(max_constraints,min_constraints,my_evaluator,
                                  population_size=10,
                                  max_evaluations=100,
                                  num_selected=20,
                                  num_offspring=100,
                                  num_elites=1,
                                  seeds=None)
my_optimizer.optimize()
<|endoftext|>


<commit_before>from unittest import skipUnless
from django import VERSION
from django.db import connection
from django.test import TestCase, TransactionTestCase
from django.db import IntegrityError
from versions_tests.models import ChainStore, Color
from versions.util.postgresql import get_uuid_like_indexes_on_table

AT_LEAST_17 = VERSION[:2] >= (1, 7)

@skipUnless(AT_LEAST_17 and connection.vendor == 'postgresql', "Postgresql-specific test")
class PostgresqlVersionUniqueTests(TransactionTestCase):
    def setUp(self):
        self.red = Color.objects.create(name='red')
        self.green = Color.objects.create(name='green')
        self.black = Color.objects.create(name='black')
        self.yellow = Color.objects.create(name='yellow')

        # - only one store with the same name and subchain_id can exist in a single city
        # - no two stores can share the same door_frame_color and door_color
        store = {
            'subchain_id': 1,
            'city': 'Santa Barbara',
            'name': 'Barbara style',
            'opening_hours': '9-9 everyday',
            'door_frame_color': self.red,
            'door_color': self.black,
        }

        self.sb1 = ChainStore.objects.create(**store)

    def test_version_unique(self):

        # It should not be possible to create another store with the same name, city, and subchain_id
        with self.assertRaises(IntegrityError):
            sb2 = ChainStore.objects.create(
                subchain_id = self.sb1.subchain_id,
                city = self.sb1.city,
                name = self.sb1.name,
                door_frame_color = self.sb1.door_frame_color,
                door_color = self.green
            )

        # It should not be possible to create another store with the same door and door_frame color
        with self.assertRaises(IntegrityError):
            sb3 = ChainStore.objects.create(
                subchain_id = self.sb1.subchain_id,
                city = self.sb1.city,
                name = "Bearded Bob's style",
                door_frame_color = self.sb1.door_frame_color,
                door_color = self.sb1.door_color
            )

        # It should be possible to create objects as long as they follow the unique constraints, though:
        sb4 = ChainStore.objects.create(
            subchain_id = self.sb1.subchain_id,
            city = self.sb1.city,
            name = "Bearded Bob's style",
            door_frame_color = self.sb1.door_frame_color,
            door_color = self.green
        )

        sb5 = ChainStore.objects.create(
            subchain_id = sb4.subchain_id + 1,
            city = sb4.city,
            name = sb4.name,
            door_frame_color = sb4.door_frame_color,
            door_color = self.yellow
        )

        # If a version is soft-deleted, it should be possible to create a new object with the
        # value of that old version
        sb4.delete()
        sb6 = ChainStore.objects.create(
            subchain_id = sb4.subchain_id,
            city = sb4.city,
            name = sb4.name,
            door_frame_color = sb4.door_frame_color,
            door_color = sb4.door_color
        )

class PostgresqlUuidLikeIndexesTest(TestCase):
    def test_no_like_indexes_on_uuid_columns(self):
        # Django creates like indexes on char columns.  In Django 1.7.x and below, there is no
        # support for native uuid columns, so CleanerVersion uses a CharField to store the
        # uuid values.  For postgresql, Django creates special indexes for char fields so that
        # like searches (e.g. WHERE foo like '%bar') are fast.
        # Those indexes are not going to be used in our case, and extra indexes will slow down
        # updates and inserts.  So, they should have been removed by the post_migrate handler in
        # versions_tests.apps.VersionsTestsConfig.ready.
        self.assertEqual(0, len(get_uuid_like_indexes_on_table(ChainStore)))
<commit_msg>restrict postgresql utils test to postgresql + django 1.7+<commit_after>from unittest import skipUnless
from django import VERSION
from django.db import connection
from django.test import TestCase, TransactionTestCase
from django.db import IntegrityError
from versions_tests.models import ChainStore, Color
from versions.util.postgresql import get_uuid_like_indexes_on_table

AT_LEAST_17 = VERSION[:2] >= (1, 7)

@skipUnless(AT_LEAST_17 and connection.vendor == 'postgresql', "Postgresql-specific test")
class PostgresqlVersionUniqueTests(TransactionTestCase):
    def setUp(self):
        self.red = Color.objects.create(name='red')
        self.green = Color.objects.create(name='green')
        self.black = Color.objects.create(name='black')
        self.yellow = Color.objects.create(name='yellow')

        # - only one store with the same name and subchain_id can exist in a single city
        # - no two stores can share the same door_frame_color and door_color
        store = {
            'subchain_id': 1,
            'city': 'Santa Barbara',
            'name': 'Barbara style',
            'opening_hours': '9-9 everyday',
            'door_frame_color': self.red,
            'door_color': self.black,
        }

        self.sb1 = ChainStore.objects.create(**store)

    def test_version_unique(self):

        # It should not be possible to create another store with the same name, city, and subchain_id
        with self.assertRaises(IntegrityError):
            sb2 = ChainStore.objects.create(
                subchain_id = self.sb1.subchain_id,
                city = self.sb1.city,
                name = self.sb1.name,
                door_frame_color = self.sb1.door_frame_color,
                door_color = self.green
            )

        # It should not be possible to create another store with the same door and door_frame color
        with self.assertRaises(IntegrityError):
            sb3 = ChainStore.objects.create(
                subchain_id = self.sb1.subchain_id,
                city = self.sb1.city,
                name = "Bearded Bob's style",
                door_frame_color = self.sb1.door_frame_color,
                door_color = self.sb1.door_color
            )

        # It should be possible to create objects as long as they follow the unique constraints, though:
        sb4 = ChainStore.objects.create(
            subchain_id = self.sb1.subchain_id,
            city = self.sb1.city,
            name = "Bearded Bob's style",
            door_frame_color = self.sb1.door_frame_color,
            door_color = self.green
        )

        sb5 = ChainStore.objects.create(
            subchain_id = sb4.subchain_id + 1,
            city = sb4.city,
            name = sb4.name,
            door_frame_color = sb4.door_frame_color,
            door_color = self.yellow
        )

        # If a version is soft-deleted, it should be possible to create a new object with the
        # value of that old version
        sb4.delete()
        sb6 = ChainStore.objects.create(
            subchain_id = sb4.subchain_id,
            city = sb4.city,
            name = sb4.name,
            door_frame_color = sb4.door_frame_color,
            door_color = sb4.door_color
        )

@skipUnless(AT_LEAST_17 and connection.vendor == 'postgresql', "Postgresql-specific test")
class PostgresqlUuidLikeIndexesTest(TestCase):
    def test_no_like_indexes_on_uuid_columns(self):
        # Django creates like indexes on char columns.  In Django 1.7.x and below, there is no
        # support for native uuid columns, so CleanerVersion uses a CharField to store the
        # uuid values.  For postgresql, Django creates special indexes for char fields so that
        # like searches (e.g. WHERE foo like '%bar') are fast.
        # Those indexes are not going to be used in our case, and extra indexes will slow down
        # updates and inserts.  So, they should have been removed by the post_migrate handler in
        # versions_tests.apps.VersionsTestsConfig.ready.
        self.assertEqual(0, len(get_uuid_like_indexes_on_table(ChainStore)))
<|endoftext|>


<commit_before>import unittest

import functools
import itertools
import mock
import numpy
import operator
import six

import chainer
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition
from chainer.utils import conv

def pooling_patches(dims, ksize, stride, pad):
    """Return tuples of slices that indicate pooling patches."""
    # Left-top indeces of each pooling patch.
    xss = itertools.product(
        *[six.moves.range(-p, d + p - k + 1, s)
          for (d, k, s, p) in six.moves.zip(dims, ksize, stride, pad)])
    # Tuple of slices for pooling patches.
    return [tuple(slice(max(x, 0), min(x + k, d))
                  for (x, d, k) in six.moves.zip(xs, dims, ksize))
            for xs in xss]

@testing.parameterize(*testing.product({
    'dims': [(4,), (4, 3), (4, 3, 2)],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
class TestAveragePoolingND(unittest.TestCase):

    def setUp(self):
        self.ndim = len(self.dims)
        self.ksize = (3,) * self.ndim
        self.stride = (2,) * self.ndim
        self.pad = (1,) * self.ndim

        x_shape = (2, 3) + self.dims
        self.x = numpy.random.uniform(-1, 1, x_shape).astype(self.dtype)

        outs = tuple(conv.get_conv_outsize(d, k, s, p, False)
                     for (d, k, s, p) in six.moves.zip(
                         self.dims, self.ksize, self.stride, self.pad))
        gy_shape = (2, 3) + outs
        self.gy = numpy.random.uniform(-1, 1, gy_shape).astype(self.dtype)

        self.check_forward_options = {}
        self.check_backward_options = {'eps': 1e-2}
        if self.dtype == numpy.float16:
            self.check_forward_options = {'atol': 5e-4, 'rtol': 5e-3}
            self.check_backward_options = {
                'eps': 1e-1, 'atol': 5e-3, 'rtol': 5e-2}

    def check_forward(self, x_data, use_cudnn=True):
        dims = self.dims
        ksize = self.ksize
        stride = self.stride
        pad = self.pad
        x = chainer.Variable(x_data)
        y = functions.average_pooling_nd(x, ksize, stride, pad,
                                         use_cudnn=use_cudnn)
        self.assertEqual(y.data.dtype, self.dtype)
        y_data = cuda.to_cpu(y.data)

        self.assertEqual(self.gy.shape, y_data.shape)
        for k in six.moves.range(2):
            for c in six.moves.range(3):
                x = self.x[k, c]
                size = functools.reduce(operator.mul, ksize)
                expect = numpy.array([x[idx].sum() for idx in pooling_patches(
                    dims, ksize, stride, pad)])
                expect = expect.reshape(y_data.shape[2:]) / size
                testing.assert_allclose(
                    expect, y_data[k, c], **self.check_forward_options)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.cudnn
    @condition.retry(3)
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x))

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu_no_cudnn(self):
        self.check_forward(cuda.to_gpu(self.x), False)

    def check_backward(self, x_data, y_grad, use_cudnn=True):
        gradient_check.check_backward(
            functions.AveragePoolingND(
                self.ndim, self.ksize, self.stride, self.pad,
                use_cudnn=use_cudnn),
            x_data, y_grad, **self.check_backward_options)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.cudnn
    @condition.retry(3)
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu_no_cudnn(self):
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy), False)

@testing.parameterize(*testing.product({
    'dims': [(4, 3, 2), (3, 2), (2,)],
    'use_cudnn': [True, False],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
@attr.cudnn
class TestAveragePoolingNDCudnnCall(unittest.TestCase):

    def setUp(self):
        self.ndim = len(self.dims)
        self.ksize = (3,) * self.ndim
        self.stride = (2,) * self.ndim
        self.pad = (1,) * self.ndim
        x_shape = (2, 3) + self.dims
        self.x = cuda.cupy.arange(functools.reduce(operator.mul, x_shape),
                                  dtype=self.dtype).reshape(x_shape)
        gy_shape = (2, 3) + tuple(
            conv.get_conv_outsize(d, k, s, p)
            for (d, k, s, p)
            in six.moves.zip(self.dims, self.ksize, self.stride, self.pad))
        self.gy = cuda.cupy.random.uniform(-1, 1, gy_shape).astype(self.dtype)

    def forward(self):
        x = chainer.Variable(self.x)
        return functions.average_pooling_nd(
            x, self.ksize, self.stride, self.pad, use_cudnn=self.use_cudnn)

    @unittest.skipIf(cuda.cudnn_enabled and
                     cuda.cudnn.cudnn.getVersion() < 3000,
                     'Only cudnn ver>=3 supports average-pooling-nd')
    def test_call_cudnn_forward(self):
        with mock.patch('cupy.cudnn.cudnn.poolingForward') as func:
            self.forward()
            self.assertEqual(func.called, self.use_cudnn and self.ndim > 1)

    @unittest.skipIf(cuda.cudnn_enabled and
                     cuda.cudnn.cudnn.getVersion() < 3000,
                     'Only cudnn ver>=3 supports average-pooling-nd')
    def test_call_cudnn_backward(self):
        y = self.forward()
        y.grad = self.gy
        with mock.patch('cupy.cudnn.cudnn.poolingBackward') as func:
            y.backward()
            self.assertEqual(func.called, self.use_cudnn and self.ndim > 1)

testing.run_module(__name__, __file__)
<commit_msg>Add regression test against average_pooling_2d.<commit_after>import unittest

import functools
import itertools
import mock
import numpy
import operator
import six

import chainer
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition
from chainer.utils import conv

def pooling_patches(dims, ksize, stride, pad):
    """Return tuples of slices that indicate pooling patches."""
    # Left-top indeces of each pooling patch.
    xss = itertools.product(
        *[six.moves.range(-p, d + p - k + 1, s)
          for (d, k, s, p) in six.moves.zip(dims, ksize, stride, pad)])
    # Tuple of slices for pooling patches.
    return [tuple(slice(max(x, 0), min(x + k, d))
                  for (x, d, k) in six.moves.zip(xs, dims, ksize))
            for xs in xss]

@testing.parameterize(*testing.product({
    'dims': [(4,), (4, 3), (4, 3, 2)],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
class TestAveragePoolingND(unittest.TestCase):

    def setUp(self):
        self.ndim = len(self.dims)
        self.ksize = (3,) * self.ndim
        self.stride = (2,) * self.ndim
        self.pad = (1,) * self.ndim

        x_shape = (2, 3) + self.dims
        self.x = numpy.random.uniform(-1, 1, x_shape).astype(self.dtype)

        outs = tuple(conv.get_conv_outsize(d, k, s, p, False)
                     for (d, k, s, p) in six.moves.zip(
                         self.dims, self.ksize, self.stride, self.pad))
        gy_shape = (2, 3) + outs
        self.gy = numpy.random.uniform(-1, 1, gy_shape).astype(self.dtype)

        self.check_forward_options = {}
        self.check_backward_options = {'eps': 1e-2}
        if self.dtype == numpy.float16:
            self.check_forward_options = {'atol': 5e-4, 'rtol': 5e-3}
            self.check_backward_options = {
                'eps': 1e-1, 'atol': 5e-3, 'rtol': 5e-2}

    def check_forward(self, x_data, use_cudnn=True):
        dims = self.dims
        ksize = self.ksize
        stride = self.stride
        pad = self.pad
        x = chainer.Variable(x_data)
        y = functions.average_pooling_nd(x, ksize, stride, pad,
                                         use_cudnn=use_cudnn)
        self.assertEqual(y.data.dtype, self.dtype)
        y_data = cuda.to_cpu(y.data)

        self.assertEqual(self.gy.shape, y_data.shape)
        for k in six.moves.range(2):
            for c in six.moves.range(3):
                x = self.x[k, c]
                size = functools.reduce(operator.mul, ksize)
                expect = numpy.array([x[idx].sum() for idx in pooling_patches(
                    dims, ksize, stride, pad)])
                expect = expect.reshape(y_data.shape[2:]) / size
                testing.assert_allclose(
                    expect, y_data[k, c], **self.check_forward_options)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.cudnn
    @condition.retry(3)
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x))

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu_no_cudnn(self):
        self.check_forward(cuda.to_gpu(self.x), False)

    def test_forward_consistency_regression(self):
        # Regression test to average_pooling_2d.

        if len(self.dims) != 2:
            return

        ksize = self.ksize
        stride = self.stride
        pad = self.pad

        y_nd = functions.average_pooling_nd(self.x, ksize, stride=stride,
                                            pad=pad, use_cudnn=False)
        y_2d = functions.average_pooling_2d(self.x, ksize, stride=stride,
                                            pad=pad, use_cudnn=False)
        testing.assert_allclose(y_nd.data, y_2d.data)

    def check_backward(self, x_data, y_grad, use_cudnn=True):
        gradient_check.check_backward(
            functions.AveragePoolingND(
                self.ndim, self.ksize, self.stride, self.pad,
                use_cudnn=use_cudnn),
            x_data, y_grad, **self.check_backward_options)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.cudnn
    @condition.retry(3)
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu_no_cudnn(self):
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy), False)

@testing.parameterize(*testing.product({
    'dims': [(4, 3, 2), (3, 2), (2,)],
    'use_cudnn': [True, False],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
@attr.cudnn
class TestAveragePoolingNDCudnnCall(unittest.TestCase):

    def setUp(self):
        self.ndim = len(self.dims)
        self.ksize = (3,) * self.ndim
        self.stride = (2,) * self.ndim
        self.pad = (1,) * self.ndim
        x_shape = (2, 3) + self.dims
        self.x = cuda.cupy.arange(functools.reduce(operator.mul, x_shape),
                                  dtype=self.dtype).reshape(x_shape)
        gy_shape = (2, 3) + tuple(
            conv.get_conv_outsize(d, k, s, p)
            for (d, k, s, p)
            in six.moves.zip(self.dims, self.ksize, self.stride, self.pad))
        self.gy = cuda.cupy.random.uniform(-1, 1, gy_shape).astype(self.dtype)

    def forward(self):
        x = chainer.Variable(self.x)
        return functions.average_pooling_nd(
            x, self.ksize, self.stride, self.pad, use_cudnn=self.use_cudnn)

    @unittest.skipIf(cuda.cudnn_enabled and
                     cuda.cudnn.cudnn.getVersion() < 3000,
                     'Only cudnn ver>=3 supports average-pooling-nd')
    def test_call_cudnn_forward(self):
        with mock.patch('cupy.cudnn.cudnn.poolingForward') as func:
            self.forward()
            self.assertEqual(func.called, self.use_cudnn and self.ndim > 1)

    @unittest.skipIf(cuda.cudnn_enabled and
                     cuda.cudnn.cudnn.getVersion() < 3000,
                     'Only cudnn ver>=3 supports average-pooling-nd')
    def test_call_cudnn_backward(self):
        y = self.forward()
        y.grad = self.gy
        with mock.patch('cupy.cudnn.cudnn.poolingBackward') as func:
            y.backward()
            self.assertEqual(func.called, self.use_cudnn and self.ndim > 1)

testing.run_module(__name__, __file__)
<|endoftext|>