<commit_before>from nose.tools import assert_equal
from KafNafParserPy import KafNafParser
import os, sys
import subprocess
from io import BytesIO

__here__ = os.path.dirname(os.path.realpath(__file__))

if not os.environ.get('ALPINO_HOME'):
    os.environ['ALPINO_HOME'] = os.path.join(os.path.expanduser('~'), 'tools', 'Alpino')

def _test_file(this_file):
    input_fd = open(this_file)

    cmd = [os.path.join(__here__),'run_parser.sh']
    result = subprocess.check_output(os.path.join(__here__,'run_parser.sh'), stdin=input_fd)
    my_obj = KafNafParser(BytesIO(result))

    #Check the terms
    terms = [term for term in my_obj.get_terms()]
    assert_equal(len(terms),12)
    assert_equal(my_obj.get_term('t_4').get_lemma(),'mooi')
    assert_equal(my_obj.get_term('t_4').get_pos(),'adj')

    #Check constituents
    trees = [tree for tree in my_obj.get_trees()]
    assert_equal(len(trees),2)
    assert_equal(trees[0].get_terminals_as_list()[1].get_span().get_span_ids(),['t_1'])

    #Check dependencies
    dependencies = [dep for dep in my_obj.get_dependencies()]
    assert_equal(len(dependencies),10)
    assert_equal(dependencies[5].get_function(),'hd/su')

def test_morphosyn_kaf():
    kaf_file = os.path.join(__here__,'examples','file1.in.kaf')
    _test_file(kaf_file)

def test_morphosyn_naf():
    naf_file = os.path.join(__here__,'examples','file1.in.naf')
    _test_file(naf_file)
<commit_msg>Removed not required line<commit_after>from nose.tools import assert_equal
from KafNafParserPy import KafNafParser
import os, sys
import subprocess
from io import BytesIO

__here__ = os.path.dirname(os.path.realpath(__file__))

if not os.environ.get('ALPINO_HOME'):
    os.environ['ALPINO_HOME'] = os.path.join(os.path.expanduser('~'), 'tools', 'Alpino')

def _test_file(this_file):
    input_fd = open(this_file)

    result = subprocess.check_output(os.path.join(__here__,'run_parser.sh'), stdin=input_fd)
    my_obj = KafNafParser(BytesIO(result))

    #Check the terms
    terms = [term for term in my_obj.get_terms()]
    assert_equal(len(terms),12)
    assert_equal(my_obj.get_term('t_4').get_lemma(),'mooi')
    assert_equal(my_obj.get_term('t_4').get_pos(),'adj')

    #Check constituents
    trees = [tree for tree in my_obj.get_trees()]
    assert_equal(len(trees),2)
    assert_equal(trees[0].get_terminals_as_list()[1].get_span().get_span_ids(),['t_1'])

    #Check dependencies
    dependencies = [dep for dep in my_obj.get_dependencies()]
    assert_equal(len(dependencies),10)
    assert_equal(dependencies[5].get_function(),'hd/su')

def test_morphosyn_kaf():
    kaf_file = os.path.join(__here__,'examples','file1.in.kaf')
    _test_file(kaf_file)

def test_morphosyn_naf():
    naf_file = os.path.join(__here__,'examples','file1.in.naf')
    _test_file(naf_file)
<|endoftext|>


<commit_before>##############################################################################
# For copyright and license notices, see __manifest__.py file in module root
# directory
##############################################################################
from odoo import models, api, _
from odoo.exceptions import ValidationError

class ProductProduct(models.Model):
    _inherit = "product.product"

    @api.constrains('product_tmpl_id', 'default_code', 'active')
    def check_unique_company_and_default_code(self):
        for rec in self:
            if rec.active and rec.default_code and rec.company_id:
                filters = [
                    ('product_tmpl_id.company_id', '=', rec.company_id.id),
                    ('default_code', '=', rec.default_code),
                    ('active', '=', True)]
                products = self.search(filters)
                if len(products) > 1:
                    raise ValidationError(_(
                        'There can not be two active products with the '
                        'same Reference code in the same company.'))
<commit_msg>[PORT] from v11 (#265)<commit_after>##############################################################################
# For copyright and license notices, see __manifest__.py file in module root
# directory
##############################################################################
from odoo import models, fields,  api, _
from odoo.exceptions import ValidationError

class ProductProduct(models.Model):
    _inherit = "product.product"

    default_code = fields.Char(
        copy=False,
    )

    @api.constrains('product_tmpl_id', 'default_code', 'active')
    def check_unique_company_and_default_code(self):
        for rec in self:
            if rec.active and rec.default_code and rec.company_id:
                filters = [
                    ('product_tmpl_id.company_id', '=', rec.company_id.id),
                    ('default_code', '=', rec.default_code),
                    ('active', '=', True)]
                products = self.search(filters)
                if len(products) > 1:
                    raise ValidationError(_(
                        'There can not be two active products with the '
                        'same Reference code in the same company.'))
<|endoftext|>


<commit_before># Copyright 2012 IBM Corp.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import netaddr
from oslo.config import cfg
import six
import webob.exc

from nova.api.openstack import extensions
from nova import exception
from nova.i18n import _
from nova import objects
from nova.openstack.common import log as logging

CONF = cfg.CONF
CONF.import_opt('default_floating_pool', 'nova.network.floating_ips')
CONF.import_opt('public_interface', 'nova.network.linux_net')

LOG = logging.getLogger(__name__)
authorize = extensions.extension_authorizer('compute', 'floating_ips_bulk')

class FloatingIPBulkController(object):

    def index(self, req):
        """Return a list of all floating ips."""
        context = req.environ['nova.context']
        authorize(context)

        return self._get_floating_ip_info(context)

    def show(self, req, id):
        """Return a list of all floating ips for a given host."""
        context = req.environ['nova.context']
        authorize(context)

        return self._get_floating_ip_info(context, id)

    def _get_floating_ip_info(self, context, host=None):
        floating_ip_info = {"floating_ip_info": []}

        if host is None:
            try:
                floating_ips = objects.FloatingIPList.get_all(context)
            except exception.NoFloatingIpsDefined:
                return floating_ip_info
        else:
            try:
                floating_ips = objects.FloatingIPList.get_by_host(context,
                                                                  host)
            except exception.FloatingIpNotFoundForHost as e:
                raise webob.exc.HTTPNotFound(explanation=e.format_message())

        for floating_ip in floating_ips:
            instance_uuid = None
            if floating_ip.fixed_ip:
                instance_uuid = floating_ip.fixed_ip.instance_uuid

            result = {'address': str(floating_ip['address']),
                      'pool': floating_ip['pool'],
                      'interface': floating_ip['interface'],
                      'project_id': floating_ip['project_id'],
                      'instance_uuid': instance_uuid}
            floating_ip_info['floating_ip_info'].append(result)

        return floating_ip_info

    def create(self, req, body):
        """Bulk create floating ips."""
        context = req.environ['nova.context']
        authorize(context)

        if 'floating_ips_bulk_create' not in body:
            raise webob.exc.HTTPUnprocessableEntity()
        params = body['floating_ips_bulk_create']

        if 'ip_range' not in params:
            raise webob.exc.HTTPUnprocessableEntity()
        ip_range = params['ip_range']

        pool = params.get('pool', CONF.default_floating_pool)
        interface = params.get('interface', CONF.public_interface)

        try:
            ips = [objects.FloatingIPList.make_ip_info(addr, pool, interface)
                   for addr in self._address_to_hosts(ip_range)]
        except exception.InvalidInput as exc:
            raise webob.exc.HTTPBadRequest(explanation=exc.format_message())

        try:
            objects.FloatingIPList.create(context, ips)
        except exception.FloatingIpExists as exc:
            raise webob.exc.HTTPBadRequest(explanation=exc.format_message())

        return {"floating_ips_bulk_create": {"ip_range": ip_range,
                                               "pool": pool,
                                               "interface": interface}}

    def update(self, req, id, body):
        """Bulk delete floating IPs."""
        context = req.environ['nova.context']
        authorize(context)

        if id != "delete":
            msg = _("Unknown action")
            raise webob.exc.HTTPNotFound(explanation=msg)

        try:
            ip_range = body['ip_range']
        except (TypeError, KeyError):
            raise webob.exc.HTTPUnprocessableEntity()

        try:
            ips = (objects.FloatingIPList.make_ip_info(address, None, None)
                   for address in self._address_to_hosts(ip_range))
        except exception.InvalidInput as exc:
            raise webob.exc.HTTPBadRequest(explanation=exc.format_message())
        objects.FloatingIPList.destroy(context, ips)

        return {"floating_ips_bulk_delete": ip_range}

    def _address_to_hosts(self, addresses):
        """Iterate over hosts within an address range.

        If an explicit range specifier is missing, the parameter is
        interpreted as a specific individual address.
        """
        try:
            return [netaddr.IPAddress(addresses)]
        except ValueError:
            net = netaddr.IPNetwork(addresses)
            if net.size < 4:
                reason = _("/%s should be specified as single address(es) "
                           "not in cidr format") % net.prefixlen
                raise exception.InvalidInput(reason=reason)
            else:
                return net.iter_hosts()
        except netaddr.AddrFormatError as exc:
            raise exception.InvalidInput(reason=six.text_type(exc))

class Floating_ips_bulk(extensions.ExtensionDescriptor):
    """Bulk handling of Floating IPs."""

    name = "FloatingIpsBulk"
    alias = "os-floating-ips-bulk"
    namespace = ("http://docs.openstack.org/compute/ext/"
                 "floating_ips_bulk/api/v2")
    updated = "2012-10-29T19:25:27Z"

    def get_resources(self):
        resources = []
        resource = extensions.ResourceExtension('os-floating-ips-bulk',
                                                FloatingIPBulkController())
        resources.append(resource)
        return resources
<commit_msg>Clean up LOG import in floating_ips_bulk v2 api<commit_after># Copyright 2012 IBM Corp.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import netaddr
from oslo.config import cfg
import six
import webob.exc

from nova.api.openstack import extensions
from nova import exception
from nova.i18n import _
from nova import objects

CONF = cfg.CONF
CONF.import_opt('default_floating_pool', 'nova.network.floating_ips')
CONF.import_opt('public_interface', 'nova.network.linux_net')

authorize = extensions.extension_authorizer('compute', 'floating_ips_bulk')

class FloatingIPBulkController(object):

    def index(self, req):
        """Return a list of all floating ips."""
        context = req.environ['nova.context']
        authorize(context)

        return self._get_floating_ip_info(context)

    def show(self, req, id):
        """Return a list of all floating ips for a given host."""
        context = req.environ['nova.context']
        authorize(context)

        return self._get_floating_ip_info(context, id)

    def _get_floating_ip_info(self, context, host=None):
        floating_ip_info = {"floating_ip_info": []}

        if host is None:
            try:
                floating_ips = objects.FloatingIPList.get_all(context)
            except exception.NoFloatingIpsDefined:
                return floating_ip_info
        else:
            try:
                floating_ips = objects.FloatingIPList.get_by_host(context,
                                                                  host)
            except exception.FloatingIpNotFoundForHost as e:
                raise webob.exc.HTTPNotFound(explanation=e.format_message())

        for floating_ip in floating_ips:
            instance_uuid = None
            if floating_ip.fixed_ip:
                instance_uuid = floating_ip.fixed_ip.instance_uuid

            result = {'address': str(floating_ip['address']),
                      'pool': floating_ip['pool'],
                      'interface': floating_ip['interface'],
                      'project_id': floating_ip['project_id'],
                      'instance_uuid': instance_uuid}
            floating_ip_info['floating_ip_info'].append(result)

        return floating_ip_info

    def create(self, req, body):
        """Bulk create floating ips."""
        context = req.environ['nova.context']
        authorize(context)

        if 'floating_ips_bulk_create' not in body:
            raise webob.exc.HTTPUnprocessableEntity()
        params = body['floating_ips_bulk_create']

        if 'ip_range' not in params:
            raise webob.exc.HTTPUnprocessableEntity()
        ip_range = params['ip_range']

        pool = params.get('pool', CONF.default_floating_pool)
        interface = params.get('interface', CONF.public_interface)

        try:
            ips = [objects.FloatingIPList.make_ip_info(addr, pool, interface)
                   for addr in self._address_to_hosts(ip_range)]
        except exception.InvalidInput as exc:
            raise webob.exc.HTTPBadRequest(explanation=exc.format_message())

        try:
            objects.FloatingIPList.create(context, ips)
        except exception.FloatingIpExists as exc:
            raise webob.exc.HTTPBadRequest(explanation=exc.format_message())

        return {"floating_ips_bulk_create": {"ip_range": ip_range,
                                               "pool": pool,
                                               "interface": interface}}

    def update(self, req, id, body):
        """Bulk delete floating IPs."""
        context = req.environ['nova.context']
        authorize(context)

        if id != "delete":
            msg = _("Unknown action")
            raise webob.exc.HTTPNotFound(explanation=msg)

        try:
            ip_range = body['ip_range']
        except (TypeError, KeyError):
            raise webob.exc.HTTPUnprocessableEntity()

        try:
            ips = (objects.FloatingIPList.make_ip_info(address, None, None)
                   for address in self._address_to_hosts(ip_range))
        except exception.InvalidInput as exc:
            raise webob.exc.HTTPBadRequest(explanation=exc.format_message())
        objects.FloatingIPList.destroy(context, ips)

        return {"floating_ips_bulk_delete": ip_range}

    def _address_to_hosts(self, addresses):
        """Iterate over hosts within an address range.

        If an explicit range specifier is missing, the parameter is
        interpreted as a specific individual address.
        """
        try:
            return [netaddr.IPAddress(addresses)]
        except ValueError:
            net = netaddr.IPNetwork(addresses)
            if net.size < 4:
                reason = _("/%s should be specified as single address(es) "
                           "not in cidr format") % net.prefixlen
                raise exception.InvalidInput(reason=reason)
            else:
                return net.iter_hosts()
        except netaddr.AddrFormatError as exc:
            raise exception.InvalidInput(reason=six.text_type(exc))

class Floating_ips_bulk(extensions.ExtensionDescriptor):
    """Bulk handling of Floating IPs."""

    name = "FloatingIpsBulk"
    alias = "os-floating-ips-bulk"
    namespace = ("http://docs.openstack.org/compute/ext/"
                 "floating_ips_bulk/api/v2")
    updated = "2012-10-29T19:25:27Z"

    def get_resources(self):
        resources = []
        resource = extensions.ResourceExtension('os-floating-ips-bulk',
                                                FloatingIPBulkController())
        resources.append(resource)
        return resources
<|endoftext|>


<commit_before>import json

from flask import g
import pytest

from dataactbroker.routes import domain_routes
from dataactcore.models.lookups import PERMISSION_SHORT_DICT
from dataactcore.models.userModel import UserAffiliation
from tests.unit.dataactcore.factories.domain import CGACFactory, FRECFactory, SubTierAgencyFactory
from tests.unit.dataactcore.factories.user import UserFactory

@pytest.fixture
def domain_app(test_app):
    domain_routes.add_domain_routes(test_app.application)
    yield test_app

@pytest.mark.usefixtures("user_constants")
def test_list_agencies_limits(domain_app, database):
    """ List agencies should limit to only the user's agencies and should not duplicate the same agency even if there
        are multiple instances of the same agency in the user permissions.
    """
    user = UserFactory()
    cgac = CGACFactory()
    frec_cgac = CGACFactory()
    frec = FRECFactory(cgac=frec_cgac)
    sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code='0', cgac=cgac, frec=None, is_frec=False,
                                      sub_tier_agency_name="Test Subtier Agency 0"),
                 SubTierAgencyFactory(sub_tier_agency_code='1', cgac=frec_cgac, frec=frec, is_frec=True,
                                      sub_tier_agency_name="Test Subtier Agency 1")]
    user.affiliations = [UserAffiliation(cgac=cgac, frec=None, permission_type_id=PERMISSION_SHORT_DICT['w']),
                         UserAffiliation(cgac=cgac, frec=None, permission_type_id=PERMISSION_SHORT_DICT['f']),
                         UserAffiliation(cgac=None, frec=frec, permission_type_id=PERMISSION_SHORT_DICT['w']),
                         UserAffiliation(cgac=None, frec=frec, permission_type_id=PERMISSION_SHORT_DICT['f'])]
    database.session.add_all([cgac] + [frec_cgac] + [frec] + sub_tiers + [user])
    database.session.commit()

    g.user = user
    result = domain_app.get('/v1/list_agencies/').data.decode('UTF-8')
    res = json.loads(result)
    assert len(res['cgac_agency_list']) == 1
    assert len(res['frec_agency_list']) == 1
    assert res['cgac_agency_list'][0]['agency_name'] == cgac.agency_name
    assert res['cgac_agency_list'][0]['cgac_code'] == cgac.cgac_code
    assert res['frec_agency_list'][0]['agency_name'] == frec.agency_name
    assert res['frec_agency_list'][0]['frec_code'] == frec.frec_code

def test_list_agencies_superuser(domain_app, database):
    """ All agencies should be visible to website admins """
    user = UserFactory(website_admin=True)
    cgacs = [CGACFactory(cgac_code=str(i)) for i in range(3)]
    frec_cgac = CGACFactory()
    frecs = [FRECFactory(frec_code=str(i), cgac=frec_cgac) for i in range(3)]
    cgac_sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code=str(i), cgac=cgacs[i], frec=None, is_frec=False,
                                           sub_tier_agency_name="Test Subtier Agency "+str(i)) for i in range(3)]
    frec_sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code=str(3+i), cgac=frec_cgac, frec=frecs[i], is_frec=True,
                                           sub_tier_agency_name="Test Subtier Agency "+str(3+i)) for i in range(3)]
    database.session.add_all(cgacs + [frec_cgac] + frecs + cgac_sub_tiers + frec_sub_tiers + [user])
    database.session.commit()

    g.user = user
    result = domain_app.get('/v1/list_agencies/').data.decode('UTF-8')
    response = json.loads(result)
    result = {el['cgac_code'] for el in response['cgac_agency_list']}
    assert result == {'0', '1', '2'}    # i.e. all of them
    result = {el['frec_code'] for el in response['frec_agency_list']}
    assert result == {'0', '1', '2'}    # i.e. all of them

@pytest.mark.usefixtures("user_constants")
def test_list_agencies_all(domain_app, database):
    """ All agencies should be visible to website admins """
    user = UserFactory()
    cgacs = [CGACFactory(cgac_code=str(i)) for i in range(3)]
    frec_cgac = CGACFactory()
    frecs = [FRECFactory(frec_code=str(i), cgac=frec_cgac) for i in range(3)]
    cgac_sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code=str(i), cgac=cgacs[i], frec=None, is_frec=False,
                                           sub_tier_agency_name="Test Subtier Agency "+str(i)) for i in range(3)]
    frec_sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code=str(3+i), cgac=frec_cgac, frec=frecs[i], is_frec=True,
                                           sub_tier_agency_name="Test Subtier Agency "+str(3+i)) for i in range(3)]
    user.affiliations = [UserAffiliation(cgac=cgacs[0], frec=frecs[0], permission_type_id=PERMISSION_SHORT_DICT['w'])]
    database.session.add_all(cgacs + [frec_cgac] + frecs + cgac_sub_tiers + frec_sub_tiers + [user])
    database.session.commit()

    g.user = user
    result = domain_app.get('/v1/list_all_agencies/').data.decode('UTF-8')
    response = json.loads(result)
    result = {el['cgac_code'] for el in response['agency_list']}
    assert result == {'0', '1', '2'}    # i.e. all of them
    result = {el['frec_code'] for el in response['shared_agency_list']}
    assert result == {'0', '1', '2'}    # i.e. all of them

@pytest.mark.usefixtures("user_constants")
def test_list_sub_tier_agencies(domain_app, database):
    """ List all sub tiers that a user has FABS permissions for """
    user = UserFactory()
    cgacs = [CGACFactory(cgac_code=str(i)) for i in range(3)]
    frec_cgac = CGACFactory()
    frecs = [FRECFactory(frec_code=str(i), cgac=frec_cgac) for i in range(3)]
    cgac_sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code=str(i), cgac=cgacs[i], frec=None, is_frec=False,
                                           sub_tier_agency_name="Test Subtier Agency " + str(i)) for i in range(3)]
    frec_sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code=str(3 + i), cgac=frec_cgac, frec=frecs[i], is_frec=True,
                                           sub_tier_agency_name="Test Subtier Agency " + str(3 + i)) for i in range(3)]
    user.affiliations = [UserAffiliation(cgac=cgacs[0], frec=None, permission_type_id=PERMISSION_SHORT_DICT['f']),
                         UserAffiliation(cgac=None, frec=frecs[2], permission_type_id=PERMISSION_SHORT_DICT['f'])]
    database.session.add_all(cgacs + [frec_cgac] + frecs + cgac_sub_tiers + frec_sub_tiers + [user])
    database.session.commit()

    g.user = user
    result = domain_app.get('/v1/list_sub_tier_agencies/').data.decode('UTF-8')
    response = json.loads(result)
    result = {el['agency_code'] for el in response['sub_tier_agency_list']}
    assert len(response["sub_tier_agency_list"]) == 2  # Only one cgac and one frec
    assert result == {'0', '5'}  # Only subtiers created from the relevant cgacs

@pytest.mark.usefixtures("user_constants")
def test_list_sub_tier_agencies_admin(domain_app, database):
    """ List all sub tiers that a user has FABS permissions for """
    user = UserFactory(website_admin=True)
    cgacs = [CGACFactory(cgac_code=str(i)) for i in range(3)]
    frec_cgac = CGACFactory()
    frecs = [FRECFactory(frec_code=str(i), cgac=frec_cgac) for i in range(3)]
    cgac_sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code=str(i), cgac=cgacs[i], frec=None, is_frec=False,
                                           sub_tier_agency_name="Test Subtier Agency " + str(i)) for i in range(3)]
    frec_sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code=str(3 + i), cgac=frec_cgac, frec=frecs[i], is_frec=True,
                                           sub_tier_agency_name="Test Subtier Agency " + str(3 + i)) for i in range(3)]
    user.affiliations = [UserAffiliation(cgac=cgacs[0], frec=None, permission_type_id=PERMISSION_SHORT_DICT['f']),
                         UserAffiliation(cgac=None, frec=frecs[2], permission_type_id=PERMISSION_SHORT_DICT['f'])]
    database.session.add_all(cgacs + [frec_cgac] + frecs + cgac_sub_tiers + frec_sub_tiers + [user])
    database.session.commit()

    g.user = user
    result = domain_app.get('/v1/list_sub_tier_agencies/').data.decode('UTF-8')
    response = json.loads(result)
    result = {el['agency_code'] for el in response['sub_tier_agency_list']}
    assert len(response["sub_tier_agency_list"]) == 6  # All of them, ignores affiliations
    assert result == {'0', '1', '2', '3', '4', '5'}  # All of them, ignores affiliations
<commit_msg>Updating list_agencies unit test<commit_after>import json

from flask import g
import pytest

from dataactbroker.routes import domain_routes
from dataactcore.models.lookups import PERMISSION_SHORT_DICT
from dataactcore.models.userModel import UserAffiliation
from tests.unit.dataactcore.factories.domain import CGACFactory, FRECFactory, SubTierAgencyFactory
from tests.unit.dataactcore.factories.user import UserFactory

@pytest.fixture
def domain_app(test_app):
    domain_routes.add_domain_routes(test_app.application)
    yield test_app

@pytest.mark.usefixtures("user_constants")
def test_list_agencies_limits(domain_app, database):
    """ List agencies should limit to only the user's agencies and should not duplicate the same agency even if there
        are multiple instances of the same agency in the user permissions.
    """
    user = UserFactory()
    cgac = CGACFactory()
    frec_cgac = CGACFactory()
    frec = FRECFactory(cgac=frec_cgac)
    sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code='0', cgac=cgac, frec=None, is_frec=False,
                                      sub_tier_agency_name="Test Subtier Agency 0"),
                 SubTierAgencyFactory(sub_tier_agency_code='1', cgac=frec_cgac, frec=frec, is_frec=True,
                                      sub_tier_agency_name="Test Subtier Agency 1")]
    user.affiliations = [UserAffiliation(cgac=cgac, frec=None, permission_type_id=PERMISSION_SHORT_DICT['w']),
                         UserAffiliation(cgac=cgac, frec=None, permission_type_id=PERMISSION_SHORT_DICT['f']),
                         UserAffiliation(cgac=None, frec=frec, permission_type_id=PERMISSION_SHORT_DICT['w']),
                         UserAffiliation(cgac=None, frec=frec, permission_type_id=PERMISSION_SHORT_DICT['f'])]
    database.session.add_all([cgac] + [frec_cgac] + [frec] + sub_tiers + [user])
    database.session.commit()

    g.user = user
    result = domain_app.get('/v1/list_agencies/').data.decode('UTF-8')
    res = json.loads(result)
    assert len(res['cgac_agency_list']) == 1
    assert len(res['frec_agency_list']) == 1
    assert res['cgac_agency_list'][0]['agency_name'] == cgac.agency_name
    assert res['cgac_agency_list'][0]['cgac_code'] == cgac.cgac_code
    assert res['frec_agency_list'][0]['agency_name'] == frec.agency_name
    assert res['frec_agency_list'][0]['frec_code'] == frec.frec_code

@pytest.mark.usefixtures("user_constants")
def test_list_agencies_perm_params(domain_app, database):
    """ Users should be able to filter their affiliations based on the arguments provided """
    user = UserFactory()
    r_cgac = CGACFactory()
    w_cgac = CGACFactory()
    s_frec_cgac = CGACFactory()
    s_frec = FRECFactory(cgac=s_frec_cgac)
    e_frec_cgac = CGACFactory()
    e_frec = FRECFactory(cgac=e_frec_cgac)
    f_cgac = CGACFactory()
    user.affiliations = [UserAffiliation(cgac=r_cgac, frec=None, permission_type_id=PERMISSION_SHORT_DICT['r']),
                         UserAffiliation(cgac=w_cgac, frec=None, permission_type_id=PERMISSION_SHORT_DICT['w']),
                         UserAffiliation(cgac=None, frec=s_frec, permission_type_id=PERMISSION_SHORT_DICT['s']),
                         UserAffiliation(cgac=None, frec=e_frec, permission_type_id=PERMISSION_SHORT_DICT['e']),
                         UserAffiliation(cgac=f_cgac, frec=None, permission_type_id=PERMISSION_SHORT_DICT['f'])]
    database.session.add_all([r_cgac, w_cgac, s_frec_cgac, s_frec, e_frec_cgac, e_frec, f_cgac])
    database.session.commit()

    g.user = user

    def call_list_agencies(perm_level='reader', perm_type='mixed'):
        result = domain_app.get('/v1/list_agencies/?perm_level={}&perm_type={}'.format(perm_level, perm_type))\
            .data.decode('UTF-8')
        resp = json.loads(result)
        cgac_codes = [agency['cgac_code'] for agency in resp['cgac_agency_list']]
        frec_codes = [agency['frec_code'] for agency in resp['frec_agency_list']]
        return cgac_codes, frec_codes

    default_cgacs, default_frecs = call_list_agencies()
    assert {r_cgac.cgac_code, w_cgac.cgac_code, f_cgac.cgac_code} == set(default_cgacs)
    assert {s_frec.frec_code, e_frec.frec_code} == set(default_frecs)

    writer_cgacs, writer_frecs = call_list_agencies(perm_level='writer')
    assert {w_cgac.cgac_code, f_cgac.cgac_code} == set(writer_cgacs)
    assert {s_frec.frec_code, e_frec.frec_code} == set(writer_frecs)

    submitter_cgacs, submitter_frecs = call_list_agencies(perm_level='submitter')
    assert {f_cgac.cgac_code} == set(submitter_cgacs)
    assert {s_frec.frec_code} == set(submitter_frecs)

    dabs_cgacs, dabs_frecs = call_list_agencies(perm_type='dabs')
    assert {r_cgac.cgac_code, w_cgac.cgac_code} == set(dabs_cgacs)
    assert {s_frec.frec_code} == set(dabs_frecs)

    fabs_cgacs, fabs_frecs = call_list_agencies(perm_type='fabs')
    assert {f_cgac.cgac_code} == set(fabs_cgacs)
    assert {e_frec.frec_code} == set(fabs_frecs)

    # mix
    mix_cgacs, mix_frecs = call_list_agencies(perm_level='submitter', perm_type='fabs')
    assert {f_cgac.cgac_code} == set(mix_cgacs)
    assert set() == set(mix_frecs)

def test_list_agencies_superuser(domain_app, database):
    """ All agencies should be visible to website admins """
    user = UserFactory(website_admin=True)
    cgacs = [CGACFactory(cgac_code=str(i)) for i in range(3)]
    frec_cgac = CGACFactory()
    frecs = [FRECFactory(frec_code=str(i), cgac=frec_cgac) for i in range(3)]
    cgac_sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code=str(i), cgac=cgacs[i], frec=None, is_frec=False,
                                           sub_tier_agency_name="Test Subtier Agency "+str(i)) for i in range(3)]
    frec_sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code=str(3+i), cgac=frec_cgac, frec=frecs[i], is_frec=True,
                                           sub_tier_agency_name="Test Subtier Agency "+str(3+i)) for i in range(3)]
    database.session.add_all(cgacs + [frec_cgac] + frecs + cgac_sub_tiers + frec_sub_tiers + [user])
    database.session.commit()

    g.user = user
    result = domain_app.get('/v1/list_agencies/').data.decode('UTF-8')
    response = json.loads(result)
    result = {el['cgac_code'] for el in response['cgac_agency_list']}
    assert result == {'0', '1', '2'}    # i.e. all of them
    result = {el['frec_code'] for el in response['frec_agency_list']}
    assert result == {'0', '1', '2'}    # i.e. all of them

@pytest.mark.usefixtures("user_constants")
def test_list_agencies_all(domain_app, database):
    """ All agencies should be visible to website admins """
    user = UserFactory()
    cgacs = [CGACFactory(cgac_code=str(i)) for i in range(3)]
    frec_cgac = CGACFactory()
    frecs = [FRECFactory(frec_code=str(i), cgac=frec_cgac) for i in range(3)]
    cgac_sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code=str(i), cgac=cgacs[i], frec=None, is_frec=False,
                                           sub_tier_agency_name="Test Subtier Agency "+str(i)) for i in range(3)]
    frec_sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code=str(3+i), cgac=frec_cgac, frec=frecs[i], is_frec=True,
                                           sub_tier_agency_name="Test Subtier Agency "+str(3+i)) for i in range(3)]
    user.affiliations = [UserAffiliation(cgac=cgacs[0], frec=frecs[0], permission_type_id=PERMISSION_SHORT_DICT['w'])]
    database.session.add_all(cgacs + [frec_cgac] + frecs + cgac_sub_tiers + frec_sub_tiers + [user])
    database.session.commit()

    g.user = user
    result = domain_app.get('/v1/list_all_agencies/').data.decode('UTF-8')
    response = json.loads(result)
    result = {el['cgac_code'] for el in response['agency_list']}
    assert result == {'0', '1', '2'}    # i.e. all of them
    result = {el['frec_code'] for el in response['shared_agency_list']}
    assert result == {'0', '1', '2'}    # i.e. all of them

@pytest.mark.usefixtures("user_constants")
def test_list_sub_tier_agencies(domain_app, database):
    """ List all sub tiers that a user has FABS permissions for """
    user = UserFactory()
    cgacs = [CGACFactory(cgac_code=str(i)) for i in range(3)]
    frec_cgac = CGACFactory()
    frecs = [FRECFactory(frec_code=str(i), cgac=frec_cgac) for i in range(3)]
    cgac_sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code=str(i), cgac=cgacs[i], frec=None, is_frec=False,
                                           sub_tier_agency_name="Test Subtier Agency " + str(i)) for i in range(3)]
    frec_sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code=str(3 + i), cgac=frec_cgac, frec=frecs[i], is_frec=True,
                                           sub_tier_agency_name="Test Subtier Agency " + str(3 + i)) for i in range(3)]
    user.affiliations = [UserAffiliation(cgac=cgacs[0], frec=None, permission_type_id=PERMISSION_SHORT_DICT['f']),
                         UserAffiliation(cgac=None, frec=frecs[2], permission_type_id=PERMISSION_SHORT_DICT['f'])]
    database.session.add_all(cgacs + [frec_cgac] + frecs + cgac_sub_tiers + frec_sub_tiers + [user])
    database.session.commit()

    g.user = user
    result = domain_app.get('/v1/list_sub_tier_agencies/').data.decode('UTF-8')
    response = json.loads(result)
    result = {el['agency_code'] for el in response['sub_tier_agency_list']}
    assert len(response["sub_tier_agency_list"]) == 2  # Only one cgac and one frec
    assert result == {'0', '5'}  # Only subtiers created from the relevant cgacs

@pytest.mark.usefixtures("user_constants")
def test_list_sub_tier_agencies_admin(domain_app, database):
    """ List all sub tiers that a user has FABS permissions for """
    user = UserFactory(website_admin=True)
    cgacs = [CGACFactory(cgac_code=str(i)) for i in range(3)]
    frec_cgac = CGACFactory()
    frecs = [FRECFactory(frec_code=str(i), cgac=frec_cgac) for i in range(3)]
    cgac_sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code=str(i), cgac=cgacs[i], frec=None, is_frec=False,
                                           sub_tier_agency_name="Test Subtier Agency " + str(i)) for i in range(3)]
    frec_sub_tiers = [SubTierAgencyFactory(sub_tier_agency_code=str(3 + i), cgac=frec_cgac, frec=frecs[i], is_frec=True,
                                           sub_tier_agency_name="Test Subtier Agency " + str(3 + i)) for i in range(3)]
    user.affiliations = [UserAffiliation(cgac=cgacs[0], frec=None, permission_type_id=PERMISSION_SHORT_DICT['f']),
                         UserAffiliation(cgac=None, frec=frecs[2], permission_type_id=PERMISSION_SHORT_DICT['f'])]
    database.session.add_all(cgacs + [frec_cgac] + frecs + cgac_sub_tiers + frec_sub_tiers + [user])
    database.session.commit()

    g.user = user
    result = domain_app.get('/v1/list_sub_tier_agencies/').data.decode('UTF-8')
    response = json.loads(result)
    result = {el['agency_code'] for el in response['sub_tier_agency_list']}
    assert len(response["sub_tier_agency_list"]) == 6  # All of them, ignores affiliations
    assert result == {'0', '1', '2', '3', '4', '5'}  # All of them, ignores affiliations
<|endoftext|>


<commit_before>import sys
import itertools
import numpy as np
import pandas as pd

from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LinearRegression, LogisticRegression, Lasso
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score, precision_score, recall_score, confusion_matrix

# Takes in the training set of features and labels, and returns the trained model:
def train(features_train, labels_train):
	model = LogisticRegression(C=1e5)
	model.fit(features_train, labels_train)
	return model

# Takes in a trained model, and evaluates it with the test set:
# It also writes the predictions, to be used by downstream simulation processes
def test_model(trained_model, features_test, raw_price_data):
	predictions = trained_model.predict(features_test)

	bank = 0
	spent = 0
	coins = 0
	for index, pred in enumerate(predictions):
		# if not index % 60 * 24 == 0:
		# 	continue
		open_price = raw_price_data.iloc[index][0]
		close_price = raw_price_data.iloc[index][1]
		if pred == 0:
			if coins > 0:
				coins -= 1
				bank += close_price
		else:
			coins += 1
			spent += open_price

	net_earn = bank - spent + coins * close_price
	print('Assume infinite funds, total input is %s, total output is %s' % (str(spent), str(bank)))
	print('We have %s coins left in our account' % (str(coins)))
	print('And our total profit is %s' % (str(net_earn)))

# trains and tests a model with the provided train and test sets, writes predictions with the given output name
def train_and_simulate_process(features_train, labels_train, features_test, raw_price_data):
	model = train(features_train, labels_train)

	raw_price_data = raw_price_data[-len(features_test):].reset_index(drop=True)[['Open', 'Close']]
	test_model(model, features_test, raw_price_data)

def load_params():
	config = configparser.ConfigParser()
	config.read('../config.ini')

	global TRANSACTION_SIZE
	TRANSACTION_SIZE = int(config['Simulation']['base'])<commit_msg>Correct daily (commented)<commit_after>import sys
import itertools
import numpy as np
import pandas as pd

from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LinearRegression, LogisticRegression, Lasso
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score, precision_score, recall_score, confusion_matrix

# Takes in the training set of features and labels, and returns the trained model:
def train(features_train, labels_train):
	model = LogisticRegression(C=1e5)
	model.fit(features_train, labels_train)
	return model

# Takes in a trained model, and evaluates it with the test set:
# It also writes the predictions, to be used by downstream simulation processes
def test_model(trained_model, features_test, raw_price_data):
	predictions = trained_model.predict(features_test)

	bank = 0
	spent = 0
	coins = 0
	for index, pred in enumerate(predictions):
		# if not index % (60 * 24) == 0:
			# continue
		open_price = raw_price_data.iloc[index][0]
		close_price = raw_price_data.iloc[index][1]
		if pred == 0:
			if coins > 0:
				coins -= 1
				bank += close_price
		else:
			coins += 1
			spent += open_price

	net_earn = bank - spent + coins * close_price
	print('Assume infinite funds, total input is %s, total output is %s' % (str(spent), str(bank)))
	print('We have %s coins left in our account' % (str(coins)))
	print('And our total profit is %s' % (str(net_earn)))

# trains and tests a model with the provided train and test sets, writes predictions with the given output name
def train_and_simulate_process(features_train, labels_train, features_test, raw_price_data):
	model = train(features_train, labels_train)

	raw_price_data = raw_price_data[-len(features_test):].reset_index(drop=True)[['Open', 'Close']]
	test_model(model, features_test, raw_price_data)

def load_params():
	config = configparser.ConfigParser()
	config.read('../config.ini')

	global TRANSACTION_SIZE
	TRANSACTION_SIZE = int(config['Simulation']['base'])<|endoftext|>


<commit_before>from __future__ import absolute_import
import inspect
import threading
import functools
from itertools import chain, count
import logging
from enum import Enum
from contextlib import contextmanager, ExitStack
import weakref

from easypy.concurrency import Futures, MultiObject
from easypy.decorations import parametrizeable_decorator, kwargs_resilient, WeakMethodDead
from easypy.exceptions import TException
from easypy.contexts import is_contextmanager

PRIORITIES = Enum("PRIORITIES", "FIRST NONE LAST")
_logger = logging.getLogger(__name__)

ids = set()

class MissingIdentifier(TException):
    template = "signal handler {_signal_name} must be called with a '{_identifier}' argument"

def make_id(name):
    # from md5 import md5
    # id = md5(name).hexdigest()[:5].upper()
    for c in range(10):
        id = "".join(word[0] for word in name.split("_")) + (str(c) if c else "")
        if id not in ids:
            break
    ids.add(id)
    return id

def get_original_func(func):
    while True:
        if isinstance(func, functools.partial):
            func = func.func
        elif hasattr(func, "__wrapped__") and not (
            is_contextmanager(func) and not is_contextmanager(func.__wrapped__)
            # This means func is the context manager itself (which wraps a non-context-manager function)
        ):
            func = func.__wrapped__
        else:
            return func

class SignalHandler(object):

    _idx_gen = count(100)

    def __init__(self, func, async=False, priority=PRIORITIES.NONE, times=None, identifier=None):
        self.func = kwargs_resilient(func)
        self._func = get_original_func(func)  # save the original funtion so we can unregister based on the function

        # identifier only applicable to methods
        if inspect.ismethod(self._func) or isinstance(self._func, weakref.WeakMethod):
            self.identifier = identifier
        else:
            self.identifier = None

        self.async = async
        self.priority = priority
        self.times = times
        self.idx = next(self._idx_gen)

        if isinstance(func, weakref.WeakMethod):
            func = func()  # to allow accessing it's __code__ and __name__
        self.filename = func.__code__.co_filename
        self.lineno = func.__code__.co_firstlineno
        self.name = self.__name__ = func.__name__

    def __repr__(self):
        return "<handler #{0.idx} '{0.name}' ({0.filename}:{0.lineno})>".format(self)

    def should_run(self, **kwargs):
        if self.times == 0:
            return False

        if self.identifier:
            try:
                handler_object = self._func.__self__
            except AttributeError:
                func = self._func()
                if func is None:
                    raise WeakMethodDead
                handler_object = func.__self__
            target_object = kwargs[self.identifier]

            if hasattr(self._func, 'identifier_path'):
                # special case if the signal method defined a user-defined attribute path for fiding the associated identifier
                path = self._func.identifier_path
                handler_object = eval('obj.%s' % path.strip('.'), dict(obj=handler_object), {})
            elif hasattr(handler_object, self.identifier):
                # the identifier exists as an attribute on the handler object, named the same
                handler_object = getattr(handler_object, self.identifier)
            elif isinstance(handler_object, target_object.__class__):
                # the handler_object is itself of the same type as the target_object, so we should regard it is the identifier itself
                pass
            else:
                # the handler_object isn't identifiable, so we'll just pass it the identifier and hope it knows what to do
                handler_object = None

            if handler_object is not None and handler_object != target_object:
                return False
        return True

    def __call__(self, *, swallow_exceptions, **kwargs):
        if self.times is not None:
            self.times -= 1

        try:
            return self.func(**kwargs)
        except:
            if not swallow_exceptions:
                raise
            _logger.silent_exception("Exception in (%s) ignored", self)

    @property
    def bound_object(self):
        if isinstance(self._func, weakref.WeakMethod):
            return weakref.ref.__call__(self._func)
        else:
            return None

class Signal:

    ALL = {}

    def __new__(cls, name, async=None, swallow_exceptions=False, log=True):
        try:
            return cls.ALL[name]
        except KeyError:
            pass
        assert threading.main_thread() is threading.current_thread(), "Can only create Signal objects from the MainThread"
        signal = object.__new__(cls)
        # we use this to track the calling of various callbacks. we use this short id so it doesn't overflow the logs
        signal.id = make_id(name)
        signal.handlers = {priority: [] for priority in PRIORITIES}
        signal.name = name
        signal.swallow_exceptions = swallow_exceptions
        signal.async = async
        signal.identifier = None
        signal.log = log
        return cls.ALL.setdefault(name, signal)

    def iter_handlers(self):
        return chain(*(self.handlers[k] for k in PRIORITIES))

    def remove_handler(self, handler):
        self.handlers[handler.priority].remove(handler)
        _logger.debug("handler removed from '%s' (%s): %s", self.name, handler.priority.name, handler)

    def register(self, func=None, async=None, priority=PRIORITIES.NONE, times=None):
        if not func:
            return functools.partial(self.register, async=async, priority=priority)
        if async is None:
            async = False if self.async is None else self.async
        elif self.async is not None:
            assert self.async == async, "Signal is set with async=%s" % self.async
        handler = SignalHandler(func, async, priority, times=times, identifier=self.identifier)
        self.handlers[priority].append(handler)
        _logger.debug("registered handler for '%s' (%s): %s", self.name, priority.name, handler)
        return func

    def unregister(self, func):
        for handler in self.iter_handlers():
            if func in (
                    handler._func,  # simple case
                    getattr(handler._func, "__wrapped__", None),  # wrapped with @wraps
                    getattr(handler._func, "func", None),  # wrapped with 'partial'
                    handler._func() if isinstance(handler._func, weakref.WeakMethod) else None,
                    ):
                self.remove_handler(handler)

    @contextmanager
    def registered(self, func, **kwargs):
        self.register(func, **kwargs)
        try:
            yield
        finally:
            self.unregister(func)

    def __call__(self, **kwargs):
        if not self.identifier:
            pass
        elif self.identifier in kwargs:
            pass
        else:
            raise MissingIdentifier(_signal_name=self.name, _identifier=self.identifier)

        if self.log:
            # log signal for centralized logging analytics.
            # minimize text message as most of the data is sent in the 'extra' dict
            # signal fields get prefixed with 'sig_', and the values are repr'ed
            signal_fields = {("sig_%s" % k): repr(v) for (k, v) in kwargs.items()}
            _logger.debug("Triggered %s", self, extra=dict(signal_fields, signal=self.name, signal_id=self.id))

        for handler in self.iter_handlers():
            if handler.times == 0:
                self.remove_handler(handler)

        kwargs.setdefault('swallow_exceptions', self.swallow_exceptions)

        with ExitStack() as STACK:
            STACK.enter_context(_logger.context(self.id))

            if any(h.async for h in self.iter_handlers()):
                futures = STACK.enter_context(Futures.executor())
            else:
                futures = None

            handlers_to_remove = []

            def run_handler(handler):
                try:
                    with _logger.context("#%03d" % handler.idx):
                        handler(**kwargs)
                except WeakMethodDead:
                    handlers_to_remove.append(handler)

            for handler in self.iter_handlers():
                try:
                    if not handler.should_run(**kwargs):
                        continue
                except WeakMethodDead:
                    handlers_to_remove.append(handler)
                    continue
                # allow handler to use our async context
                if handler.async:
                    futures.submit(run_handler, handler)
                else:
                    run_handler(handler)

            if futures:
                for future in futures.as_completed():
                    future.result()        # bubble up exceptions

        for handler in handlers_to_remove:
            self.remove_handler(handler)

    def __str__(self):
        return "<Signal %s (%s)>" % (self.name, self.id)

    __repr__ = __str__

@parametrizeable_decorator
def signal_identifier_path(func, path):
    func.identifier_path = path
    return func

class ContextManagerSignal(Signal):
    @contextmanager
    def __call__(self, **kwargs):
        if self.log:
            # log signal for centralized logging analytics.
            # minimize text message as most of the data is sent in the 'extra' dict
            # signal fields get prefixed with 'sig_', and the values are repr'ed
            signal_fields = {("sig_%s" % k): repr(v) for (k, v) in kwargs.items()}
            _logger.debug("Triggered '%s' (%s) - entering", self.name, self.id,
                          extra=dict(signal_fields, signal=self.name, signal_id=self.id))

        for handler in self.iter_handlers():
            if handler.times == 0:
                self.unregister(handler.func)

        kwargs.setdefault('swallow_exceptions', self.swallow_exceptions)

        with ExitStack() as handlers_stack:
            async_handlers = MultiObject()
            handlers = []

            handlers_to_remove = []

            @contextmanager
            def run_handler(handler):
                already_yielded = False
                try:
                    with _logger.context(self.id), _logger.context("%02d" % index):
                        with handler(**kwargs):
                            yield
                            already_yielded = True
                except WeakMethodDead:
                    handlers_to_remove.append(handler)
                    if not already_yielded:
                        yield

            for index, handler in enumerate(self.iter_handlers()):
                try:
                    if not handler.should_run(**kwargs):
                        continue
                except WeakMethodDead:
                    handlers_to_remove.append(handler)
                    continue
                # allow handler to use our async context
                if handler.async:
                    async_handlers.append(handler)
                else:
                    handlers.append(handler)
            if async_handlers:
                handlers_stack.enter_context(async_handlers.call(run_handler))
            for handler in handlers:
                res = run_handler(handler)
                if res:
                    handlers_stack.enter_context(res)
            yield

            for handler in handlers_to_remove:
                self.remove_handler(handler)

####################################
def register_signal(name, func, **kwargs):
    signal = Signal(name)
    signal.register(func, **kwargs)
    return functools.partial(signal.unregister, func)

def unregister_signal(name, func):
    try:
        signal = Signal.ALL[name]
    except KeyError:
        return
    return signal.unregister(func)

@parametrizeable_decorator
def register(func, **kwargs):
    register_signal(func.__name__, func, **kwargs)
    return func

def unregister(func):
    return unregister_signal(func.__name__, func)

def _set_handler_params(**kw):
    def inner(func):
        params = getattr(func, '_signal_handler_params', {})
        params.update(kw)
        func._signal_handler_params = params
        return func
    return inner

run_first = _set_handler_params(priority=PRIORITIES.FIRST)
run_last = _set_handler_params(priority=PRIORITIES.LAST)
run_async = _set_handler_params(async=True)
run_sync = _set_handler_params(async=False)

@functools.lru_cache(None)
def get_signals_for_type(typ):
    return {n for typ in inspect.getmro(typ)
            for n in dir(typ) if n.startswith("on_")}

class __FakeCode:
    def __init__(self, fake_filename, fake_firstlineno):
        self.co_filename = fake_filename
        self.co_firstlineno = fake_firstlineno

def __fake_meth_type(func, obj, name, code):
    method = func.__get__(obj, type(obj))
    method.__name__ = name
    method.__code__ = code
    return method

def register_object(obj, **kwargs):
    for method_name in get_signals_for_type(type(obj)):
        method = getattr(obj, method_name)
        assert callable(method), "'%s' (%r) is not callable" % (method, obj)

        # Don't use static/class methods for automatic event registration - they
        # could be registered multiple times if the are multiple objects of that
        # type in the system, or not at all if there are no objects of that type
        # in the system.
        assert method is not getattr(type(obj), method_name, None), "'%s' is a static/class method" % method

        params = getattr(method, '_signal_handler_params', {})
        intersection = set(params).intersection(kwargs)
        assert not intersection, "parameter conflict in signal object registration (%s)" % (intersection)
        params.update(kwargs)

        method_name, *_ = method_name.partition("__")  # allows multiple methods for the same signal

        if not inspect.ismethod(method):
            # Fake method attributes for WeakMethod
            method.__self__ = obj
            method.__func__ = getattr(type(obj), method_name)
            if not hasattr(method, '__code__'):
                method.__code__ = __FakeCode('', '')
            fake_meth_type = functools.partial(
                __fake_meth_type,
                # Best effort: fake name and code for SignalHandler. We don't know the line number.
                name=method_name,
                code=__FakeCode(type(obj).__module__, '?'))

        method = weakref.WeakMethod(method)
        try:
            method._meth_type = fake_meth_type
        except UnboundLocalError:
            pass

        register_signal(method_name, method, **params)

def unregister_object(obj):
    for signal_name in {method_name.split('__', 1)[0] for method_name in get_signals_for_type(type(obj))}:
        for handlers in Signal.ALL[signal_name].handlers.values():
            handlers[:] = (handler for handler in handlers if handler.bound_object is not obj)

def call_signal(name, **kwargs):
    return Signal(name)(**kwargs)

def log_all_signal_ids(logger=_logger, level=logging.DEBUG):
    for signal in sorted(Signal.ALL.itervalues(), key=lambda signal: signal.name):
        logger.log(level, "%s - '%s'", signal.id, signal.name)

# ===================================================================================================
# Module hack: ``from easypy.signals import on_some_signal``
#              ``from easypy.signals import on_ctx_some_signal`` for a context-manager signal
# ===================================================================================================
def _hack():
    import sys
    from types import ModuleType
    this = sys.modules[__name__]

    class SignalsModule(ModuleType):
        """The module-hack that allows us to use ``from easypy.signals import on_some_signal``"""
        __all__ = ()  # to make help() happy
        __package__ = __name__
        __path__ = []
        __file__ = __file__

        def __getattr__(self, name):
            try:
                return getattr(this, name)
            except AttributeError:
                if name.startswith("on_ctx_"):
                    return ContextManagerSignal(name)
                elif name.startswith("on_"):
                    return Signal(name)
                raise

        def __dir__(self):
            return sorted(dir(this) + list(Signal.ALL))

    module = SignalsModule(__name__, SignalsModule.__doc__)
    sys.modules[module.__name__] = module

_hack()
<commit_msg>Be resilient to signal handler removal on GC<commit_after>from __future__ import absolute_import
import inspect
import threading
import functools
from itertools import chain, count
import logging
from enum import Enum
from contextlib import contextmanager, ExitStack
import weakref

from easypy.concurrency import Futures, MultiObject
from easypy.decorations import parametrizeable_decorator, kwargs_resilient, WeakMethodDead
from easypy.exceptions import TException
from easypy.contexts import is_contextmanager

PRIORITIES = Enum("PRIORITIES", "FIRST NONE LAST")
_logger = logging.getLogger(__name__)

ids = set()

class MissingIdentifier(TException):
    template = "signal handler {_signal_name} must be called with a '{_identifier}' argument"

def make_id(name):
    # from md5 import md5
    # id = md5(name).hexdigest()[:5].upper()
    for c in range(10):
        id = "".join(word[0] for word in name.split("_")) + (str(c) if c else "")
        if id not in ids:
            break
    ids.add(id)
    return id

def get_original_func(func):
    while True:
        if isinstance(func, functools.partial):
            func = func.func
        elif hasattr(func, "__wrapped__") and not (
            is_contextmanager(func) and not is_contextmanager(func.__wrapped__)
            # This means func is the context manager itself (which wraps a non-context-manager function)
        ):
            func = func.__wrapped__
        else:
            return func

class SignalHandler(object):

    _idx_gen = count(100)

    def __init__(self, func, async=False, priority=PRIORITIES.NONE, times=None, identifier=None):
        self.func = kwargs_resilient(func)
        self._func = get_original_func(func)  # save the original funtion so we can unregister based on the function

        # identifier only applicable to methods
        if inspect.ismethod(self._func) or isinstance(self._func, weakref.WeakMethod):
            self.identifier = identifier
        else:
            self.identifier = None

        self.async = async
        self.priority = priority
        self.times = times
        self.idx = next(self._idx_gen)

        if isinstance(func, weakref.WeakMethod):
            func = func()  # to allow accessing it's __code__ and __name__
        self.filename = func.__code__.co_filename
        self.lineno = func.__code__.co_firstlineno
        self.name = self.__name__ = func.__name__

    def __repr__(self):
        return "<handler #{0.idx} '{0.name}' ({0.filename}:{0.lineno})>".format(self)

    def should_run(self, **kwargs):
        if self.times == 0:
            return False

        if self.identifier:
            try:
                handler_object = self._func.__self__
            except AttributeError:
                func = self._func()
                if func is None:
                    raise WeakMethodDead
                handler_object = func.__self__
            target_object = kwargs[self.identifier]

            if hasattr(self._func, 'identifier_path'):
                # special case if the signal method defined a user-defined attribute path for fiding the associated identifier
                path = self._func.identifier_path
                handler_object = eval('obj.%s' % path.strip('.'), dict(obj=handler_object), {})
            elif hasattr(handler_object, self.identifier):
                # the identifier exists as an attribute on the handler object, named the same
                handler_object = getattr(handler_object, self.identifier)
            elif isinstance(handler_object, target_object.__class__):
                # the handler_object is itself of the same type as the target_object, so we should regard it is the identifier itself
                pass
            else:
                # the handler_object isn't identifiable, so we'll just pass it the identifier and hope it knows what to do
                handler_object = None

            if handler_object is not None and handler_object != target_object:
                return False
        return True

    def __call__(self, *, swallow_exceptions, **kwargs):
        if self.times is not None:
            self.times -= 1

        try:
            return self.func(**kwargs)
        except:
            if not swallow_exceptions:
                raise
            _logger.silent_exception("Exception in (%s) ignored", self)

    @property
    def bound_object(self):
        if isinstance(self._func, weakref.WeakMethod):
            return weakref.ref.__call__(self._func)
        else:
            return None

class Signal:

    ALL = {}

    def __new__(cls, name, async=None, swallow_exceptions=False, log=True):
        try:
            return cls.ALL[name]
        except KeyError:
            pass
        assert threading.main_thread() is threading.current_thread(), "Can only create Signal objects from the MainThread"
        signal = object.__new__(cls)
        # we use this to track the calling of various callbacks. we use this short id so it doesn't overflow the logs
        signal.id = make_id(name)
        signal.handlers = {priority: [] for priority in PRIORITIES}
        signal.name = name
        signal.swallow_exceptions = swallow_exceptions
        signal.async = async
        signal.identifier = None
        signal.log = log
        return cls.ALL.setdefault(name, signal)

    def iter_handlers(self):
        return chain(*(self.handlers[k] for k in PRIORITIES))

    def remove_handler(self, handler):
        self.handlers[handler.priority].remove(handler)
        _logger.debug("handler removed from '%s' (%s): %s", self.name, handler.priority.name, handler)

    def remove_handlers_if_exist(self, handlers):
        if not handlers:
            return
        handlers = set(handlers)
        for priority in {h.priority for h in handlers}:
            self.handlers[priority][:] = (handler for handler in self.handlers[priority] if handler not in handlers)

    def register(self, func=None, async=None, priority=PRIORITIES.NONE, times=None):
        if not func:
            return functools.partial(self.register, async=async, priority=priority)
        if async is None:
            async = False if self.async is None else self.async
        elif self.async is not None:
            assert self.async == async, "Signal is set with async=%s" % self.async
        handler = SignalHandler(func, async, priority, times=times, identifier=self.identifier)
        self.handlers[priority].append(handler)
        _logger.debug("registered handler for '%s' (%s): %s", self.name, priority.name, handler)
        return func

    def unregister(self, func):
        for handler in self.iter_handlers():
            if func in (
                    handler._func,  # simple case
                    getattr(handler._func, "__wrapped__", None),  # wrapped with @wraps
                    getattr(handler._func, "func", None),  # wrapped with 'partial'
                    handler._func() if isinstance(handler._func, weakref.WeakMethod) else None,
                    ):
                self.remove_handler(handler)

    @contextmanager
    def registered(self, func, **kwargs):
        self.register(func, **kwargs)
        try:
            yield
        finally:
            self.unregister(func)

    def __call__(self, **kwargs):
        if not self.identifier:
            pass
        elif self.identifier in kwargs:
            pass
        else:
            raise MissingIdentifier(_signal_name=self.name, _identifier=self.identifier)

        if self.log:
            # log signal for centralized logging analytics.
            # minimize text message as most of the data is sent in the 'extra' dict
            # signal fields get prefixed with 'sig_', and the values are repr'ed
            signal_fields = {("sig_%s" % k): repr(v) for (k, v) in kwargs.items()}
            _logger.debug("Triggered %s", self, extra=dict(signal_fields, signal=self.name, signal_id=self.id))

        for handler in self.iter_handlers():
            if handler.times == 0:
                self.remove_handler(handler)

        kwargs.setdefault('swallow_exceptions', self.swallow_exceptions)

        with ExitStack() as STACK:
            STACK.enter_context(_logger.context(self.id))

            if any(h.async for h in self.iter_handlers()):
                futures = STACK.enter_context(Futures.executor())
            else:
                futures = None

            handlers_to_remove = []

            def run_handler(handler):
                try:
                    with _logger.context("#%03d" % handler.idx):
                        handler(**kwargs)
                except WeakMethodDead:
                    handlers_to_remove.append(handler)

            for handler in self.iter_handlers():
                try:
                    if not handler.should_run(**kwargs):
                        continue
                except WeakMethodDead:
                    handlers_to_remove.append(handler)
                    continue
                # allow handler to use our async context
                if handler.async:
                    futures.submit(run_handler, handler)
                else:
                    run_handler(handler)

            if futures:
                for future in futures.as_completed():
                    future.result()        # bubble up exceptions

        self.remove_handlers_if_exist(handlers_to_remove)

    def __str__(self):
        return "<Signal %s (%s)>" % (self.name, self.id)

    __repr__ = __str__

@parametrizeable_decorator
def signal_identifier_path(func, path):
    func.identifier_path = path
    return func

class ContextManagerSignal(Signal):
    @contextmanager
    def __call__(self, **kwargs):
        if self.log:
            # log signal for centralized logging analytics.
            # minimize text message as most of the data is sent in the 'extra' dict
            # signal fields get prefixed with 'sig_', and the values are repr'ed
            signal_fields = {("sig_%s" % k): repr(v) for (k, v) in kwargs.items()}
            _logger.debug("Triggered '%s' (%s) - entering", self.name, self.id,
                          extra=dict(signal_fields, signal=self.name, signal_id=self.id))

        for handler in self.iter_handlers():
            if handler.times == 0:
                self.unregister(handler.func)

        kwargs.setdefault('swallow_exceptions', self.swallow_exceptions)

        with ExitStack() as handlers_stack:
            async_handlers = MultiObject()
            handlers = []

            handlers_to_remove = []

            @contextmanager
            def run_handler(handler):
                already_yielded = False
                try:
                    with _logger.context(self.id), _logger.context("%02d" % index):
                        with handler(**kwargs):
                            yield
                            already_yielded = True
                except WeakMethodDead:
                    handlers_to_remove.append(handler)
                    if not already_yielded:
                        yield

            for index, handler in enumerate(self.iter_handlers()):
                try:
                    if not handler.should_run(**kwargs):
                        continue
                except WeakMethodDead:
                    handlers_to_remove.append(handler)
                    continue
                # allow handler to use our async context
                if handler.async:
                    async_handlers.append(handler)
                else:
                    handlers.append(handler)
            if async_handlers:
                handlers_stack.enter_context(async_handlers.call(run_handler))
            for handler in handlers:
                res = run_handler(handler)
                if res:
                    handlers_stack.enter_context(res)
            yield

        self.remove_handlers_if_exist(handlers_to_remove)

####################################
def register_signal(name, func, **kwargs):
    signal = Signal(name)
    signal.register(func, **kwargs)
    return functools.partial(signal.unregister, func)

def unregister_signal(name, func):
    try:
        signal = Signal.ALL[name]
    except KeyError:
        return
    return signal.unregister(func)

@parametrizeable_decorator
def register(func, **kwargs):
    register_signal(func.__name__, func, **kwargs)
    return func

def unregister(func):
    return unregister_signal(func.__name__, func)

def _set_handler_params(**kw):
    def inner(func):
        params = getattr(func, '_signal_handler_params', {})
        params.update(kw)
        func._signal_handler_params = params
        return func
    return inner

run_first = _set_handler_params(priority=PRIORITIES.FIRST)
run_last = _set_handler_params(priority=PRIORITIES.LAST)
run_async = _set_handler_params(async=True)
run_sync = _set_handler_params(async=False)

@functools.lru_cache(None)
def get_signals_for_type(typ):
    return {n for typ in inspect.getmro(typ)
            for n in dir(typ) if n.startswith("on_")}

class __FakeCode:
    def __init__(self, fake_filename, fake_firstlineno):
        self.co_filename = fake_filename
        self.co_firstlineno = fake_firstlineno

def __fake_meth_type(func, obj, name, code):
    method = func.__get__(obj, type(obj))
    method.__name__ = name
    method.__code__ = code
    return method

def register_object(obj, **kwargs):
    for method_name in get_signals_for_type(type(obj)):
        method = getattr(obj, method_name)
        assert callable(method), "'%s' (%r) is not callable" % (method, obj)

        # Don't use static/class methods for automatic event registration - they
        # could be registered multiple times if the are multiple objects of that
        # type in the system, or not at all if there are no objects of that type
        # in the system.
        assert method is not getattr(type(obj), method_name, None), "'%s' is a static/class method" % method

        params = getattr(method, '_signal_handler_params', {})
        intersection = set(params).intersection(kwargs)
        assert not intersection, "parameter conflict in signal object registration (%s)" % (intersection)
        params.update(kwargs)

        method_name, *_ = method_name.partition("__")  # allows multiple methods for the same signal

        if not inspect.ismethod(method):
            # Fake method attributes for WeakMethod
            method.__self__ = obj
            method.__func__ = getattr(type(obj), method_name)
            if not hasattr(method, '__code__'):
                method.__code__ = __FakeCode('', '')
            fake_meth_type = functools.partial(
                __fake_meth_type,
                # Best effort: fake name and code for SignalHandler. We don't know the line number.
                name=method_name,
                code=__FakeCode(type(obj).__module__, '?'))

        method = weakref.WeakMethod(method)
        try:
            method._meth_type = fake_meth_type
        except UnboundLocalError:
            pass

        register_signal(method_name, method, **params)

def unregister_object(obj):
    for signal_name in {method_name.split('__', 1)[0] for method_name in get_signals_for_type(type(obj))}:
        for handlers in Signal.ALL[signal_name].handlers.values():
            handlers[:] = (handler for handler in handlers if handler.bound_object is not obj)

def call_signal(name, **kwargs):
    return Signal(name)(**kwargs)

def log_all_signal_ids(logger=_logger, level=logging.DEBUG):
    for signal in sorted(Signal.ALL.itervalues(), key=lambda signal: signal.name):
        logger.log(level, "%s - '%s'", signal.id, signal.name)

# ===================================================================================================
# Module hack: ``from easypy.signals import on_some_signal``
#              ``from easypy.signals import on_ctx_some_signal`` for a context-manager signal
# ===================================================================================================
def _hack():
    import sys
    from types import ModuleType
    this = sys.modules[__name__]

    class SignalsModule(ModuleType):
        """The module-hack that allows us to use ``from easypy.signals import on_some_signal``"""
        __all__ = ()  # to make help() happy
        __package__ = __name__
        __path__ = []
        __file__ = __file__

        def __getattr__(self, name):
            try:
                return getattr(this, name)
            except AttributeError:
                if name.startswith("on_ctx_"):
                    return ContextManagerSignal(name)
                elif name.startswith("on_"):
                    return Signal(name)
                raise

        def __dir__(self):
            return sorted(dir(this) + list(Signal.ALL))

    module = SignalsModule(__name__, SignalsModule.__doc__)
    sys.modules[module.__name__] = module

_hack()
<|endoftext|>


<commit_before>"""
Logging configuration.

Mostly based off http://www.structlog.org/en/0.4.1/standard-library.html.

"""
import sys
import traceback
import logging
import logging.handlers

import raven
import raven.processors
import colorlog
import structlog
from structlog.threadlocal import wrap_dict

from inbox.config import config
import gevent

MAX_EXCEPTION_LENGTH = 10000

def _find_first_app_frame_and_name(ignores=None):
    """
    Remove ignorable calls and return the relevant app frame. Borrowed from
    structlog, but fixes an issue when the stack includes an 'exec' statement
    or similar (f.f_globals doesn't have a '__name__' key in that case).

    Parameters
    ----------
    ignores: list, optional
        Additional names with which the first frame must not start.

    Returns
    -------
    tuple of (frame, name)
    """
    ignores = ignores or []
    f = sys._getframe()
    name = f.f_globals.get('__name__')
    while f is not None and (name is None or
                             any(name.startswith(i) for i in ignores)):
        f = f.f_back
        name = f.f_globals.get('__name__')
    return f, name

def _record_level(logger, name, event_dict):
    """Processor that records the log level ('info', 'warning', etc.) in the
    structlog event dictionary."""
    event_dict['level'] = name
    return event_dict

def _record_module(logger, name, event_dict):
    """Processor that records the module and line where the logging call was
    invoked."""
    f, name = _find_first_app_frame_and_name(
        ignores=['structlog', 'inbox.log', 'inbox.sqlalchemy_ext.util',
                 'inbox.models.session', 'sqlalchemy'])
    event_dict['module'] = '{}:{}'.format(name, f.f_lineno)
    return event_dict

def _format_string_renderer(_, __, event_dict):
    """Processor to be used with the BoundLogger class below to properly handle
    messages of the form
    `log.info('some message to format %s', some_value')`."""
    positional_args = event_dict.get('_positional_args')
    if positional_args:
        event_dict['event'] = event_dict['event'] % positional_args
        del event_dict['_positional_args']
    return event_dict

def _safe_exc_info_renderer(_, __, event_dict):
    """Processor that formats exception info safely."""
    exc_info = event_dict.pop('exc_info', None)
    if exc_info:
        if not isinstance(exc_info, tuple):
            exc_info = sys.exc_info()
        event_dict['exception'] = safe_format_exception(*exc_info)
    return event_dict

class BoundLogger(structlog._base.BoundLoggerBase):
    """Adaptation of structlog.stdlib.BoundLogger to accept positional
    arguments. See https://github.com/hynek/structlog/pull/23/
    (we can remove this if that ever gets merged)."""
    def debug(self, event=None, *args, **kw):
        return self._proxy_to_logger('debug', event, *args, **kw)

    def info(self, event=None, *args, **kw):
        return self._proxy_to_logger('info', event, *args, **kw)

    def warning(self, event=None, *args, **kw):
        return self._proxy_to_logger('warning', event, *args, **kw)

    warn = warning

    def error(self, event=None, *args, **kw):
        return self._proxy_to_logger('error', event, *args, **kw)

    def critical(self, event=None, *args, **kw):
        return self._proxy_to_logger('critical', event, *args, **kw)

    def exception(self, event=None, *args, **kw):
        kw['exc_info'] = True
        return self._proxy_to_logger('error', event, *args, **kw)

    def _proxy_to_logger(self, method_name, event=None, *event_args,
                         **event_kw):
        if event_args:
            event_kw['_positional_args'] = event_args
        event_kw['greenlet_id'] = id(gevent.getcurrent())
        return super(BoundLogger, self)._proxy_to_logger(method_name, event,
                                                         **event_kw)

structlog.configure(
    processors=[
        structlog.stdlib.filter_by_level,
        structlog.processors.TimeStamper(fmt='iso', utc=True),
        structlog.processors.StackInfoRenderer(),
        _safe_exc_info_renderer,
        _record_module,
        _record_level,
        _format_string_renderer,
        structlog.processors.JSONRenderer(),
    ],
    context_class=wrap_dict(dict),
    logger_factory=structlog.stdlib.LoggerFactory(),
    wrapper_class=BoundLogger,
    cache_logger_on_first_use=True,
)
get_logger = structlog.get_logger

def configure_logging(is_prod=False):
    # The is_prod argument is ignored and only retained for compatibility.
    tty_handler = logging.StreamHandler(sys.stdout)
    if sys.stdout.isatty():
        # Use a more human-friendly format.
        formatter = colorlog.ColoredFormatter(
            '%(log_color)s[%(levelname)s]%(reset)s %(message)s',
            reset=True, log_colors={'DEBUG': 'cyan', 'INFO': 'green',
                                    'WARNING': 'yellow', 'ERROR': 'red',
                                    'CRITICAL': 'red'})
    else:
        formatter = logging.Formatter('%(message)s')
    tty_handler.setFormatter(formatter)
    tty_handler._inbox = True

    # Configure the root logger.
    root_logger = logging.getLogger()
    for handler in root_logger.handlers:
        # If the handler was previously installed, remove it so that repeated
        # calls to configure_logging() are idempotent.
        if getattr(handler, '_inbox', False):
            root_logger.removeHandler(handler)
    root_logger.addHandler(tty_handler)
    # Set loglevel DEBUG if config value is missing.
    root_logger.setLevel(config.get('LOGLEVEL', 10))

_sentry_client = None

def get_sentry_client():
    global sentry_client
    if _sentry_client is None:
        sentry_dsn = config.get_required('SENTRY_DSN')
        return raven.Client(
            sentry_dsn,
            processors=('inbox.log.TruncatingProcessor',))
    return _sentry_client

def safe_format_exception(etype, value, tb, limit=None):
    """Similar to structlog._format_exception, but truncate the exception part.
    This is because SQLAlchemy exceptions can sometimes have ludicrously large
    exception strings."""
    if tb:
        list = ['Traceback (most recent call last):\n']
        list = list + traceback.format_tb(tb, limit)
    else:
        list = []
    exc_only = traceback.format_exception_only(etype, value)
    # Normally exc_only is a list containing a single string.  For syntax
    # errors it may contain multiple elements, but we don't really need to
    # worry about that here.
    exc_only[0] = exc_only[0][:MAX_EXCEPTION_LENGTH]
    list = list + exc_only
    return '\t'.join(list)

class TruncatingProcessor(raven.processors.Processor):
    """Truncates the exception value string, and strips stack locals.
    Sending stack locals could potentially leak information."""

    # A whitelist of locals we don't want to strip.
    # They must be non-PII!
    locals_whitelist = ['account_id', 'message_id']

    # Note(emfree): raven.processors.Processor provides a filter_stacktrace
    # method to implement, but won't actually call it correctly. We can
    # simplify this if it gets fixed upstream.
    def process(self, data, **kwargs):
        if 'exception' not in data:
            return data
        if 'values' not in data['exception']:
            return data
        for item in data['exception']['values']:
            item['value'] = item['value'][:MAX_EXCEPTION_LENGTH]
            stacktrace = item.get('stacktrace')
            if stacktrace is not None:
                if 'frames' in stacktrace:
                    for frame in stacktrace['frames']:
                        for stack_local in frame['vars'].keys():
                            if stack_local not in self.locals_whitelist:
                                frame['vars'].pop(stack_local)
        return data

def sentry_alert(*args, **kwargs):
    if config.get('SENTRY_EXCEPTIONS'):
        get_sentry_client().captureException(*args, **kwargs)

def log_uncaught_errors(logger=None, account_id=None):
    """
    Helper to log uncaught exceptions.

    Parameters
    ----------
    logger: structlog.BoundLogger, optional
        The logging object to write to.
    """
    logger = logger or get_logger()
    logger.error('Uncaught error', exc_info=True)
    user_data = {'account_id': account_id}
    sentry_alert(extra=user_data)
<commit_msg>Optional action_id for error logging.<commit_after>"""
Logging configuration.

Mostly based off http://www.structlog.org/en/0.4.1/standard-library.html.

"""
import sys
import traceback
import logging
import logging.handlers

import raven
import raven.processors
import colorlog
import structlog
from structlog.threadlocal import wrap_dict

from inbox.config import config
import gevent

MAX_EXCEPTION_LENGTH = 10000

def _find_first_app_frame_and_name(ignores=None):
    """
    Remove ignorable calls and return the relevant app frame. Borrowed from
    structlog, but fixes an issue when the stack includes an 'exec' statement
    or similar (f.f_globals doesn't have a '__name__' key in that case).

    Parameters
    ----------
    ignores: list, optional
        Additional names with which the first frame must not start.

    Returns
    -------
    tuple of (frame, name)
    """
    ignores = ignores or []
    f = sys._getframe()
    name = f.f_globals.get('__name__')
    while f is not None and (name is None or
                             any(name.startswith(i) for i in ignores)):
        f = f.f_back
        name = f.f_globals.get('__name__')
    return f, name

def _record_level(logger, name, event_dict):
    """Processor that records the log level ('info', 'warning', etc.) in the
    structlog event dictionary."""
    event_dict['level'] = name
    return event_dict

def _record_module(logger, name, event_dict):
    """Processor that records the module and line where the logging call was
    invoked."""
    f, name = _find_first_app_frame_and_name(
        ignores=['structlog', 'inbox.log', 'inbox.sqlalchemy_ext.util',
                 'inbox.models.session', 'sqlalchemy'])
    event_dict['module'] = '{}:{}'.format(name, f.f_lineno)
    return event_dict

def _format_string_renderer(_, __, event_dict):
    """Processor to be used with the BoundLogger class below to properly handle
    messages of the form
    `log.info('some message to format %s', some_value')`."""
    positional_args = event_dict.get('_positional_args')
    if positional_args:
        event_dict['event'] = event_dict['event'] % positional_args
        del event_dict['_positional_args']
    return event_dict

def _safe_exc_info_renderer(_, __, event_dict):
    """Processor that formats exception info safely."""
    exc_info = event_dict.pop('exc_info', None)
    if exc_info:
        if not isinstance(exc_info, tuple):
            exc_info = sys.exc_info()
        event_dict['exception'] = safe_format_exception(*exc_info)
    return event_dict

class BoundLogger(structlog._base.BoundLoggerBase):
    """Adaptation of structlog.stdlib.BoundLogger to accept positional
    arguments. See https://github.com/hynek/structlog/pull/23/
    (we can remove this if that ever gets merged)."""
    def debug(self, event=None, *args, **kw):
        return self._proxy_to_logger('debug', event, *args, **kw)

    def info(self, event=None, *args, **kw):
        return self._proxy_to_logger('info', event, *args, **kw)

    def warning(self, event=None, *args, **kw):
        return self._proxy_to_logger('warning', event, *args, **kw)

    warn = warning

    def error(self, event=None, *args, **kw):
        return self._proxy_to_logger('error', event, *args, **kw)

    def critical(self, event=None, *args, **kw):
        return self._proxy_to_logger('critical', event, *args, **kw)

    def exception(self, event=None, *args, **kw):
        kw['exc_info'] = True
        return self._proxy_to_logger('error', event, *args, **kw)

    def _proxy_to_logger(self, method_name, event=None, *event_args,
                         **event_kw):
        if event_args:
            event_kw['_positional_args'] = event_args
        event_kw['greenlet_id'] = id(gevent.getcurrent())
        return super(BoundLogger, self)._proxy_to_logger(method_name, event,
                                                         **event_kw)

structlog.configure(
    processors=[
        structlog.stdlib.filter_by_level,
        structlog.processors.TimeStamper(fmt='iso', utc=True),
        structlog.processors.StackInfoRenderer(),
        _safe_exc_info_renderer,
        _record_module,
        _record_level,
        _format_string_renderer,
        structlog.processors.JSONRenderer(),
    ],
    context_class=wrap_dict(dict),
    logger_factory=structlog.stdlib.LoggerFactory(),
    wrapper_class=BoundLogger,
    cache_logger_on_first_use=True,
)
get_logger = structlog.get_logger

def configure_logging(is_prod=False):
    # The is_prod argument is ignored and only retained for compatibility.
    tty_handler = logging.StreamHandler(sys.stdout)
    if sys.stdout.isatty():
        # Use a more human-friendly format.
        formatter = colorlog.ColoredFormatter(
            '%(log_color)s[%(levelname)s]%(reset)s %(message)s',
            reset=True, log_colors={'DEBUG': 'cyan', 'INFO': 'green',
                                    'WARNING': 'yellow', 'ERROR': 'red',
                                    'CRITICAL': 'red'})
    else:
        formatter = logging.Formatter('%(message)s')
    tty_handler.setFormatter(formatter)
    tty_handler._inbox = True

    # Configure the root logger.
    root_logger = logging.getLogger()
    for handler in root_logger.handlers:
        # If the handler was previously installed, remove it so that repeated
        # calls to configure_logging() are idempotent.
        if getattr(handler, '_inbox', False):
            root_logger.removeHandler(handler)
    root_logger.addHandler(tty_handler)
    # Set loglevel DEBUG if config value is missing.
    root_logger.setLevel(config.get('LOGLEVEL', 10))

_sentry_client = None

def get_sentry_client():
    global sentry_client
    if _sentry_client is None:
        sentry_dsn = config.get_required('SENTRY_DSN')
        return raven.Client(
            sentry_dsn,
            processors=('inbox.log.TruncatingProcessor',))
    return _sentry_client

def safe_format_exception(etype, value, tb, limit=None):
    """Similar to structlog._format_exception, but truncate the exception part.
    This is because SQLAlchemy exceptions can sometimes have ludicrously large
    exception strings."""
    if tb:
        list = ['Traceback (most recent call last):\n']
        list = list + traceback.format_tb(tb, limit)
    else:
        list = []
    exc_only = traceback.format_exception_only(etype, value)
    # Normally exc_only is a list containing a single string.  For syntax
    # errors it may contain multiple elements, but we don't really need to
    # worry about that here.
    exc_only[0] = exc_only[0][:MAX_EXCEPTION_LENGTH]
    list = list + exc_only
    return '\t'.join(list)

class TruncatingProcessor(raven.processors.Processor):
    """Truncates the exception value string, and strips stack locals.
    Sending stack locals could potentially leak information."""

    # A whitelist of locals we don't want to strip.
    # They must be non-PII!
    locals_whitelist = ['account_id', 'message_id']

    # Note(emfree): raven.processors.Processor provides a filter_stacktrace
    # method to implement, but won't actually call it correctly. We can
    # simplify this if it gets fixed upstream.
    def process(self, data, **kwargs):
        if 'exception' not in data:
            return data
        if 'values' not in data['exception']:
            return data
        for item in data['exception']['values']:
            item['value'] = item['value'][:MAX_EXCEPTION_LENGTH]
            stacktrace = item.get('stacktrace')
            if stacktrace is not None:
                if 'frames' in stacktrace:
                    for frame in stacktrace['frames']:
                        for stack_local in frame['vars'].keys():
                            if stack_local not in self.locals_whitelist:
                                frame['vars'].pop(stack_local)
        return data

def sentry_alert(*args, **kwargs):
    if config.get('SENTRY_EXCEPTIONS'):
        get_sentry_client().captureException(*args, **kwargs)

def log_uncaught_errors(logger=None, account_id=None, action_id=None):
    """
    Helper to log uncaught exceptions.

    Parameters
    ----------
    logger: structlog.BoundLogger, optional
        The logging object to write to.

    """
    logger = logger or get_logger()
    logger.error('Uncaught error', exc_info=True, action_id=action_id)
    user_data = {'account_id': account_id, 'action_id': action_id}
    sentry_alert(extra=user_data)
<|endoftext|>


<commit_before>from copy import copy
from tri.declarative import creation_ordered, declarative
from tri.struct import Struct, FrozenStruct

__version__ = '0.7.0'

@creation_ordered
class NamedStructField(object):
    """
    Field declaration for :code:`NamedStruct` classes
    """

    def __init__(self, default=None):
        self.default = default
        super(NamedStructField, self).__init__()

def _get_declared(self):
    # Fancy getter to not stumble over our own __getitem__ implementations
    return object.__getattribute__(self, '__class__').get_declared()

def _build_kwargs(self, args, kwargs):
    members = _get_declared(self)

    if len(args) > len(members):
        raise TypeError("%s() takes at most %d arguments (%d given)" % (self.__class__.__name__, len(members), len(args)))

    values_by_name = dict(zip(members.keys(), args))
    for kwargs_name, value in kwargs.items():
        if kwargs_name in values_by_name:
            raise TypeError("%s() got multiple values for keyword argument '%s'" % (self.__class__.__name__, kwargs_name, ))
        values_by_name[kwargs_name] = value

    for kwargs_name in values_by_name:
        if kwargs_name not in members:
            raise TypeError("%s() got an unexpected keyword argument '%s'" % (self.__class__.__name__, kwargs_name))

    return {name: values_by_name.get(name, copy(field.default)) for name, field in members.items()}

@declarative(NamedStructField, add_init_kwargs=False)
class NamedStruct(Struct):
    """
    Class extending :code:`tri.struct.Struct` to only allow a defined subset of string keys.
    """

    def __init__(self, *args, **kwargs):
        super(NamedStruct, self).__init__(**_build_kwargs(self, args, kwargs))

    def __getitem__(self, key):
        if key not in _get_declared(self):
            raise KeyError(key)
        return super(NamedStruct, self).__getitem__(key)

    def __setitem__(self, key, value):
        if key not in _get_declared(self):
            raise KeyError(key)
        super(NamedStruct, self).__setitem__(key, value)

    def __setattr__(self, k, v):
        try:
            self[k] = v
        except KeyError:
            raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, k))

def named_struct(field_names, typename="NamedStruct"):
    """
    Procedural way to define a :code:`NamedStruct` subclass, similar to the :code:`named_tuple` builtin.
    """
    return _build_named_struct(NamedStruct, field_names, typename)

@declarative(NamedStructField, add_init_kwargs=False)
class NamedFrozenStruct(FrozenStruct):
    """
    Class extending :code:`tri.struct.FrozenStruct` to only allow a defined subset of string keys.
    """

    def __init__(self, *args, **kwargs):
        super(NamedFrozenStruct, self).__init__(**_build_kwargs(self, args, kwargs))

    def __getitem__(self, key):
        if key not in _get_declared(self):
            raise KeyError(key)
        return super(NamedFrozenStruct, self).__getitem__(key)

def named_frozen_struct(field_names, typename='FrozenNamedStruct'):
    """
    Procedural way to define a :code:`FrozenNamedStruct` subclass, similar to the :code:`named_tuple` builtin.
    """
    return _build_named_struct(NamedFrozenStruct, field_names, typename)

def _build_named_struct(base, field_names, typename):
    if isinstance(field_names, str):
        field_names = field_names.replace(',', ' ').split()
    field_names = map(str, field_names)
    typename = str(typename)

    return type(typename, (base, ), {field: NamedStructField() for field in field_names})
<commit_msg>Refactor to use Frozen mixin<commit_after>from copy import copy
from tri.declarative import creation_ordered, declarative
from tri.struct import Struct, Frozen

__version__ = '0.7.0'

@creation_ordered
class NamedStructField(object):
    """
    Field declaration for :code:`NamedStruct` classes
    """

    def __init__(self, default=None):
        self.default = default
        super(NamedStructField, self).__init__()

def _get_declared(self):
    # Fancy getter to not stumble over our own __getitem__ implementations
    return object.__getattribute__(self, '__class__').get_declared()

def _build_kwargs(self, args, kwargs):
    members = _get_declared(self)

    if len(args) > len(members):
        raise TypeError("%s() takes at most %d arguments (%d given)" % (self.__class__.__name__, len(members), len(args)))

    values_by_name = dict(zip(members.keys(), args))
    for kwargs_name, value in kwargs.items():
        if kwargs_name in values_by_name:
            raise TypeError("%s() got multiple values for keyword argument '%s'" % (self.__class__.__name__, kwargs_name, ))
        values_by_name[kwargs_name] = value

    for kwargs_name in values_by_name:
        if kwargs_name not in members:
            raise TypeError("%s() got an unexpected keyword argument '%s'" % (self.__class__.__name__, kwargs_name))

    return {name: values_by_name.get(name, copy(field.default)) for name, field in members.items()}

@declarative(NamedStructField, add_init_kwargs=False)
class NamedStruct(Struct):
    """
    Class extending :code:`tri.struct.Struct` to only allow a defined subset of string keys.
    """
    __slots__ = ()

    def __init__(self, *args, **kwargs):
        super(NamedStruct, self).__init__(**_build_kwargs(self, args, kwargs))

    def __getitem__(self, key):
        if key not in _get_declared(self):
            raise KeyError(key)
        return super(NamedStruct, self).__getitem__(key)

    def __setitem__(self, key, value):
        if key not in _get_declared(self):
            raise KeyError(key)
        super(NamedStruct, self).__setitem__(key, value)

    def __setattr__(self, k, v):
        try:
            self[k] = v
        except KeyError:
            raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, k))

def named_struct(field_names, typename="NamedStruct"):
    """
    Procedural way to define a :code:`NamedStruct` subclass, similar to the :code:`named_tuple` builtin.
    """
    return _build_named_struct(NamedStruct, field_names, typename)

@declarative(NamedStructField, add_init_kwargs=False)
class NamedFrozenStruct(Frozen, NamedStruct):
    """
    Class extending :code:`tri.struct.FrozenStruct` to only allow a defined subset of string keys.
    """
    pass

def named_frozen_struct(field_names, typename='FrozenNamedStruct'):
    """
    Procedural way to define a :code:`FrozenNamedStruct` subclass, similar to the :code:`named_tuple` builtin.
    """
    return _build_named_struct(NamedFrozenStruct, field_names, typename)

def _build_named_struct(base, field_names, typename):
    if isinstance(field_names, str):
        field_names = field_names.replace(',', ' ').split()
    field_names = map(str, field_names)
    typename = str(typename)

    return type(typename, (base, ), {field: NamedStructField() for field in field_names})
<|endoftext|>


<commit_before>from __future__ import print_function, division, absolute_import

import unittest
import gitnl

class GitnlTestCase(unittest.TestCase):
    """Tests from 'gitnl.py'."""

    def test_push_remotename_branchfrom(self):
        desired = 'push remotename branchfrom'
        actual = gitnl.parse_to_git('push my branch branchfrom to a remote called remotename')
        self.assertEqual(actual, desired)

    def test_rename_branch(self):
        desired = 'branch -m old_branch new_branch'
        actual = gitnl.parse_to_git('branch rename branch old_branch to new_branch')
        self.assertEqual(actual, desired)

if __name__ == '__main__':
    unittest.main()
<commit_msg>rename method<commit_after>from __future__ import print_function, division, absolute_import

import unittest
import gitnl

class GitnlTestCase(unittest.TestCase):
    """Tests from 'gitnl.py'."""

    def test_push_remotename_branchfrom(self):
        desired = 'push remotename branchfrom'
        actual = gitnl.parse_to_git('push my branch branchfrom to a remote called remotename')
        self.assertEqual(actual, desired)

    def test_rename_branch_locally(self):
        desired = 'branch -m old_branch new_branch'
        actual = gitnl.parse_to_git('branch rename branch old_branch to new_branch')
        self.assertEqual(actual, desired)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>