<commit_before># -*- coding: UTF-8 -*-
"""
Tests for Cloudwatch to Graphite (leadbutt)

WISHLIST: supress chatty stderr and stdout in tests
"""
from __future__ import unicode_literals

from subprocess import call
import datetime
import os
import unittest

import mock

import leadbutt

class get_configTest(unittest.TestCase):
    def test_example_config_loads(self):
        config = leadbutt.get_config('config.yaml.example')
        self.assertIn('Metrics', config)

    @mock.patch('sys.stdin')
    def test_config_can_be_stdin(self, mock_stdin):
        # simulate reading stdin
        mock_stdin.read.side_effect = ['test: "123"\n', '']
        # mock_stdin.name = 'oops'
        config = leadbutt.get_config('-')
        self.assertIn('test', config)

    @mock.patch('sys.stderr')
    @mock.patch('sys.stdin')
    def test_config_handles_malformed_yaml(self, mock_stdin, mock_stderr):
        mock_stdin.read.side_effect = ['-\nmalformed yaml', '']
        mock_stdin.name = 'oops'
        with self.assertRaises(SystemExit) as e:
            leadbutt.get_config('-')
        self.assertEqual(e.exception.code, 1)
        self.assertTrue(mock_stderr.write.called)

    @mock.patch('sys.stderr')
    def test_config_handles_missing_file(self, mock_stderr):
        with self.assertRaises(SystemExit) as e:
            leadbutt.get_config('whatever_the_default_config_is')
        self.assertEqual(e.exception.code, 2)
        self.assertTrue(mock_stderr.write.called)

class get_optionsTest(unittest.TestCase):
    def test_get_options_returns_right_option(self):
        # only have the defaults
        options = leadbutt.get_options(None, None, None)
        self.assertEqual(options['Period'], 1)
        self.assertEqual(options['Count'], 5)

        # config options were specified
        config_options = {
            'Period': 2,
        }
        options = leadbutt.get_options(config_options, None, None)
        self.assertEqual(options['Period'], 2)
        self.assertEqual(options['Count'], 5)

        # local_options were specified
        local_options = {
            'Period': 3,
        }
        options = leadbutt.get_options(config_options, local_options, None)
        self.assertEqual(options['Period'], 3)
        self.assertEqual(options['Count'], 5)

        # cli_options were specified
        cli_options = {
            'Period': 4,
            'Count': 10,
        }
        options = leadbutt.get_options(config_options, local_options, cli_options)
        self.assertEqual(options['Period'], 4)
        self.assertEqual(options['Count'], 10)

class output_resultsTest(unittest.TestCase):
    @mock.patch('sys.stdout')
    def test_default_formatter_used(self, mock_sysout):
        mock_results = [{
            'Timestamp': datetime.datetime.utcnow(),
            'Unit': 'Count',
            'Sum': 1337.0,
        }]
        metric = {
            'Namespace': 'AWS/Foo',
            'MetricName': 'RequestCount',
            'Statistics': 'Sum',
            'Unit': 'Count',
            'Dimensions': {'Krang': 'X'},
        }
        options = leadbutt.get_options(None, metric.get('Options'), None)
        leadbutt.output_results(mock_results, metric, options)
        self.assertTrue(mock_sysout.write.called)
        out = mock_sysout.write.call_args[0][0]
        name, value, timestamp = out.split()
        # assert default formatter was used
        self.assertEqual(name, 'cloudwatch.aws.foo.x.requestcount.sum.count')
        self.assertEqual(value, '1337.0')

    @mock.patch('sys.stdout')
    def test_custom_formatter_used(self, mock_sysout):
        mock_results = [{
            'Timestamp': datetime.datetime.utcnow(),
            'Unit': 'Count',
            'Sum': 1337.0,
        }]
        metric = {
            'Namespace': 'AWS/Foo',
            'MetricName': 'RequestCount',
            'Statistics': 'Sum',
            'Unit': 'Count',
            'Dimensions': {'Krang': 'X'},
            'Options': {'Formatter': 'tmnt.%(dimension)s'}
        }
        options = leadbutt.get_options(None, metric.get('Options'), None)
        leadbutt.output_results(mock_results, metric, options)
        self.assertTrue(mock_sysout.write.called)
        out = mock_sysout.write.call_args[0][0]
        name, value, timestamp = out.split()
        # assert custom formatter was used
        self.assertEqual(name, 'tmnt.x')
        self.assertEqual(value, '1337.0')

    @mock.patch('sys.stdout')
    def test_multiple_statistics_get_multiple_lines(self, mock_sysout):
        mock_results = [{
            'Timestamp': datetime.datetime.utcnow(),
            'Maximum': 9001.0,
            'Average': 1337.0,
            'Unit': 'Count',
        }]
        metric = {
            'Namespace': 'AWS/Foo',
            'MetricName': 'RequestCount',
            'Statistics': ['Maximum', 'Average'],
            'Unit': 'Count',
            'Dimensions': {'Krang': 'X'},
        }
        options = leadbutt.get_options(None, metric.get('Options'), None)
        leadbutt.output_results(mock_results, metric, options)

        self.assertEqual(
            mock_sysout.write.call_count, len(metric['Statistics']))

class leadbuttTest(unittest.TestCase):
    @mock.patch('boto.ec2.cloudwatch.connect_to_region')
    @mock.patch('leadbutt.get_config')
    def test_can_get_auth_from_config(self, mock_get_config, mock_connect):
        mock_get_config.return_value = {
            'Metrics': [],
            'Auth': {
                'aws_access_key_id': 'foo',
                'aws_secret_access_key': 'bar',
            }
        }
        leadbutt.leadbutt('dummy_config_file', 'dummy_cli_options')
        self.assertTrue(mock_connect.called)
        args, kwargs = mock_connect.call_args
        self.assertEqual(kwargs['aws_access_key_id'], 'foo')
        self.assertEqual(kwargs['aws_secret_access_key'], 'bar')

@unittest.skipUnless('TOX_TEST_ENTRYPOINT' in os.environ,
    'This is only applicable if leadbutt is installed')
class mainTest(unittest.TestCase):
    def test_entry_point(self):
        # assert this does not raise an exception
        call(['leadbutt', '--help'])

if __name__ == '__main__':
    unittest.main()
<commit_msg>Improve mock data in a test.<commit_after># -*- coding: UTF-8 -*-
"""
Tests for Cloudwatch to Graphite (leadbutt)

WISHLIST: supress chatty stderr and stdout in tests
"""
from __future__ import unicode_literals

from subprocess import call
import datetime
import os
import unittest

import mock

import leadbutt

class get_configTest(unittest.TestCase):
    def test_example_config_loads(self):
        config = leadbutt.get_config('config.yaml.example')
        self.assertIn('Metrics', config)

    @mock.patch('sys.stdin')
    def test_config_can_be_stdin(self, mock_stdin):
        # simulate reading stdin
        mock_stdin.read.side_effect = ['test: "123"\n', '']
        # mock_stdin.name = 'oops'
        config = leadbutt.get_config('-')
        self.assertIn('test', config)

    @mock.patch('sys.stderr')
    @mock.patch('sys.stdin')
    def test_config_handles_malformed_yaml(self, mock_stdin, mock_stderr):
        mock_stdin.read.side_effect = ['-\nmalformed yaml', '']
        mock_stdin.name = 'oops'
        with self.assertRaises(SystemExit) as e:
            leadbutt.get_config('-')
        self.assertEqual(e.exception.code, 1)
        self.assertTrue(mock_stderr.write.called)

    @mock.patch('sys.stderr')
    def test_config_handles_missing_file(self, mock_stderr):
        with self.assertRaises(SystemExit) as e:
            leadbutt.get_config('whatever_the_default_config_is')
        self.assertEqual(e.exception.code, 2)
        self.assertTrue(mock_stderr.write.called)

class get_optionsTest(unittest.TestCase):
    def test_get_options_returns_right_option(self):
        # only have the defaults
        options = leadbutt.get_options(None, None, None)
        self.assertEqual(options['Period'], 1)
        self.assertEqual(options['Count'], 5)

        # config options were specified
        config_options = {
            'Period': 2,
        }
        options = leadbutt.get_options(config_options, None, None)
        self.assertEqual(options['Period'], 2)
        self.assertEqual(options['Count'], 5)

        # local_options were specified
        local_options = {
            'Period': 3,
        }
        options = leadbutt.get_options(config_options, local_options, None)
        self.assertEqual(options['Period'], 3)
        self.assertEqual(options['Count'], 5)

        # cli_options were specified
        cli_options = {
            'Period': 4,
            'Count': 10,
        }
        options = leadbutt.get_options(config_options, local_options, cli_options)
        self.assertEqual(options['Period'], 4)
        self.assertEqual(options['Count'], 10)

class output_resultsTest(unittest.TestCase):
    @mock.patch('sys.stdout')
    def test_default_formatter_used(self, mock_sysout):
        mock_results = [{
            'Timestamp': datetime.datetime.utcnow(),
            'Unit': 'Count',
            'Sum': 1337.0,
        }]
        metric = {
            'Namespace': 'AWS/Foo',
            'MetricName': 'RequestCount',
            'Statistics': 'Sum',
            'Unit': 'Count',
            'Dimensions': {'Krang': 'X'},
        }
        options = leadbutt.get_options(None, metric.get('Options'), None)
        leadbutt.output_results(mock_results, metric, options)
        self.assertTrue(mock_sysout.write.called)
        out = mock_sysout.write.call_args[0][0]
        name, value, timestamp = out.split()
        # assert default formatter was used
        self.assertEqual(name, 'cloudwatch.aws.foo.x.requestcount.sum.count')
        self.assertEqual(value, '1337.0')

    @mock.patch('sys.stdout')
    def test_custom_formatter_used(self, mock_sysout):
        mock_results = [{
            'Timestamp': datetime.datetime.utcnow(),
            'Unit': 'Count',
            'Sum': 1337.0,
        }]
        metric = {
            'Namespace': 'AWS/Foo',
            'MetricName': 'RequestCount',
            'Statistics': 'Sum',
            'Unit': 'Count',
            'Dimensions': {'Krang': 'X'},
            'Options': {'Formatter': 'tmnt.%(dimension)s'}
        }
        options = leadbutt.get_options(None, metric.get('Options'), None)
        leadbutt.output_results(mock_results, metric, options)
        self.assertTrue(mock_sysout.write.called)
        out = mock_sysout.write.call_args[0][0]
        name, value, timestamp = out.split()
        # assert custom formatter was used
        self.assertEqual(name, 'tmnt.x')
        self.assertEqual(value, '1337.0')

    @mock.patch('sys.stdout')
    def test_multiple_statistics_get_multiple_lines(self, mock_sysout):
        mock_results = [{
            'Timestamp': datetime.datetime.utcnow(),
            'Maximum': 9001.0,
            'Average': 1337.0,
            'Unit': 'Count',
        }]
        metric = {
            'Namespace': 'AWS/Foo',
            'MetricName': 'RequestCount',
            'Statistics': ['Maximum', 'Average'],
            'Unit': 'Count',
            'Dimensions': {'Krang': 'X'},
        }
        options = leadbutt.get_options(None, metric.get('Options'), None)
        leadbutt.output_results(mock_results, metric, options)

        self.assertEqual(
            mock_sysout.write.call_count, len(metric['Statistics']))

class leadbuttTest(unittest.TestCase):
    @mock.patch('boto.ec2.cloudwatch.connect_to_region')
    @mock.patch('leadbutt.get_config')
    def test_can_get_auth_from_config(self, mock_get_config, mock_connect):
        mock_get_config.return_value = {
            'Metrics': [],
            'Auth': {
                'aws_access_key_id': 'foo',
                'aws_secret_access_key': 'bar',
            }
        }
        leadbutt.leadbutt('dummy_config_file', {'Count': 1, 'Period': 5})
        self.assertTrue(mock_connect.called)
        args, kwargs = mock_connect.call_args
        self.assertEqual(kwargs['aws_access_key_id'], 'foo')
        self.assertEqual(kwargs['aws_secret_access_key'], 'bar')

@unittest.skipUnless('TOX_TEST_ENTRYPOINT' in os.environ,
    'This is only applicable if leadbutt is installed')
class mainTest(unittest.TestCase):
    def test_entry_point(self):
        # assert this does not raise an exception
        call(['leadbutt', '--help'])

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>"""Urls for authentication views"""

from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^google/$', views.google, name='google')
]
<commit_msg>auth(google): Remove slash from Google Auth url<commit_after>"""Urls for authentication views"""

from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^google$', views.google, name='google')
]
<|endoftext|>


<commit_before># Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""Custom Django field and formfield types (for models and forms."""

import numpy
import re
try:
    import simplejson as json
except ImportError:
    import json
try:
    import cPickle as pickle
except ImportError:
    import pickle

from django.contrib.gis import forms
from django.contrib.gis.db import models as djm

#: regex for splitting string lists on whitespace and/or commas
ARRAY_RE = re.compile('[\s,]+')

# Disable pylint for 'Too many public methods'
# pylint: disable=R0904

class StringArrayFormField(forms.Field):
    """
    Base class containing general functionality for handling list-like
    parameters.
    """
    # No change; customize this in subclasses if a cast is needed (to `float`,
    # for example)
    cast = str
    value_type = 'str'

    # Disable pylint for 'Method could be a function'
    # pylint: disable=R0201
    def clean(self, value):
        """Try to coerce either a string list of values (separated by
        whitespace and/or commas or a list/tuple of values to a list of
        floats. If unsuccessful, raise a
        :exc:`django.forms.ValidationError`
        """
        if value is None:
            return None

        if isinstance(value, (tuple, list)):
            try:
                value = [self.cast(x) for x in value]
            except (TypeError, ValueError):
                raise forms.ValidationError(
                    'Could not coerce sequence values to `float` values'
                )
        elif isinstance(value, str):
            # it could be a string list, like this: "1, 2,3 , 4 5"
            # try to convert it to a an actual list and cast the values to the
            # chosen type
            if len(value) == 0:
                # It's an empty string list
                value = []
            else:
                try:
                    value = [self.cast(x) for x in ARRAY_RE.split(value)]
                except ValueError:
                    raise forms.ValidationError(
                        'Could not coerce `str` to a list of `%s` values'
                        % self.value_type
                    )
        else:
            raise forms.ValidationError(
                'Could not convert value to `list` of `%s` values: %s'
                % (self.value_type, value)
            )
        return value

class FloatArrayFormField(StringArrayFormField):
    """Form field for properly handling float arrays/lists."""

    cast = float
    value_type = 'float'

class PickleFormField(forms.Field):
    """Form field for Python objects which are pickle and saved to the
    database."""

    def clean(self, value):
        """We assume that the Python value specified for this field is exactly
        what we want to pickle and save to the database.

        The value will not modified.
        """
        return value

class FloatArrayField(djm.Field):
    """This field models a postgres `float` array."""

    def db_type(self, connection):
        return 'float[]'

    def get_prep_value(self, value):
        if value is None:
            return None

        # Normally, the value passed in here will be a list.
        # It could also be a string list, each value separated by
        # comma/whitespace.
        if isinstance(value, str):
            if len(value) == 0:
                # It's an empty string list
                value = []
            else:
                # try to coerce the string to a list of floats
                value = [float(x) for x in ARRAY_RE.split(value)]
                # If there's an exception here, just let it be raised.
        return "{" + ', '.join(str(v) for v in value) + "}"

    def formfield(self, **kwargs):
        """Specify a custom form field type so forms know how to handle fields
        of this type.
        """
        defaults = {'form_class': FloatArrayFormField}
        defaults.update(kwargs)
        return super(FloatArrayField, self).formfield(**defaults)

class IntArrayField(djm.Field):
    """This field models a postgresql `int` array"""

    # TODO(lp) implement the corresponding form field
    def db_type(self, connection):
        return 'int[]'

    def get_prep_value(self, value):
        return "{%s}" % ','.join(str(v) for v in value)

class CharArrayField(djm.Field):
    """This field models a postgres `varchar` array."""

    def db_type(self, _connection):
        return 'varchar[]'

    def get_prep_value(self, value):
        """Return data in a format that has been prepared for use as a
        parameter in a query.

        :param value: sequence of string values to be saved in a varchar[]
            field
        :type value: list or tuple

        >>> caf = CharArrayField()
        >>> caf.get_prep_value(['foo', 'bar', 'baz123'])
        '{"foo", "bar", "baz123"}'
        """
        if value is None:
            return None

        # Normally, the value passed in here will be a list.
        # It could also be a string list, each value separated by
        # comma/whitespace.
        if isinstance(value, str):
            if len(value) == 0:
                # It's an empty string list
                value = []
            else:
                # try to coerce the string to a list of strings
                value = list(ARRAY_RE.split(value))
        return '{' + ', '.join('"%s"' % str(v) for v in value) + '}'

    def formfield(self, **kwargs):
        """
        Specify a custom form field type so forms know how to handle fields of
        this type.
        """
        defaults = {'form_class': StringArrayFormField}
        defaults.update(kwargs)
        return super(CharArrayField, self).formfield(**defaults)

class PickleField(djm.Field):
    """Field for transparent pickling and unpickling of python objects."""

    __metaclass__ = djm.SubfieldBase

    SUPPORTED_BACKENDS = set((
        'django.contrib.gis.db.backends.postgis',
        'django.db.backends.postgresql_psycopg2'
    ))

    def db_type(self, connection):
        """Return "bytea" as postgres' column type."""
        assert connection.settings_dict['ENGINE'] in self.SUPPORTED_BACKENDS
        return 'bytea'

    def to_python(self, value):
        """Unpickle the value."""
        if isinstance(value, (buffer, str, bytearray)) and value:
            return pickle.loads(str(value))
        else:
            return value

    def get_prep_value(self, value):
        """Pickle the value."""
        return bytearray(pickle.dumps(value, pickle.HIGHEST_PROTOCOL))

    def formfield(self, **kwargs):
        """Specify a custom form field type so forms don't treat this as a
        default type (such as a string). Any Python object is valid for this
        field type.
        """
        defaults = {'form_class': PickleFormField}
        defaults.update(kwargs)
        return super(PickleField, self).formfield(**defaults)

class DictField(PickleField):
    """Field for storing Python `dict` objects (or a JSON text representation.
    """

    def to_python(self, value):
        """The value of a DictField can obviously be a `dict`. The value can
        also be specified as a JSON string. If it is, convert it to a `dict`.
        """
        if isinstance(value, str):
            try:
                value = json.loads(value)
            except ValueError:
                # This string is not JSON.
                value = super(DictField, self).to_python(value)
        else:
            value = super(DictField, self).to_python(value)

        return value

class NumpyListField(PickleField):
    """
    Field for storing numpy arrays as pickled blobs. The actual blob stored in
    the database is simply a pickled `list`. When the field is instantiated,
    the value is converted back to a numpy array.
    """

    def to_python(self, value):
        """
        Try to reconstruct a `numpy.ndarray` from the given ``value``.

        :param value:
            The pickled representation of an object which can be reconstituted
            using `pickle.loads`.
        """
        if value is None:
            return None

        # first, unpickle:
        value = super(NumpyListField, self).to_python(value)
        if isinstance(value, (list, tuple, numpy.ndarray)):
            return numpy.array(value)

        # NOTE: If the value is not a list or tuple, raise an exception.
        # The reason we do this is because this field type is intended to be
        # used only for storing list-like data. (Technically any object can be
        # wrapped in a numpy array, like `numpy.array(object())`, but this is
        # not our use case.
        raise ValueError(
            "Unexpected value of type '%s'. Expected 'list' or 'tuple'."
            % type(value)
        )

    def get_prep_value(self, value):
        """
        Convert the ``value`` to the pickled representation of a `list`. If
        ``value`` is a `numpy.ndarray`, it will be converted to a list of the
        same size and shape before being pickled.

        :param value:
            A `list`, `tuple`, or `numpy.ndarray`.
        """
        # convert to list first before pickling, if it's a numpy array
        if isinstance(value, numpy.ndarray):
            return super(NumpyListField, self).get_prep_value(value.tolist())
        else:
            if not instance(value, (list, tuple)):
                raise ValueError(
                    "Unexpected value of type '%s'. Expected 'list', 'tuple', "
                    "or 'numpy.ndarray'"
                    % type(value)
                )

class OqNullBooleanField(djm.NullBooleanField):
    """
    A `NullBooleanField` that can convert meaningful strings to boolean
    values (in the case of config file parameters).
    """

    def to_python(self, value):
        """
        If ``value`` is a `str`, try to extract some boolean value from it.
        """
        if isinstance(value, str):
            if value.lower() in ('t', 'true', 'y', 'yes'):
                value = True
            elif value.lower() in ('f', 'false', 'n', 'no'):
                value = False
        # otherwise, it will just get cast to a bool, which could produce some
        # strange results
        value = super(OqNullBooleanField, self).to_python(value)
        return value

class NullFloatField(djm.FloatField):
    """
    A nullable float field that handles blank input values properly.
    """

    def get_prep_value(self, value):
        if isinstance(value, basestring) and value.strip():
            return super(NullFloatField, self).get_prep_value(value)
<commit_msg>db/fields:<commit_after># Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""Custom Django field and formfield types (for models and forms."""

import numpy
import re
try:
    import simplejson as json
except ImportError:
    import json
try:
    import cPickle as pickle
except ImportError:
    import pickle

from django.contrib.gis import forms
from django.contrib.gis.db import models as djm

#: regex for splitting string lists on whitespace and/or commas
ARRAY_RE = re.compile('[\s,]+')

# Disable pylint for 'Too many public methods'
# pylint: disable=R0904

class StringArrayFormField(forms.Field):
    """
    Base class containing general functionality for handling list-like
    parameters.
    """
    # No change; customize this in subclasses if a cast is needed (to `float`,
    # for example)
    cast = str
    value_type = 'str'

    # Disable pylint for 'Method could be a function'
    # pylint: disable=R0201
    def clean(self, value):
        """Try to coerce either a string list of values (separated by
        whitespace and/or commas or a list/tuple of values to a list of
        floats. If unsuccessful, raise a
        :exc:`django.forms.ValidationError`
        """
        if value is None:
            return None

        if isinstance(value, (tuple, list)):
            try:
                value = [self.cast(x) for x in value]
            except (TypeError, ValueError):
                raise forms.ValidationError(
                    'Could not coerce sequence values to `float` values'
                )
        elif isinstance(value, str):
            # it could be a string list, like this: "1, 2,3 , 4 5"
            # try to convert it to a an actual list and cast the values to the
            # chosen type
            if len(value) == 0:
                # It's an empty string list
                value = []
            else:
                try:
                    value = [self.cast(x) for x in ARRAY_RE.split(value)]
                except ValueError:
                    raise forms.ValidationError(
                        'Could not coerce `str` to a list of `%s` values'
                        % self.value_type
                    )
        else:
            raise forms.ValidationError(
                'Could not convert value to `list` of `%s` values: %s'
                % (self.value_type, value)
            )
        return value

class FloatArrayFormField(StringArrayFormField):
    """Form field for properly handling float arrays/lists."""

    cast = float
    value_type = 'float'

class PickleFormField(forms.Field):
    """Form field for Python objects which are pickle and saved to the
    database."""

    def clean(self, value):
        """We assume that the Python value specified for this field is exactly
        what we want to pickle and save to the database.

        The value will not modified.
        """
        return value

class FloatArrayField(djm.Field):
    """This field models a postgres `float` array."""

    def db_type(self, connection):
        return 'float[]'

    def get_prep_value(self, value):
        if value is None:
            return None

        # Normally, the value passed in here will be a list.
        # It could also be a string list, each value separated by
        # comma/whitespace.
        if isinstance(value, str):
            if len(value) == 0:
                # It's an empty string list
                value = []
            else:
                # try to coerce the string to a list of floats
                value = [float(x) for x in ARRAY_RE.split(value)]
                # If there's an exception here, just let it be raised.
        return "{" + ', '.join(str(v) for v in value) + "}"

    def formfield(self, **kwargs):
        """Specify a custom form field type so forms know how to handle fields
        of this type.
        """
        defaults = {'form_class': FloatArrayFormField}
        defaults.update(kwargs)
        return super(FloatArrayField, self).formfield(**defaults)

class IntArrayField(djm.Field):
    """This field models a postgresql `int` array"""

    # TODO(lp) implement the corresponding form field
    def db_type(self, connection):
        return 'int[]'

    def get_prep_value(self, value):
        return "{%s}" % ','.join(str(v) for v in value)

class CharArrayField(djm.Field):
    """This field models a postgres `varchar` array."""

    def db_type(self, _connection):
        return 'varchar[]'

    def get_prep_value(self, value):
        """Return data in a format that has been prepared for use as a
        parameter in a query.

        :param value: sequence of string values to be saved in a varchar[]
            field
        :type value: list or tuple

        >>> caf = CharArrayField()
        >>> caf.get_prep_value(['foo', 'bar', 'baz123'])
        '{"foo", "bar", "baz123"}'
        """
        if value is None:
            return None

        # Normally, the value passed in here will be a list.
        # It could also be a string list, each value separated by
        # comma/whitespace.
        if isinstance(value, str):
            if len(value) == 0:
                # It's an empty string list
                value = []
            else:
                # try to coerce the string to a list of strings
                value = list(ARRAY_RE.split(value))
        return '{' + ', '.join('"%s"' % str(v) for v in value) + '}'

    def formfield(self, **kwargs):
        """
        Specify a custom form field type so forms know how to handle fields of
        this type.
        """
        defaults = {'form_class': StringArrayFormField}
        defaults.update(kwargs)
        return super(CharArrayField, self).formfield(**defaults)

class PickleField(djm.Field):
    """Field for transparent pickling and unpickling of python objects."""

    __metaclass__ = djm.SubfieldBase

    SUPPORTED_BACKENDS = set((
        'django.contrib.gis.db.backends.postgis',
        'django.db.backends.postgresql_psycopg2'
    ))

    def db_type(self, connection):
        """Return "bytea" as postgres' column type."""
        assert connection.settings_dict['ENGINE'] in self.SUPPORTED_BACKENDS
        return 'bytea'

    def to_python(self, value):
        """Unpickle the value."""
        if isinstance(value, (buffer, str, bytearray)) and value:
            return pickle.loads(str(value))
        else:
            return value

    def get_prep_value(self, value):
        """Pickle the value."""
        return bytearray(pickle.dumps(value, pickle.HIGHEST_PROTOCOL))

    def formfield(self, **kwargs):
        """Specify a custom form field type so forms don't treat this as a
        default type (such as a string). Any Python object is valid for this
        field type.
        """
        defaults = {'form_class': PickleFormField}
        defaults.update(kwargs)
        return super(PickleField, self).formfield(**defaults)

class DictField(PickleField):
    """Field for storing Python `dict` objects (or a JSON text representation.
    """

    def to_python(self, value):
        """The value of a DictField can obviously be a `dict`. The value can
        also be specified as a JSON string. If it is, convert it to a `dict`.
        """
        if isinstance(value, str):
            try:
                value = json.loads(value)
            except ValueError:
                # This string is not JSON.
                value = super(DictField, self).to_python(value)
        else:
            value = super(DictField, self).to_python(value)

        return value

class NumpyListField(PickleField):
    """
    Field for storing numpy arrays as pickled blobs. The actual blob stored in
    the database is simply a pickled `list`. When the field is instantiated,
    the value is converted back to a numpy array.
    """

    def to_python(self, value):
        """
        Try to reconstruct a `numpy.ndarray` from the given ``value``.

        :param value:
            The pickled representation of an object which can be reconstituted
            using `pickle.loads`.
        """
        if value is None:
            return None

        # first, unpickle:
        value = super(NumpyListField, self).to_python(value)
        if isinstance(value, (list, tuple, numpy.ndarray)):
            return numpy.array(value)

        # NOTE: If the value is not a list or tuple, raise an exception.
        # The reason we do this is because this field type is intended to be
        # used only for storing list-like data. (Technically any object can be
        # wrapped in a numpy array, like `numpy.array(object())`, but this is
        # not our use case.
        raise ValueError(
            "Unexpected value of type '%s'. Expected 'list' or 'tuple'."
            % type(value)
        )

    def get_prep_value(self, value):
        """
        Convert the ``value`` to the pickled representation of a `list`. If
        ``value`` is a `numpy.ndarray`, it will be converted to a list of the
        same size and shape before being pickled.

        :param value:
            A `list`, `tuple`, or `numpy.ndarray`.
        """
        # convert to list first before pickling, if it's a numpy array
        if isinstance(value, numpy.ndarray):
            return super(NumpyListField, self).get_prep_value(value.tolist())
        else:
            if not instance(value, (list, tuple)):
                raise ValueError(
                    "Unexpected value of type '%s'. Expected 'list', 'tuple', "
                    "or 'numpy.ndarray'"
                    % type(value)
                )

class OqNullBooleanField(djm.NullBooleanField):
    """
    A `NullBooleanField` that can convert meaningful strings to boolean
    values (in the case of config file parameters).
    """

    def to_python(self, value):
        """
        If ``value`` is a `str`, try to extract some boolean value from it.
        """
        if isinstance(value, str):
            if value.lower() in ('t', 'true', 'y', 'yes'):
                value = True
            elif value.lower() in ('f', 'false', 'n', 'no'):
                value = False
        # otherwise, it will just get cast to a bool, which could produce some
        # strange results
        value = super(OqNullBooleanField, self).to_python(value)
        return value

class NullFloatField(djm.FloatField):
    """
    A nullable float field that handles blank input values properly.
    """

    def get_prep_value(self, value):
        if isinstance(value, basestring) and value.strip():
            return super(NullFloatField, self).get_prep_value(value)
        else:
            return value
<|endoftext|>


<commit_before>
VERSION = '0.21.5.8'

<commit_msg>version bump for 0.21.5.9.<commit_after>
VERSION = '0.21.5.9'

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Implementation of Geodis Api."""
from datetime import datetime
from roulier.codec import Encoder
from roulier.exception import InvalidApiInput
from .geodis_api_edi import GeodisApiEdi

class GeodisEncoderEdi(Encoder):

    def api(self):
        api = GeodisApiEdi()
        return api.api_values()

    def encode(self, api_input):
        api = GeodisApiEdi()
        if not api.validate(api_input):
            raise InvalidApiInput(
                'Input error : %s' % api.errors(api_input))
        data = api.normalize(api_input)

        return {
            "body": self.encode_agency(
                agency_address=data['agency_address'],
                from_address=data['from_address'],
                shipments=data['shipments'],
                service=data['service']),
            "headers": data['service'],
        }

    def encode_shipment(self, shipment, service, idx):
        packs = shipment['parcels']
        to_address = shipment['to_address']
        lines = [
            ['CNI', '%s' % idx, shipment['shippingId']],
            ['TSR', '2', shipment['product'], shipment['productOption']],
            # /*MOA si envoi international */
            # ['MOA', '43', $shipment['cost'], 'EUR'],
            ['FTX', 'DEL', '', '', shipment['reference2']],
            ['TOD', '6', 'PP', shipment['productTOD']],
            ['RFF', ['ADE', service['customerId']]],
            ['RFF', ['ACL', shipment['reference1']]],
            ["NAD", "CN",
                ""
                "",  # C058
                to_address['name'],
                [to_address['street1'], to_address['street2']],
                to_address['city'],
                "",  # 3164
                to_address['zip'],
                to_address['country']
             ],
            ['CTA', 'IC', ['', to_address['name']]],
            ['COM', [to_address['phone'], 'TE']],
        ]
        j = 0
        for pack in packs:
            j = j + 1
            lines += [
                ['GID', '%s' % j, ['1', 'PC', '21', '6']],
                ['MEA', 'AAE', 'AAD', ['KGM', '%s' % pack['weight']]],
                ['PCI', '18'],
                ['GIN', 'BN', pack['barcode']]
            ]
        return lines

    def encode_agency(self, agency_address, from_address, shipments, service):
        shipment_lines = []
        i = 0
        deposit = service['deposit']
        date = datetime.now()
        for shipment in shipments:
            i = i + 1
            shipment_lines += self.encode_shipment(shipment, service, i)

        lines = [
            ['UNH', deposit, ['IFCSUM', 'D', '96A', 'UN', 'ETT021']],
            ['BGM', '630', deposit],
            ['DTM', ['137', date.strftime('%y%m%d%H%M'), '203']],
            ['RFF', ['ADE', service['customerId']]],
            ['TDT', '20', '', '30', '31'],
            ["NAD", "CA",
                [agency_address['siret'], "100"],
                "",  # C058
                agency_address['name'],
                [agency_address['street1'], agency_address['street2']],
                agency_address['city'],
                "",  # 3164
                agency_address['zip'],
                agency_address['country']
             ],
            ["NAD", "CZ",
                [from_address['siret'], "100"],
                "",  # C058
                from_address['name'],
                [from_address['street1'], from_address['street2']],
                from_address['city'],
                "",  # 3164
                from_address['zip'],
                from_address['country']
             ],
            ['DOC', '630', deposit],
        ] + shipment_lines
        lines += [
            ['UNT', '%s' % (len(lines) + 6), deposit]
        ]
        return lines

<commit_msg>fix depositId<commit_after># -*- coding: utf-8 -*-
"""Implementation of Geodis Api."""
from datetime import datetime
from roulier.codec import Encoder
from roulier.exception import InvalidApiInput
from .geodis_api_edi import GeodisApiEdi

class GeodisEncoderEdi(Encoder):

    def api(self):
        api = GeodisApiEdi()
        return api.api_values()

    def encode(self, api_input):
        api = GeodisApiEdi()
        if not api.validate(api_input):
            raise InvalidApiInput(
                'Input error : %s' % api.errors(api_input))
        data = api.normalize(api_input)

        return {
            "body": self.encode_agency(
                agency_address=data['agency_address'],
                from_address=data['from_address'],
                shipments=data['shipments'],
                service=data['service']),
            "headers": data['service'],
        }

    def encode_shipment(self, shipment, service, idx):
        packs = shipment['parcels']
        to_address = shipment['to_address']
        lines = [
            ['CNI', '%s' % idx, shipment['shippingId']],
            ['TSR', '2', shipment['product'], shipment['productOption']],
            # /*MOA si envoi international */
            # ['MOA', '43', $shipment['cost'], 'EUR'],
            ['FTX', 'DEL', '', '', shipment['reference2']],
            ['TOD', '6', 'PP', shipment['productTOD']],
            ['RFF', ['ADE', service['customerId']]],
            ['RFF', ['ACL', shipment['reference1']]],
            ["NAD", "CN",
                ""
                "",  # C058
                to_address['name'],
                [to_address['street1'], to_address['street2']],
                to_address['city'],
                "",  # 3164
                to_address['zip'],
                to_address['country']
             ],
            ['CTA', 'IC', ['', to_address['name']]],
            ['COM', [to_address['phone'], 'TE']],
        ]
        j = 0
        for pack in packs:
            j = j + 1
            lines += [
                ['GID', '%s' % j, ['1', 'PC', '21', '6']],
                ['MEA', 'AAE', 'AAD', ['KGM', '%s' % pack['weight']]],
                ['PCI', '18'],
                ['GIN', 'BN', pack['barcode']]
            ]
        return lines

    def encode_agency(self, agency_address, from_address, shipments, service):
        shipment_lines = []
        i = 0
        deposit = service['depositId']
        date = datetime.now()
        for shipment in shipments:
            i = i + 1
            shipment_lines += self.encode_shipment(shipment, service, i)

        lines = [
            ['UNH', deposit, ['IFCSUM', 'D', '96A', 'UN', 'ETT021']],
            ['BGM', '630', deposit],
            ['DTM', ['137', date.strftime('%y%m%d%H%M'), '203']],
            ['RFF', ['ADE', service['customerId']]],
            ['TDT', '20', '', '30', '31'],
            ["NAD", "CA",
                [agency_address['siret'], "100"],
                "",  # C058
                agency_address['name'],
                [agency_address['street1'], agency_address['street2']],
                agency_address['city'],
                "",  # 3164
                agency_address['zip'],
                agency_address['country']
             ],
            ["NAD", "CZ",
                [from_address['siret'], "100"],
                "",  # C058
                from_address['name'],
                [from_address['street1'], from_address['street2']],
                from_address['city'],
                "",  # 3164
                from_address['zip'],
                from_address['country']
             ],
            ['DOC', '630', deposit],
        ] + shipment_lines
        lines += [
            ['UNT', '%s' % (len(lines) + 6), deposit]
        ]
        return lines

<|endoftext|>


<commit_before>""":mod:`geofront.backends.dbapi` --- Key store using DB-API 2.0
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

.. seealso::

   :pep:`249` --- Python Database API Specification v2.0

.. versionadded:: 0.2.0

"""
import base64
import collections.abc
import contextlib
import types

from paramiko.pkey import PKey

from ..identity import Identity
from ..keystore import (KEY_TYPES, DuplicatePublicKeyError, KeyStore,
                        KeyTypeError)
from ..util import typed

__all__ = 'DatabaseKeyStore',

class DatabaseKeyStore(KeyStore):
    """Store public keys into database through DB-API 2.0.  It takes
    a module that implements DB-API 2.0, and arguments/keywords to
    its ``connect()`` method.  For example, the following code stores
    public keys into SQLite 3 database::

        import sqlite3
        DatabaseKeyStore(sqlite3, 'geofront.db')

    The following code stores public keys into PostgreSQL database
    through psycopg2_::

        import psycopg2
        DatabaseKeyStore(psycopg2, database='geofront', user='postgres')

    It will create a table named ``geofront_public_key`` into the database.

    :param db_module: :pep:`249` DB-API 2.0 compliant module
    :type db_module: :class:`types.ModuleType`
    :param \*args: arguments to ``db_module.connect()`` function
    :param \*kwargs: keyword arguments to ``db_module.connect()`` function

    .. _psycopg2: http://initd.org/psycopg/

    """

    @typed
    def __init__(self, db_module: types.ModuleType, *args, **kwargs):
        if not callable(getattr(db_module, 'connect', None)):
            module_name = db_module.__name__
            raise TypeError('db_module must be DB-API 2.0 compliant, but {} '
                            'lacks connect() function'.format(module_name))
        self.db_module = db_module
        self.integrity_error = db_module.IntegrityError
        self.connection_args = args
        self.connection_kwargs = kwargs

    @contextlib.contextmanager
    def _connect(self):
        connection = self.db_module.connect(*self.connection_args,
                                            **self.connection_kwargs)
        cursor = connection.cursor()
        try:
            cursor.execute('''
                CREATE TABLE IF NOT EXISTS geofront_public_key (
                    key_type VARCHAR(64) NOT NULL,
                    key_base64 VARCHAR(767) NOT NULL,
                    team_type VARCHAR(128) NOT NULL,
                    identifier VARCHAR(128) NOT NULL,
                    PRIMARY KEY (key_type, key_base64)
                )
            ''')
            connection.commit()
        finally:
            cursor.close()
        yield connection
        connection.close()

    def _execute(self, cursor, sql: str, params: tuple):
        """To support various paramstyles.  See the following specification:

        http://legacy.python.org/dev/peps/pep-0249/#paramstyle

        """
        paramstyle = self.db_module.paramstyle
        if paramstyle == 'format':
            sql = sql.replace('?', '%s')
        elif paramstyle != 'qmark':
            if paramstyle == 'numeric':
                fmt = ':{}'
                i = 1
            else:
                if paramstyle == 'named':
                    fmt = ':p{}'
                else:  # pyformat
                    fmt = '%(p{})s'
                params = {'p' + str(i): val for i, val in enumerate(params)}
                i = 0
            while '?' in sql:
                sql = sql.replace('?', fmt.format(i), 1)
                i += 1
        cursor.execute(sql, params)

    def _get_key_params(self, public_key: PKey) -> tuple:
        return public_key.get_name(), public_key.get_base64()

    def _get_identity_params(self, identity: Identity) -> tuple:
        return ('{0.__module__}.{0.__qualname__}'.format(identity.team_type),
                str(identity.identifier))

    def _get_key_class(self, keytype: str) -> type:
        try:
            return KEY_TYPES[keytype]
        except KeyError:
            raise KeyTypeError('unsupported key type: ' + repr(keytype))

    @typed
    def register(self, identity: Identity, public_key: PKey):
        with self._connect() as connection:
            cursor = connection.cursor()
            try:
                params = (self._get_key_params(public_key) +
                          self._get_identity_params(identity))
                self._execute(cursor, '''
                    INSERT INTO geofront_public_key (
                        key_type, key_base64, team_type, identifier
                    ) VALUES (?, ?, ?, ?)
                ''', params)
                connection.commit()
            except self.integrity_error as e:
                raise DuplicatePublicKeyError(str(e))
            finally:
                cursor.close()

    @typed
    def list_keys(self, identity: Identity) -> collections.abc.Set:
        with self._connect() as connection:
            cursor = connection.cursor()
            try:
                self._execute(cursor, '''
                    SELECT key_type, key_base64
                    FROM geofront_public_key
                    WHERE team_type = ? AND identifier = ?
                ''', self._get_identity_params(identity))
                return frozenset(
                    self._get_key_class(keytype)(data=base64.b64decode(b64))
                    for keytype, b64 in cursor.fetchall()
                )
            finally:
                cursor.close()

    @typed
    def deregister(self, identity: Identity, public_key: PKey):
        with self._connect() as connection:
            cursor = connection.cursor()
            try:
                params = (self._get_key_params(public_key) +
                          self._get_identity_params(identity))
                self._execute(cursor, '''
                    DELETE FROM geofront_public_key
                    WHERE key_type = ? AND key_base64 = ? AND
                          team_type = ? AND identifier = ?
                ''', params)
                connection.commit()
            finally:
                cursor.close()
<commit_msg>There's maximum length for pk in MySQL<commit_after>""":mod:`geofront.backends.dbapi` --- Key store using DB-API 2.0
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

.. seealso::

   :pep:`249` --- Python Database API Specification v2.0

.. versionadded:: 0.2.0

"""
import base64
import collections.abc
import contextlib
import types

from paramiko.pkey import PKey

from ..identity import Identity
from ..keystore import (KEY_TYPES, DuplicatePublicKeyError, KeyStore,
                        KeyTypeError)
from ..util import typed

__all__ = 'DatabaseKeyStore',

class DatabaseKeyStore(KeyStore):
    """Store public keys into database through DB-API 2.0.  It takes
    a module that implements DB-API 2.0, and arguments/keywords to
    its ``connect()`` method.  For example, the following code stores
    public keys into SQLite 3 database::

        import sqlite3
        DatabaseKeyStore(sqlite3, 'geofront.db')

    The following code stores public keys into PostgreSQL database
    through psycopg2_::

        import psycopg2
        DatabaseKeyStore(psycopg2, database='geofront', user='postgres')

    It will create a table named ``geofront_public_key`` into the database.

    :param db_module: :pep:`249` DB-API 2.0 compliant module
    :type db_module: :class:`types.ModuleType`
    :param \*args: arguments to ``db_module.connect()`` function
    :param \*kwargs: keyword arguments to ``db_module.connect()`` function

    .. _psycopg2: http://initd.org/psycopg/

    """

    @typed
    def __init__(self, db_module: types.ModuleType, *args, **kwargs):
        if not callable(getattr(db_module, 'connect', None)):
            module_name = db_module.__name__
            raise TypeError('db_module must be DB-API 2.0 compliant, but {} '
                            'lacks connect() function'.format(module_name))
        self.db_module = db_module
        self.integrity_error = db_module.IntegrityError
        self.connection_args = args
        self.connection_kwargs = kwargs

    @contextlib.contextmanager
    def _connect(self):
        connection = self.db_module.connect(*self.connection_args,
                                            **self.connection_kwargs)
        cursor = connection.cursor()
        try:
            cursor.execute('''
                CREATE TABLE IF NOT EXISTS geofront_public_key (
                    key_type VARCHAR(64) NOT NULL,
                    key_base64 VARCHAR(700) NOT NULL,
                    team_type VARCHAR(128) NOT NULL,
                    identifier VARCHAR(128) NOT NULL,
                    PRIMARY KEY (key_type, key_base64)
                )
            ''')
            connection.commit()
        finally:
            cursor.close()
        yield connection
        connection.close()

    def _execute(self, cursor, sql: str, params: tuple):
        """To support various paramstyles.  See the following specification:

        http://legacy.python.org/dev/peps/pep-0249/#paramstyle

        """
        paramstyle = self.db_module.paramstyle
        if paramstyle == 'format':
            sql = sql.replace('?', '%s')
        elif paramstyle != 'qmark':
            if paramstyle == 'numeric':
                fmt = ':{}'
                i = 1
            else:
                if paramstyle == 'named':
                    fmt = ':p{}'
                else:  # pyformat
                    fmt = '%(p{})s'
                params = {'p' + str(i): val for i, val in enumerate(params)}
                i = 0
            while '?' in sql:
                sql = sql.replace('?', fmt.format(i), 1)
                i += 1
        cursor.execute(sql, params)

    def _get_key_params(self, public_key: PKey) -> tuple:
        return public_key.get_name(), public_key.get_base64()

    def _get_identity_params(self, identity: Identity) -> tuple:
        return ('{0.__module__}.{0.__qualname__}'.format(identity.team_type),
                str(identity.identifier))

    def _get_key_class(self, keytype: str) -> type:
        try:
            return KEY_TYPES[keytype]
        except KeyError:
            raise KeyTypeError('unsupported key type: ' + repr(keytype))

    @typed
    def register(self, identity: Identity, public_key: PKey):
        with self._connect() as connection:
            cursor = connection.cursor()
            try:
                params = (self._get_key_params(public_key) +
                          self._get_identity_params(identity))
                self._execute(cursor, '''
                    INSERT INTO geofront_public_key (
                        key_type, key_base64, team_type, identifier
                    ) VALUES (?, ?, ?, ?)
                ''', params)
                connection.commit()
            except self.integrity_error as e:
                raise DuplicatePublicKeyError(str(e))
            finally:
                cursor.close()

    @typed
    def list_keys(self, identity: Identity) -> collections.abc.Set:
        with self._connect() as connection:
            cursor = connection.cursor()
            try:
                self._execute(cursor, '''
                    SELECT key_type, key_base64
                    FROM geofront_public_key
                    WHERE team_type = ? AND identifier = ?
                ''', self._get_identity_params(identity))
                return frozenset(
                    self._get_key_class(keytype)(data=base64.b64decode(b64))
                    for keytype, b64 in cursor.fetchall()
                )
            finally:
                cursor.close()

    @typed
    def deregister(self, identity: Identity, public_key: PKey):
        with self._connect() as connection:
            cursor = connection.cursor()
            try:
                params = (self._get_key_params(public_key) +
                          self._get_identity_params(identity))
                self._execute(cursor, '''
                    DELETE FROM geofront_public_key
                    WHERE key_type = ? AND key_base64 = ? AND
                          team_type = ? AND identifier = ?
                ''', params)
                connection.commit()
            finally:
                cursor.close()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# weestats.py, version 0.1 for WeeChat version 0.3
# Latest development version: https://github.com/FiXato/weechat_scripts
#
# Inserts some statistics into your input field about the buffers/windows
#  you have open.
# Example: 9 windows used. 70 (of which 12 merged) buffers open: 1 core,
#  1 xfer, 2 python, 3 perl, 47 irc channels, 7 irc servers, 9 irc queries
#
## History:
### 2012-03-29: FiXato:
#
# * version 0.1: initial release.
#     * Display a count of all the different buffers you have open.
#     * Display a count of all the open windows.
#
## Acknowledgements:
# * Sebastien "Flashcode" Helleu, for developing the kick-ass chat/IRC
#    client WeeChat
#
## TODO:
#   - Add more statistics, such as:
#     - average and total history lines.
#     - average and total topic/title lengths
#     - how many are displayed in a window
#     - how many vertical and horizontal splits are used.
#
## Copyright (c) 2012 Filip H.F. "FiXato" Slagter,
#   <FiXato+WeeChat [at] Gmail [dot] com>
#   https://google.com/profiles/FiXato
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
SCRIPT_NAME     = "weestats"
SCRIPT_AUTHOR   = "Filip H.F. 'FiXato' Slagter <fixato [at] gmail [dot] com>"
SCRIPT_VERSION  = "0.1"
SCRIPT_LICENSE  = "MIT"
SCRIPT_DESC     = "Useless statistics about your open buffers and windows"
SCRIPT_COMMAND  = "weestats"
SCRIPT_CLOSE_CB = "close_cb"

import_ok = True

try:
  import weechat as w
except ImportError:
  print "This script must be run under WeeChat."
  import_ok = False

def close_cb(*kwargs):
  return w.WEECHAT_RC_OK

def command_main(data, buffer, args):
  infolist = w.infolist_get("buffer", "", "")
  buffer_groups = {}
  results = []
  buffer_count = 0
  merge_count = 0
  numbers = set()
  while w.infolist_next(infolist):
    bplugin = w.infolist_string(infolist, "plugin_name")
    bname = w.infolist_string(infolist, "name")
    bpointer = w.infolist_pointer(infolist, "pointer")
    bnumber = w.infolist_integer(infolist, "number")
    if not bnumber in numbers:
      numbers.add(bnumber)
    else:
      merge_count += 1
    btype = bplugin
    if bplugin == 'irc':
      if  'server.' in bname:
        btype = '%s servers' % btype
      elif '#' in bname:
        btype = '%s channels' % btype
      else:
        btype = '%s queries' % btype

    buffer_groups.setdefault(btype,[]).append({'name': bname, 'pointer': bpointer})

  w.infolist_free(infolist)

  infolist = w.infolist_get("window", "", "")
  rc = w.infolist_prev(infolist)
  if rc:
    window_count = w.infolist_integer(infolist, "number")
  else:
    window_count = 1

  for bplugin, buffers in buffer_groups.iteritems():
    buffer_count += len(buffers)
    results.append('%i %s' % (len(buffers), bplugin))

  buffer_stats = ', '.join(sorted(results))
  stats_string = '%i windows used. %i (of which %i merged) buffers open: %s' % (window_count, buffer_count, merge_count, buffer_stats)
  w.command("", "/input insert %s" % stats_string)
  return w.WEECHAT_RC_OK

if __name__ == "__main__" and import_ok:
  if w.register(SCRIPT_NAME, SCRIPT_AUTHOR, SCRIPT_VERSION,
                      SCRIPT_LICENSE, SCRIPT_DESC, SCRIPT_CLOSE_CB, ""):

    w.hook_command(SCRIPT_COMMAND,
                          SCRIPT_DESC,
                          "",
                          "Inserts useless statistics about your open windows and buffers into your input line.\n",

                          "",

                          "command_main", "")
<commit_msg>[weestats.py] Version 0.2: Added some code to count the splits.<commit_after># -*- coding: utf-8 -*-
#
# weestats.py, version 0.2 for WeeChat version 0.3
# Latest development version: https://github.com/FiXato/weechat_scripts
#
# Inserts some statistics into your input field about the buffers/windows
#  you have open.
# Example: 9 windows used. 70 (of which 12 merged) buffers open: 1 core,
#  1 xfer, 2 python, 3 perl, 47 irc channels, 7 irc servers, 9 irc queries
#
## History:
### 2012-03-29: FiXato:
#
# * version 0.1: initial release.
#     * Display a count of all the different buffers you have open.
#     * Display a count of all the open windows.
# * version 0.2: Getting the splits.
#     * Displays the how many vertical and horizontal windows.
#       (not quite sure if my approximation is correct though..)
#
## Acknowledgements:
# * Sebastien "Flashcode" Helleu, for developing the kick-ass chat/IRC
#    client WeeChat
#
## TODO:
#   - Add more statistics, such as:
#     - average and total history lines.
#     - average and total topic/title lengths
#     - how many are displayed in a window
#
## Copyright (c) 2012 Filip H.F. "FiXato" Slagter,
#   <FiXato+WeeChat [at] Gmail [dot] com>
#   https://google.com/profiles/FiXato
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
SCRIPT_NAME     = "weestats"
SCRIPT_AUTHOR   = "Filip H.F. 'FiXato' Slagter <fixato [at] gmail [dot] com>"
SCRIPT_VERSION  = "0.2"
SCRIPT_LICENSE  = "MIT"
SCRIPT_DESC     = "Useless statistics about your open buffers and windows"
SCRIPT_COMMAND  = "weestats"
SCRIPT_CLOSE_CB = "close_cb"

import_ok = True

try:
  import weechat as w
except ImportError:
  print "This script must be run under WeeChat."
  import_ok = False

def close_cb(*kwargs):
  return w.WEECHAT_RC_OK

def command_main(data, buffer, args):
  infolist = w.infolist_get("buffer", "", "")
  buffer_groups = {}
  results = []
  buffer_count = 0
  merge_count = 0
  numbers = set()
  while w.infolist_next(infolist):
    bplugin = w.infolist_string(infolist, "plugin_name")
    bname = w.infolist_string(infolist, "name")
    bpointer = w.infolist_pointer(infolist, "pointer")
    bnumber = w.infolist_integer(infolist, "number")
    if not bnumber in numbers:
      numbers.add(bnumber)
    else:
      merge_count += 1
    btype = bplugin
    if bplugin == 'irc':
      if  'server.' in bname:
        btype = '%s servers' % btype
      elif '#' in bname:
        btype = '%s channels' % btype
      else:
        btype = '%s queries' % btype

    buffer_groups.setdefault(btype,[]).append({'name': bname, 'pointer': bpointer})

  w.infolist_free(infolist)

  infolist = w.infolist_get("window", "", "")
  windows_v = set()
  windows_h = set()
  windows = set()
  while w.infolist_next(infolist):
    window = w.infolist_pointer(infolist, "pointer")
    window_w = w.infolist_integer(infolist, "width_pct")
    window_h = w.infolist_integer(infolist, "height_pct")
    windows.add(window)
    if window_h == 100 and window_w != 100:
      windows_v.add(window)
    elif window_w == 100 and window_h != 100:
      windows_h.add(window)
    #else: #both 100%, thus no splits

  window_count = len(windows)

  for bplugin, buffers in buffer_groups.iteritems():
    buffer_count += len(buffers)
    results.append('%i %s' % (len(buffers), bplugin))

  buffer_stats = ', '.join(sorted(results))
  stats_string = '%i windows used (%i vertically / %i horizontally split). %i (of which %i merged) buffers open: %s' % (window_count, len(windows_v), len(windows_h), buffer_count, merge_count, buffer_stats)
  w.command("", "/input insert %s" % stats_string)
  return w.WEECHAT_RC_OK

if __name__ == "__main__" and import_ok:
  if w.register(SCRIPT_NAME, SCRIPT_AUTHOR, SCRIPT_VERSION,
                      SCRIPT_LICENSE, SCRIPT_DESC, SCRIPT_CLOSE_CB, ""):

    w.hook_command(SCRIPT_COMMAND,
                          SCRIPT_DESC,
                          "",
                          "Inserts useless statistics about your open windows and buffers into your input line.\n",

                          "",

                          "command_main", "")
<|endoftext|>


<commit_before># flake8: noqa
import asyncio
import base64
import concurrent.futures as cfutures
import hashlib
import json
import logging
import os
import sys
import traceback

import aiohttp
import asyncio
import voluptuous
from aiohttp import web
from prometheus_async.aio import time
from prometheus_client import Counter, Histogram, Summary

from repour import exception
from repour.config import config
from repour.lib.io import file_utils
from repour.lib.logs import log_util
from repour.lib.logs import file_callback_log
from repour.server.endpoint import validation

logger = logging.getLogger(__name__)

ERROR_VALIDATION_JSON_COUNTER = Counter(
    "error_validation_json_counter",
    "User sent bad JSON requests that failed validation",
)
ERROR_RESPONSE_400_COUNTER = Counter(
    "error_response_400_counter", "App returned 400 status"
)
ERROR_RESPONSE_500_COUNTER = Counter(
    "error_response_500_counter", "App returned 500 status"
)
ERROR_CALLBACK_COUNTER = Counter(
    "error_callback_counter", "Errors calling callback url"
)

REQ_TIME = Summary("callback_time", "time spent with calling callback")
REQ_HISTOGRAM_TIME = Histogram("callback_histogram", "Histogram for calling callback")

def create_log_context_id():
    return "repour-" + base64.b32encode(os.urandom(20)).decode("ascii").lower()

def create_callback_id():
    return base64.b32encode(os.urandom(30)).decode("ascii")

def create_traceback_id():
    tb = traceback.format_exc()
    h = hashlib.md5()
    h.update(tb.encode("utf-8"))
    return h.hexdigest()

def described_error_to_obj(exception):
    traceback_id = create_traceback_id()
    error = {k: v for k, v in exception.__dict__.items() if not k.startswith("_")}
    error["error_type"] = exception.__class__.__name__
    error["error_traceback"] = traceback_id
    return (traceback_id, error)

def exception_to_obj(exception):
    traceback_id = create_traceback_id()
    error = {
        "error_type": exception.__class__.__name__,
        "error_traceback": traceback_id,
    }
    return (traceback_id, error)

def log_traceback_multi_line():
    text = traceback.format_exc()
    for line in text.split("\n"):
        if line != "":
            logger.error(line)

def validated_json_endpoint(shutdown_callbacks, validator, coro, repour_url):
    client_session = aiohttp.ClientSession()  # pylint: disable=no-member
    shutdown_callbacks.append(client_session.close)

    async def handler(request):
        c = await config.get_configuration()

        log_context = request.headers.get("LOG-CONTEXT", "").strip()
        if log_context == "":
            log_context = create_log_context_id()

        log_user_id = request.headers.get("log-user-id", "").strip()
        log_request_context = request.headers.get("log-request-context", "").strip()
        log_process_context = request.headers.get("log-process-context", "").strip()
        log_expires = request.headers.get("log-expires", "").strip()
        log_tmp = request.headers.get("log-tmp", "").strip()
        log_process_context_variant = request.headers.get(
            "process-context-variant", ""
        ).strip()
        trace_id = request.headers.get("trace-id", "").strip()
        span_id = request.headers.get("span-id", "").strip()
        # Some implementations use parent-id instead of span-id
        if span_id == "":
            span_id = request.headers.get("parent-id", "").strip()

        callback_id = create_callback_id()

        asyncio.current_task().log_context = log_context

        # required for bifrost for streaming logs
        asyncio.current_task().loggerName = "org.jboss.pnc._userlog_.alignment-log"

        log_util.add_update_mdc_key_value_in_task("userId", log_user_id)
        log_util.add_update_mdc_key_value_in_task("requestContext", log_request_context)
        log_util.add_update_mdc_key_value_in_task("processContext", log_process_context)
        log_util.add_update_mdc_key_value_in_task("expires", log_expires)
        log_util.add_update_mdc_key_value_in_task("tmp", log_tmp)
        log_util.add_update_mdc_key_value_in_task(
            "processContextVariant", log_process_context_variant
        )
        log_util.add_update_mdc_key_value_in_task("trace_id", trace_id)
        log_util.add_update_mdc_key_value_in_task("span_id", span_id)

        asyncio.current_task().callback_id = callback_id

        try:
            spec = await request.json()
        except ValueError:
            logger.error(
                "Rejected {method} {path}: body is not parsable as json".format(
                    method=request.method, path=request.path
                )
            )

            rejected_data = await request.text()

            ERROR_VALIDATION_JSON_COUNTER.inc()
            logger.error("Request data: {data}".format(data=rejected_data))

            return web.Response(
                status=400,
                content_type="application/json",
                text=json.dumps(
                    obj=[
                        {
                            "error_message": "expected json",
                            "error_type": "json parsability",
                            "path": [],
                        }
                    ],
                    ensure_ascii=False,
                ),
            )
        try:
            validator(spec)
        except voluptuous.MultipleInvalid as x:

            logger.error(
                "Rejected {method} {path}: body failed input validation".format(
                    method=request.method, path=request.path
                )
            )

            ERROR_VALIDATION_JSON_COUNTER.inc()

            logger.error("Request data: {data}".format(data=spec))

            return web.Response(
                status=400,
                content_type="application/json",
                text=json.dumps(obj=[e.__dict__ for e in x.errors], ensure_ascii=False),
            )
        logger.info(
            "Accepted {method} {path}: {params}".format(
                method=request.method, path=request.path, params=spec
            )
        )

        try:
            validation.callback(spec)
        except voluptuous.MultipleInvalid as x:
            callback_mode = False
        else:
            callback_mode = True

        # Set the task_id if provided in the request
        task_id = spec.get("taskId", None)
        if task_id:
            asyncio.current_task().task_id = task_id

        async def do_call():
            try:
                ret = await coro(spec, **request.app)
            except cfutures.CancelledError as e:
                # do nothing else
                logger.info("Cancellation request received")
                raise e

            except exception.DescribedError as e:
                status = 400
                traceback_id, obj = described_error_to_obj(e)
                ERROR_RESPONSE_400_COUNTER.inc()
                logger.error(
                    "Failed ({e.__class__.__name__}), traceback hash: {traceback_id}".format(
                        **locals()
                    )
                )
                log_traceback_multi_line()
            except Exception as e:
                status = 500
                traceback_id, obj = exception_to_obj(e)
                ERROR_RESPONSE_500_COUNTER.inc()
                logger.error(
                    "Internal failure ({e.__class__.__name__}), traceback hash: {traceback_id}".format(
                        **locals()
                    )
                )
                log_traceback_multi_line()
            else:
                status = 200
                obj = ret
                logger.info("Completed ok")

            log_file = file_callback_log.get_callback_log_path(callback_id)
            logs = ""

            if os.path.isfile(log_file):

                # if not defined in log, set it to max size of an int, aka no maximum log size
                max_size_bytes = c.get("max_log_size_bytes", sys.maxsize)
                size_log = os.path.getsize(log_file)

                if size_log > max_size_bytes:
                    # This log should be printed before we grab the final log file so that this error is included in the log file
                    logger.error(
                        "Log file is too big: {} > {}".format(size_log, max_size_bytes)
                    )
                    logs = file_utils.read_last_bytes_of_file(log_file, max_size_bytes)
                    status = 400
                else:
                    with open(log_file, "r") as f:
                        logs = f.read()

            obj["log"] = logs
            return status, obj

        if callback_mode:

            async def do_callback(callback_spec):
                status, obj = await do_call()

                obj["callback"] = {"status": status, "id": callback_id}

                logger.info("Callback data: {}".format(obj))

                @time(REQ_TIME)
                @time(REQ_HISTOGRAM_TIME)
                async def send_result():
                    try:
                        # TODO refactor this into auth.py, we cannot use middleware for callbacks
                        headers = {}

                        current_task = asyncio.current_task()

                        auth_provider = c.get("auth", {}).get("provider", None)
                        if auth_provider == "oauth2_jwt" and request.headers.get(
                            "Authorization", None
                        ):
                            auth_header = {
                                "Authorization": request.headers["Authorization"]
                            }
                            logger.debug(
                                "Authorization enabled, adding header to callback: "
                                + str(auth_header)
                            )
                            headers.update(auth_header)

                        context_headers = {
                            "log-user-id": current_task.mdc["userId"],
                            "log-request-context": current_task.mdc["requestContext"],
                            "log-process-context": current_task.mdc["processContext"],
                            "log-expires": current_task.mdc["expires"],
                            "log_tmp": current_task.mdc["tmp"],
                            "process-context-variant": current_task.mdc[
                                "processContextVariant"
                            ],
                            "trace_id": current_task.mdc["trace_id"],
                            "span_id": current_task.mdc["span_id"],
                        }

                        headers.update(context_headers)
                        headers.update({"Content-Type": "application/json"})

                        resp = await client_session.request(
                            callback_spec.get("method", "POST"),
                            callback_spec["url"],
                            headers=headers,
                            data=json.dumps(obj=obj, ensure_ascii=False).encode(
                                "utf-8"
                            ),
                        )
                    except Exception as e:
                        ERROR_CALLBACK_COUNTER.inc()
                        logger.error(
                            "Unable to send result of callback, exception {ename}, attempt {backoff}/{max_attempts}".format(
                                ename=e.__class__.__name__,
                                backoff=backoff,
                                max_attempts=max_attempts,
                            )
                        )
                        logger.error(e)
                        resp = None
                    return resp

                backoff = 1
                max_attempts = 9
                resp = await send_result()

                while resp is None or resp.status // 100 != 2:
                    if resp is not None:
                        logger.info(
                            "Unable to send result of callback, status {resp.status}, attempt {backoff}/{max_attempts}".format(
                                **locals()
                            )
                        )

                    sleep_period = 2**backoff
                    logger.debug("Sleeping for {sleep_period}".format(**locals()))
                    await asyncio.sleep(sleep_period)

                    backoff += 1

                    if backoff > max_attempts:
                        logger.error(
                            "Giving up on callback after {max_attempts} attempts".format(
                                **locals()
                            )
                        )
                        break

                    resp = await send_result()

                if backoff <= max_attempts:
                    logger.info("Callback result sent successfully")

            logger.info(
                "Creating callback task {callback_id}, returning ID now".format(
                    **locals()
                )
            )
            callback_task = request.app.loop.create_task(do_callback(spec["callback"]))
            callback_task.log_context = log_context
            callback_task.loggerName = asyncio.current_task().loggerName
            callback_task.mdc = asyncio.current_task().mdc
            callback_task.callback_id = callback_id
            # Set the task_id if provided in the request
            task_id = spec.get("taskId", None)
            if task_id:
                callback_task.task_id = task_id

            status = 202
            obj = {
                "callback": {
                    "id": callback_id,
                    "websocket": "ws://" + repour_url + "/callback/" + callback_id,
                }
            }

        else:
            status, obj = await do_call()

        response = web.Response(
            status=status,
            content_type="application/json",
            text=json.dumps(obj=obj, ensure_ascii=False),
        )
        return response

    return handler
<commit_msg>[NCL-7340] Properly generate a span for opentelemetry<commit_after># flake8: noqa
import asyncio
import base64
import concurrent.futures as cfutures
import hashlib
import json
import logging
import os
import sys
import traceback

import aiohttp
import asyncio
import voluptuous
from aiohttp import web
from prometheus_async.aio import time
from prometheus_client import Counter, Histogram, Summary

from repour import exception
from repour.config import config
from repour.lib.io import file_utils
from repour.lib.logs import log_util
from repour.lib.logs import file_callback_log
from repour.server.endpoint import validation
from opentelemetry import trace
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator

logger = logging.getLogger(__name__)

ERROR_VALIDATION_JSON_COUNTER = Counter(
    "error_validation_json_counter",
    "User sent bad JSON requests that failed validation",
)
ERROR_RESPONSE_400_COUNTER = Counter(
    "error_response_400_counter", "App returned 400 status"
)
ERROR_RESPONSE_500_COUNTER = Counter(
    "error_response_500_counter", "App returned 500 status"
)
ERROR_CALLBACK_COUNTER = Counter(
    "error_callback_counter", "Errors calling callback url"
)

REQ_TIME = Summary("callback_time", "time spent with calling callback")
REQ_HISTOGRAM_TIME = Histogram("callback_histogram", "Histogram for calling callback")

def create_log_context_id():
    return "repour-" + base64.b32encode(os.urandom(20)).decode("ascii").lower()

def create_callback_id():
    return base64.b32encode(os.urandom(30)).decode("ascii")

def create_traceback_id():
    tb = traceback.format_exc()
    h = hashlib.md5()
    h.update(tb.encode("utf-8"))
    return h.hexdigest()

def described_error_to_obj(exception):
    traceback_id = create_traceback_id()
    error = {k: v for k, v in exception.__dict__.items() if not k.startswith("_")}
    error["error_type"] = exception.__class__.__name__
    error["error_traceback"] = traceback_id
    return (traceback_id, error)

def exception_to_obj(exception):
    traceback_id = create_traceback_id()
    error = {
        "error_type": exception.__class__.__name__,
        "error_traceback": traceback_id,
    }
    return (traceback_id, error)

def log_traceback_multi_line():
    text = traceback.format_exc()
    for line in text.split("\n"):
        if line != "":
            logger.error(line)

def validated_json_endpoint(shutdown_callbacks, validator, coro, repour_url):
    client_session = aiohttp.ClientSession()  # pylint: disable=no-member
    shutdown_callbacks.append(client_session.close)

    async def handler(request):
        c = await config.get_configuration()

        log_context = request.headers.get("LOG-CONTEXT", "").strip()
        if log_context == "":
            log_context = create_log_context_id()

        log_user_id = request.headers.get("log-user-id", "").strip()
        log_request_context = request.headers.get("log-request-context", "").strip()
        log_process_context = request.headers.get("log-process-context", "").strip()
        log_expires = request.headers.get("log-expires", "").strip()
        log_tmp = request.headers.get("log-tmp", "").strip()
        log_process_context_variant = request.headers.get(
            "process-context-variant", ""
        ).strip()
        trace_id = request.headers.get("trace-id", "").strip()
        span_id = request.headers.get("span-id", "").strip()
        traceparent = request.headers.get("traceparent", "").strip()
        tracestate = request.headers.get("tracestate", "").strip()
        logger.info(">> traceparent: " + traceparent)
        logger.info(">> tracestate: " + tracestate)
        # Some implementations use parent-id instead of span-id
        if span_id == "":
            span_id = request.headers.get("parent-id", "").strip()

        callback_id = create_callback_id()

        asyncio.current_task().log_context = log_context

        # required for bifrost for streaming logs
        asyncio.current_task().loggerName = "org.jboss.pnc._userlog_.alignment-log"

        log_util.add_update_mdc_key_value_in_task("userId", log_user_id)
        log_util.add_update_mdc_key_value_in_task("requestContext", log_request_context)
        log_util.add_update_mdc_key_value_in_task("processContext", log_process_context)
        log_util.add_update_mdc_key_value_in_task("expires", log_expires)
        log_util.add_update_mdc_key_value_in_task("tmp", log_tmp)
        log_util.add_update_mdc_key_value_in_task(
            "processContextVariant", log_process_context_variant
        )
        log_util.add_update_mdc_key_value_in_task("trace_id", trace_id)
        log_util.add_update_mdc_key_value_in_task("span_id", span_id)
        log_util.add_update_mdc_key_value_in_task("traceparent", traceparent)

        asyncio.current_task().callback_id = callback_id

        try:
            spec = await request.json()
        except ValueError:
            logger.error(
                "Rejected {method} {path}: body is not parsable as json".format(
                    method=request.method, path=request.path
                )
            )

            rejected_data = await request.text()

            ERROR_VALIDATION_JSON_COUNTER.inc()
            logger.error("Request data: {data}".format(data=rejected_data))

            return web.Response(
                status=400,
                content_type="application/json",
                text=json.dumps(
                    obj=[
                        {
                            "error_message": "expected json",
                            "error_type": "json parsability",
                            "path": [],
                        }
                    ],
                    ensure_ascii=False,
                ),
            )
        try:
            validator(spec)
        except voluptuous.MultipleInvalid as x:

            logger.error(
                "Rejected {method} {path}: body failed input validation".format(
                    method=request.method, path=request.path
                )
            )

            ERROR_VALIDATION_JSON_COUNTER.inc()

            logger.error("Request data: {data}".format(data=spec))

            return web.Response(
                status=400,
                content_type="application/json",
                text=json.dumps(obj=[e.__dict__ for e in x.errors], ensure_ascii=False),
            )
        logger.info(
            "Accepted {method} {path}: {params}".format(
                method=request.method, path=request.path, params=spec
            )
        )

        try:
            validation.callback(spec)
        except voluptuous.MultipleInvalid as x:
            callback_mode = False
        else:
            callback_mode = True

        # Set the task_id if provided in the request
        task_id = spec.get("taskId", None)
        if task_id:
            asyncio.current_task().task_id = task_id

        async def do_call():
            try:
                ret = await coro(spec, **request.app)
            except cfutures.CancelledError as e:
                # do nothing else
                logger.info("Cancellation request received")
                raise e

            except exception.DescribedError as e:
                status = 400
                traceback_id, obj = described_error_to_obj(e)
                ERROR_RESPONSE_400_COUNTER.inc()
                logger.error(
                    "Failed ({e.__class__.__name__}), traceback hash: {traceback_id}".format(
                        **locals()
                    )
                )
                log_traceback_multi_line()
            except Exception as e:
                status = 500
                traceback_id, obj = exception_to_obj(e)
                ERROR_RESPONSE_500_COUNTER.inc()
                logger.error(
                    "Internal failure ({e.__class__.__name__}), traceback hash: {traceback_id}".format(
                        **locals()
                    )
                )
                log_traceback_multi_line()
            else:
                status = 200
                obj = ret
                logger.info("Completed ok")

            log_file = file_callback_log.get_callback_log_path(callback_id)
            logs = ""

            if os.path.isfile(log_file):

                # if not defined in log, set it to max size of an int, aka no maximum log size
                max_size_bytes = c.get("max_log_size_bytes", sys.maxsize)
                size_log = os.path.getsize(log_file)

                if size_log > max_size_bytes:
                    # This log should be printed before we grab the final log file so that this error is included in the log file
                    logger.error(
                        "Log file is too big: {} > {}".format(size_log, max_size_bytes)
                    )
                    logs = file_utils.read_last_bytes_of_file(log_file, max_size_bytes)
                    status = 400
                else:
                    with open(log_file, "r") as f:
                        logs = f.read()

            obj["log"] = logs
            return status, obj

        if callback_mode:

            async def do_callback(callback_spec):
                status, obj = await do_call()

                obj["callback"] = {"status": status, "id": callback_id}

                logger.info("Callback data: {}".format(obj))

                @time(REQ_TIME)
                @time(REQ_HISTOGRAM_TIME)
                async def send_result():
                    try:
                        # TODO refactor this into auth.py, we cannot use middleware for callbacks
                        headers = {}

                        current_task = asyncio.current_task()

                        auth_provider = c.get("auth", {}).get("provider", None)
                        if auth_provider == "oauth2_jwt" and request.headers.get(
                            "Authorization", None
                        ):
                            auth_header = {
                                "Authorization": request.headers["Authorization"]
                            }
                            logger.debug(
                                "Authorization enabled, adding header to callback: "
                                + str(auth_header)
                            )
                            headers.update(auth_header)

                        context_headers = {
                            "log-user-id": current_task.mdc["userId"],
                            "log-request-context": current_task.mdc["requestContext"],
                            "log-process-context": current_task.mdc["processContext"],
                            "log-expires": current_task.mdc["expires"],
                            "log_tmp": current_task.mdc["tmp"],
                            "process-context-variant": current_task.mdc[
                                "processContextVariant"
                            ],
                            "trace-id": current_task.mdc["trace_id"],
                            "span-id": current_task.mdc["span_id"],
                            # TODO: do we need to include this?
                            "traceparent": current_task.mdc["traceparent"],
                        }

                        headers.update(context_headers)
                        headers.update({"Content-Type": "application/json"})

                        resp = await client_session.request(
                            callback_spec.get("method", "POST"),
                            callback_spec["url"],
                            headers=headers,
                            data=json.dumps(obj=obj, ensure_ascii=False).encode(
                                "utf-8"
                            ),
                        )
                    except Exception as e:
                        ERROR_CALLBACK_COUNTER.inc()
                        logger.error(
                            "Unable to send result of callback, exception {ename}, attempt {backoff}/{max_attempts}".format(
                                ename=e.__class__.__name__,
                                backoff=backoff,
                                max_attempts=max_attempts,
                            )
                        )
                        logger.error(e)
                        resp = None
                    return resp

                backoff = 1
                max_attempts = 9
                resp = await send_result()

                while resp is None or resp.status // 100 != 2:
                    if resp is not None:
                        logger.info(
                            "Unable to send result of callback, status {resp.status}, attempt {backoff}/{max_attempts}".format(
                                **locals()
                            )
                        )

                    sleep_period = 2**backoff
                    logger.debug("Sleeping for {sleep_period}".format(**locals()))
                    await asyncio.sleep(sleep_period)

                    backoff += 1

                    if backoff > max_attempts:
                        logger.error(
                            "Giving up on callback after {max_attempts} attempts".format(
                                **locals()
                            )
                        )
                        break

                    resp = await send_result()

                if backoff <= max_attempts:
                    logger.info("Callback result sent successfully")

            logger.info(
                "Creating callback task {callback_id}, returning ID now".format(
                    **locals()
                )
            )

            # trying
            ctx = TraceContextTextMapPropagator().extract(
                carrier={"traceparent": traceparent}
            )
            tracer = trace.get_tracer(__name__)
            with tracer.start_as_current_span(request.path, ctx) as span:

                callback_task = request.app.loop.create_task(
                    do_callback(spec["callback"])
                )
                callback_task.log_context = log_context
                callback_task.loggerName = asyncio.current_task().loggerName
                callback_task.mdc = asyncio.current_task().mdc
                callback_task.callback_id = callback_id
                # Set the task_id if provided in the request
                task_id = spec.get("taskId", None)
                if task_id:
                    callback_task.task_id = task_id

                status = 202
                obj = {
                    "callback": {
                        "id": callback_id,
                        "websocket": "ws://" + repour_url + "/callback/" + callback_id,
                    }
                }

        else:
            status, obj = await do_call()

        response = web.Response(
            status=status,
            content_type="application/json",
            text=json.dumps(obj=obj, ensure_ascii=False),
        )
        return response

    return handler
<|endoftext|>