<commit_before># -*- coding: utf-8 -*-
# vim: sw=4:ts=4:expandtab

"""
pygogo
~~~~~~

A Python logging library with super powers

Examples:
    basic usage::

        >>> logger = Gogo('basic').logger
        >>> logger.debug('hello world')
        hello world

    intermediate usage::

        >>> formatter = logging.Formatter('IP: %(ip)s - %(message)s')
        >>> kwargs = {'low_formatter': formatter}
        >>> logger = Gogo('intermediate', **kwargs).get_logger(ip='1.1.1.1')
        >>> logger.debug('hello world')
        IP: 1.1.1.1 - hello world

    advanced usage::

        >>> from io import StringIO
        >>> from json import loads
        >>>
        >>> high = StringIO()
        >>> low = StringIO()
        >>> kwargs = {
        ...     'monolog': True, 'high_hdlr': handlers.fileobj_hdlr(high),
        ...     'low_hdlr': handlers.fileobj_hdlr(low)}
        >>> logger = Gogo('adv', **kwargs).get_structured_logger(ip='1.1.1.1')
        >>> logger.debug('debug')
        >>> logger.error('error')
        >>> loads(low.getvalue()) == {'ip': '1.1.1.1', 'message': 'debug'}
        True
        >>> loads(high.getvalue()) == {'ip': '1.1.1.1', 'message': 'error'}
        True
"""

from __future__ import (
    absolute_import, division, print_function, unicode_literals)

import logging
import hashlib
import sys

from copy import copy
from builtins import *
from . import formatters, handlers, utils

__version__ = '0.11.0'

__all__ = ['formatters', 'handlers', 'utils']
__title__ = 'pygogo'
__author__ = 'Reuben Cummings'
__description__ = 'A Python logging library with super powers'
__email__ = 'reubano@gmail.com'
__license__ = 'MIT'
__copyright__ = 'Copyright 2015 Reuben Cummings'

module_hdlr = logging.StreamHandler(sys.stdout)
module_fltr = logging.Filter(name='%s.init' % __name__)

# prevent handler from logging `pygogo.main` events
module_hdlr.addFilter(module_fltr)
module_logger = logging.getLogger(__name__)
module_logger.addHandler(module_hdlr)

class Gogo(object):
    """A logging class that logs events via different handlers depending on the
    severity

    http://stackoverflow.com/a/28743317/408556

    Attributes:
        loggers (set[str]): Set of existing loggers

        monolog (bool): Log high level events only to high pass handler.

        name (string): The logger name.

        levels (dict): The min levels to log to handlers.

        handlers (dict): The high/low pass log handlers.

        formatters (dict): The high/low pass log formatter.

        +------------------------------------+-----------------+
        | messages level                     | message handler |
        +====================================+=================+
        | < levels['low']                    | none            |
        +------------------------------------+-----------------+
        | >= levels['low'], < levels['high'] | low handler     |
        +------------------------------------+-----------------+
        | >= levels['high']                  | both handlers * |
        +------------------------------------+-----------------+

        * This is the case when :attr:`monolog` is `False`. If :attr:`monolog`
          is True, then :attr:`handlers['high']` will be the only
          message handler

    Args:
        name (string): The logger name.

        high_level (string): The min level to log to high_hdlr
            (default: warning).

        low_level (string): The min level to log to low_hdlr
            (default: debug).

        kwargs (dict): Keyword arguments.

    Kwargs:
        high_hdlr (obj): The high pass log handler (a :class:`logging.handlers`
            instance, default: `stderr` StreamHandler).

        low_hdlr (obj): The low pass log handler (a :class:`logging.handlers`
            instance, default: `stdout` StreamHandler).

        high_formatter (obj): The high pass log formatter (a
            :class:`logging.Formatter` instance, default:
            :data:`pygogo.formatters.basic_formatter`).

        low_formatter (obj): The low pass log formatter (a
            :class:`logging.Formatter` instance, default:
            :data:`pygogo.formatters.basic_formatter`).

        verbose (bool): If False, set low level to `info`, if True, set low
            level to `debug`, overrides `levels['low']` if specified
            (default: None).

        monolog (bool): Log high level events only to high pass handler (
            default: False)

    Returns:
        New instance of :class:`pygogo.Gogo`

    Examples:
        >>> Gogo('name').logger.debug('message')
        message
    """

    def __init__(self, name='root', high_level=None, low_level=None, **kwargs):
        """Initialization method.

        Examples:
            >>> Gogo('name') # doctest: +ELLIPSIS
            <pygogo.Gogo object at 0x...>
        """
        verbose = kwargs.get('verbose')
        high_level = high_level or 'warning'

        if verbose is None:
            low_level = low_level or 'debug'
        elif verbose:
            low_level = 'debug'
        else:
            low_level = 'info'

        self.levels = {
            'high': getattr(logging, high_level.upper(), None),
            'low': getattr(logging, low_level.upper(), None)}

        if not isinstance(self.levels['high'], int):
            raise ValueError('Invalid high_level: %s' % self.levels['high'])
        elif not isinstance(self.levels['low'], int):
            raise ValueError('Invalid low_level: %s' % self.levels['low'])
        elif self.levels['high'] < self.levels['low']:
            raise ValueError('high_level must be >= low_level')

        self.loggers = set()
        self.name = name
        self.handlers = {
            'high': kwargs.get('high_hdlr', handlers.stderr_hdlr()),
            'low': kwargs.get('low_hdlr', handlers.stdout_hdlr())}

        self.formatters = {
            'high': kwargs.get('high_formatter', formatters.basic_formatter),
            'low': kwargs.get('low_formatter', formatters.basic_formatter)}

        self.monolog = kwargs.get('monolog')

    @property
    def logger(self):
        """The logger property.

        Returns:
            New instance of :class:`logging.Logger`

        Examples:
            >>> from io import StringIO

            >>> s = StringIO()
            >>> sys.stderr = s
            >>> logger = Gogo('default').logger
            >>> logger  # doctest: +ELLIPSIS
            <...Logger...>
            >>> logger.debug('stdout')
            stdout
            >>> kwargs = {'low_level': 'info', 'monolog': True}
            >>> logger = Gogo('ignore_if_lt_info', **kwargs).logger
            >>> logger.debug('ignored')
            >>> logger.info('stdout')
            stdout
            >>> logger.warning('stderr')
            >>> s.getvalue().strip() == 'stderr'
            True
        """
        return self.get_logger()

    def update_hdlr(self, hdlr, level, formatter=None, monolog=False, **kwargs):
        """Update a handler with a formatter, level, and filters.

        Args:
            hdlr (obj): A :class:`logging.handlers` instance.

            level (int): The min level to log to to `hdlr`.

            formatter (obj): The log formatter (a :class:`logging.Formatter`
                instance, default: :data:`pygogo.formatters.basic_formatter`)

            monolog (bool): Log high level events only to high pass handler (
                default: False)

            kwargs (dict): Keyword arguemnts passed to
                `pygogo.utils.get_structured_filter`

        See also:
            :meth:`pygogo.Gogo.get_logger`

            :meth:`pygogo.Gogo.get_structured_logger`

            :func:`pygogo.utils.LogFilter`

            :func:`pygogo.utils.get_structured_filter`

        Examples:
            >>> low_hdlr = logging.StreamHandler(sys.stdout)
            >>> going = Gogo(low_hdlr=low_hdlr, monolog=True)
            >>> hdlr = going.handlers['low']
            >>> [hdlr.formatter, hdlr.filters, hdlr.level]
            [None, [], 0]
            >>> fmtr = going.formatters['low']
            >>> kwargs = {'formatter': fmtr, 'monolog': going.monolog}
            >>> going.update_hdlr(hdlr, going.levels['low'], **kwargs)
            >>> [hdlr.formatter, hdlr.filters, hdlr.level]
            ...  # doctest: +ELLIPSIS
            [<...Formatter obj...>, [<pygogo.utils.LogFilter obj...>], 10]
        """
        hdlr.setLevel(level)

        if monolog:
            log_filter = utils.LogFilter(self.levels['high'])
            hdlr.addFilter(log_filter)

        if kwargs:
            structured_filter = utils.get_structured_filter(**kwargs)
            hdlr.addFilter(structured_filter)

        hdlr.setFormatter(formatter)

    def zip(self, *fmtrs):
        """Format high/low handler properties so that they can be conveniently
            passed to `update_hdlr`.

        Args:
            fmtrs (seq[obj]): A sequence of :class:`logging.Formatter`
                instances.

        See also:
            :meth:`pygogo.Gogo.update_hdlr`

            :meth:`pygogo.Gogo.get_logger`

            :meth:`pygogo.Gogo.get_structured_logger`

        Returns:
            New instance of :class:`logging.handlers`

        Examples:
            >>> hdlr = logging.StreamHandler(sys.stdout)
            >>> copy_hdlr(hdlr) # doctest: +ELLIPSIS
            <...StreamHandler...>
        """
        hdlrs = [self.handlers['high'], self.handlers['low']]
        levels = [self.levels['high'], self.levels['low']]
        monologs = [False, self.monolog]
        return zip(hdlrs, levels, fmtrs, monologs)

    def get_logger(self, name='base', **kwargs):
        """Retrieve a named logger.

        Args:
            name (string): The logger name.

        See also:
            :func:`pygogo.copy_hdlr`

            :meth:`pygogo.Gogo.update_hdlr`

            :meth:`pygogo.Gogo.zip`

        Returns:
            New instance of :class:`logging.Logger`

        Examples:
            >>> going = Gogo()
            >>> logger = going.get_logger('default')
            >>> logger # doctest: +ELLIPSIS
            <...Logger...>
            >>> logger.info('from default')
            from default
            >>> going.get_logger('new').info('from new')
            from new
        """
        lggr_name = '%s.%s' % (self.name, name)
        logger = logging.getLogger(lggr_name)

        if lggr_name not in self.loggers:
            self.loggers.add(lggr_name)

            if kwargs:
                kwargs['name'] = lggr_name

            fmtrs = [self.formatters['high'], self.formatters['low']]

            for zipped in self.zip(*fmtrs):
                hdlr, level, fmtr, monolog = zipped
                copied_hdlr = copy_hdlr(hdlr)
                self.update_hdlr(copied_hdlr, level, fmtr, monolog, **kwargs)
                logger.addHandler(copied_hdlr)

            logger.setLevel(self.levels['low'])

        return logger

    def get_structured_logger(self, name=None, **kwargs):
        """Retrieve a structured data logger

        Args:
            name (string): The logger name.

            kwargs (dict): Keyword arguments to include in every log message.

        See also:
            :func:`pygogo.copy_hdlr`

            :meth:`pygogo.Gogo.update_hdlr`

            :meth:`pygogo.Gogo.zip`

            :class:`pygogo.utils.StructuredAdapter`

        Returns:
            New instance of :class:`pygogo.utils.StructuredAdapter`

        Examples
            >>> from io import StringIO
            >>> from json import loads

            >>> s = StringIO()
            >>> going = Gogo('structured', low_hdlr=handlers.fileobj_hdlr(s))
            >>> logger = going.get_structured_logger(all='true')
            >>> logger  # doctest: +ELLIPSIS
            <...StructuredAdapter...>
            >>> logger.debug('hello')
            >>> logger.debug('extra', extra={'key': 'value'})
            >>> s.seek(0) or 0
            0
            >>> loads(next(s)) == {'all': 'true', 'message': 'hello'}
            True
            >>> loads(next(s)) == {
            ...     'all': 'true', 'message': 'extra', 'key': 'value'}
            True
        """
        # pylint: disable=dict-items-not-iterating
        values = frozenset(kwargs.items())
        name = name or hashlib.md5(str(values).encode('utf-8')).hexdigest()
        lggr_name = '%s.structured.%s' % (self.name, name)
        logger = logging.getLogger(lggr_name)

        if lggr_name not in self.loggers:
            self.loggers.add(lggr_name)
            formatter = formatters.basic_formatter

            for zipped in self.zip(formatter, formatter):
                hdlr, level, fmtr, monolog = zipped
                copied_hdlr = copy_hdlr(hdlr)
                self.update_hdlr(copied_hdlr, level, fmtr, monolog)
                logger.addHandler(copied_hdlr)

            logger.setLevel(self.levels['low'])

        return utils.StructuredAdapter(logger, kwargs)

def copy_hdlr(hdlr):
    """Safely copy a handler and its associated filters.

    Args:
        hdlr (obj): A :class:`logging.handlers` instance.

    See also:
        :meth:`pygogo.Gogo.get_logger`

        :meth:`pygogo.Gogo.get_structured_logger`

    Returns:
        New instance of :class:`logging.handlers`

    Examples:
        >>> hdlr = logging.StreamHandler(sys.stdout)
        >>> copy_hdlr(hdlr) # doctest: +ELLIPSIS
        <...StreamHandler...>
    """
    copied_hdlr = copy(hdlr)
    copied_hdlr.filters = [copy(f) for f in hdlr.filters]
    return copied_hdlr

logger = Gogo().logger
<commit_msg>Bump to version 0.12.0<commit_after># -*- coding: utf-8 -*-
# vim: sw=4:ts=4:expandtab

"""
pygogo
~~~~~~

A Python logging library with super powers

Examples:
    basic usage::

        >>> logger = Gogo('basic').logger
        >>> logger.debug('hello world')
        hello world

    intermediate usage::

        >>> formatter = logging.Formatter('IP: %(ip)s - %(message)s')
        >>> kwargs = {'low_formatter': formatter}
        >>> logger = Gogo('intermediate', **kwargs).get_logger(ip='1.1.1.1')
        >>> logger.debug('hello world')
        IP: 1.1.1.1 - hello world

    advanced usage::

        >>> from io import StringIO
        >>> from json import loads
        >>>
        >>> high = StringIO()
        >>> low = StringIO()
        >>> kwargs = {
        ...     'monolog': True, 'high_hdlr': handlers.fileobj_hdlr(high),
        ...     'low_hdlr': handlers.fileobj_hdlr(low)}
        >>> logger = Gogo('adv', **kwargs).get_structured_logger(ip='1.1.1.1')
        >>> logger.debug('debug')
        >>> logger.error('error')
        >>> loads(low.getvalue()) == {'ip': '1.1.1.1', 'message': 'debug'}
        True
        >>> loads(high.getvalue()) == {'ip': '1.1.1.1', 'message': 'error'}
        True
"""

from __future__ import (
    absolute_import, division, print_function, unicode_literals)

import logging
import hashlib
import sys

from copy import copy
from builtins import *
from . import formatters, handlers, utils

__version__ = '0.12.0'

__all__ = ['formatters', 'handlers', 'utils']
__title__ = 'pygogo'
__author__ = 'Reuben Cummings'
__description__ = 'A Python logging library with super powers'
__email__ = 'reubano@gmail.com'
__license__ = 'MIT'
__copyright__ = 'Copyright 2015 Reuben Cummings'

module_hdlr = logging.StreamHandler(sys.stdout)
module_fltr = logging.Filter(name='%s.init' % __name__)

# prevent handler from logging `pygogo.main` events
module_hdlr.addFilter(module_fltr)
module_logger = logging.getLogger(__name__)
module_logger.addHandler(module_hdlr)

class Gogo(object):
    """A logging class that logs events via different handlers depending on the
    severity

    http://stackoverflow.com/a/28743317/408556

    Attributes:
        loggers (set[str]): Set of existing loggers

        monolog (bool): Log high level events only to high pass handler.

        name (string): The logger name.

        levels (dict): The min levels to log to handlers.

        handlers (dict): The high/low pass log handlers.

        formatters (dict): The high/low pass log formatter.

        +------------------------------------+-----------------+
        | messages level                     | message handler |
        +====================================+=================+
        | < levels['low']                    | none            |
        +------------------------------------+-----------------+
        | >= levels['low'], < levels['high'] | low handler     |
        +------------------------------------+-----------------+
        | >= levels['high']                  | both handlers * |
        +------------------------------------+-----------------+

        * This is the case when :attr:`monolog` is `False`. If :attr:`monolog`
          is True, then :attr:`handlers['high']` will be the only
          message handler

    Args:
        name (string): The logger name.

        high_level (string): The min level to log to high_hdlr
            (default: warning).

        low_level (string): The min level to log to low_hdlr
            (default: debug).

        kwargs (dict): Keyword arguments.

    Kwargs:
        high_hdlr (obj): The high pass log handler (a :class:`logging.handlers`
            instance, default: `stderr` StreamHandler).

        low_hdlr (obj): The low pass log handler (a :class:`logging.handlers`
            instance, default: `stdout` StreamHandler).

        high_formatter (obj): The high pass log formatter (a
            :class:`logging.Formatter` instance, default:
            :data:`pygogo.formatters.basic_formatter`).

        low_formatter (obj): The low pass log formatter (a
            :class:`logging.Formatter` instance, default:
            :data:`pygogo.formatters.basic_formatter`).

        verbose (bool): If False, set low level to `info`, if True, set low
            level to `debug`, overrides `levels['low']` if specified
            (default: None).

        monolog (bool): Log high level events only to high pass handler (
            default: False)

    Returns:
        New instance of :class:`pygogo.Gogo`

    Examples:
        >>> Gogo('name').logger.debug('message')
        message
    """

    def __init__(self, name='root', high_level=None, low_level=None, **kwargs):
        """Initialization method.

        Examples:
            >>> Gogo('name') # doctest: +ELLIPSIS
            <pygogo.Gogo object at 0x...>
        """
        verbose = kwargs.get('verbose')
        high_level = high_level or 'warning'

        if verbose is None:
            low_level = low_level or 'debug'
        elif verbose:
            low_level = 'debug'
        else:
            low_level = 'info'

        self.levels = {
            'high': getattr(logging, high_level.upper(), None),
            'low': getattr(logging, low_level.upper(), None)}

        if not isinstance(self.levels['high'], int):
            raise ValueError('Invalid high_level: %s' % self.levels['high'])
        elif not isinstance(self.levels['low'], int):
            raise ValueError('Invalid low_level: %s' % self.levels['low'])
        elif self.levels['high'] < self.levels['low']:
            raise ValueError('high_level must be >= low_level')

        self.loggers = set()
        self.name = name
        self.handlers = {
            'high': kwargs.get('high_hdlr', handlers.stderr_hdlr()),
            'low': kwargs.get('low_hdlr', handlers.stdout_hdlr())}

        self.formatters = {
            'high': kwargs.get('high_formatter', formatters.basic_formatter),
            'low': kwargs.get('low_formatter', formatters.basic_formatter)}

        self.monolog = kwargs.get('monolog')

    @property
    def logger(self):
        """The logger property.

        Returns:
            New instance of :class:`logging.Logger`

        Examples:
            >>> from io import StringIO

            >>> s = StringIO()
            >>> sys.stderr = s
            >>> logger = Gogo('default').logger
            >>> logger  # doctest: +ELLIPSIS
            <...Logger...>
            >>> logger.debug('stdout')
            stdout
            >>> kwargs = {'low_level': 'info', 'monolog': True}
            >>> logger = Gogo('ignore_if_lt_info', **kwargs).logger
            >>> logger.debug('ignored')
            >>> logger.info('stdout')
            stdout
            >>> logger.warning('stderr')
            >>> s.getvalue().strip() == 'stderr'
            True
        """
        return self.get_logger()

    def update_hdlr(self, hdlr, level, formatter=None, monolog=False, **kwargs):
        """Update a handler with a formatter, level, and filters.

        Args:
            hdlr (obj): A :class:`logging.handlers` instance.

            level (int): The min level to log to to `hdlr`.

            formatter (obj): The log formatter (a :class:`logging.Formatter`
                instance, default: :data:`pygogo.formatters.basic_formatter`)

            monolog (bool): Log high level events only to high pass handler (
                default: False)

            kwargs (dict): Keyword arguemnts passed to
                `pygogo.utils.get_structured_filter`

        See also:
            :meth:`pygogo.Gogo.get_logger`

            :meth:`pygogo.Gogo.get_structured_logger`

            :func:`pygogo.utils.LogFilter`

            :func:`pygogo.utils.get_structured_filter`

        Examples:
            >>> low_hdlr = logging.StreamHandler(sys.stdout)
            >>> going = Gogo(low_hdlr=low_hdlr, monolog=True)
            >>> hdlr = going.handlers['low']
            >>> [hdlr.formatter, hdlr.filters, hdlr.level]
            [None, [], 0]
            >>> fmtr = going.formatters['low']
            >>> kwargs = {'formatter': fmtr, 'monolog': going.monolog}
            >>> going.update_hdlr(hdlr, going.levels['low'], **kwargs)
            >>> [hdlr.formatter, hdlr.filters, hdlr.level]
            ...  # doctest: +ELLIPSIS
            [<...Formatter obj...>, [<pygogo.utils.LogFilter obj...>], 10]
        """
        hdlr.setLevel(level)

        if monolog:
            log_filter = utils.LogFilter(self.levels['high'])
            hdlr.addFilter(log_filter)

        if kwargs:
            structured_filter = utils.get_structured_filter(**kwargs)
            hdlr.addFilter(structured_filter)

        hdlr.setFormatter(formatter)

    def zip(self, *fmtrs):
        """Format high/low handler properties so that they can be conveniently
            passed to `update_hdlr`.

        Args:
            fmtrs (seq[obj]): A sequence of :class:`logging.Formatter`
                instances.

        See also:
            :meth:`pygogo.Gogo.update_hdlr`

            :meth:`pygogo.Gogo.get_logger`

            :meth:`pygogo.Gogo.get_structured_logger`

        Returns:
            New instance of :class:`logging.handlers`

        Examples:
            >>> hdlr = logging.StreamHandler(sys.stdout)
            >>> copy_hdlr(hdlr) # doctest: +ELLIPSIS
            <...StreamHandler...>
        """
        hdlrs = [self.handlers['high'], self.handlers['low']]
        levels = [self.levels['high'], self.levels['low']]
        monologs = [False, self.monolog]
        return zip(hdlrs, levels, fmtrs, monologs)

    def get_logger(self, name='base', **kwargs):
        """Retrieve a named logger.

        Args:
            name (string): The logger name.

        See also:
            :func:`pygogo.copy_hdlr`

            :meth:`pygogo.Gogo.update_hdlr`

            :meth:`pygogo.Gogo.zip`

        Returns:
            New instance of :class:`logging.Logger`

        Examples:
            >>> going = Gogo()
            >>> logger = going.get_logger('default')
            >>> logger # doctest: +ELLIPSIS
            <...Logger...>
            >>> logger.info('from default')
            from default
            >>> going.get_logger('new').info('from new')
            from new
        """
        lggr_name = '%s.%s' % (self.name, name)
        logger = logging.getLogger(lggr_name)

        if lggr_name not in self.loggers:
            self.loggers.add(lggr_name)

            if kwargs:
                kwargs['name'] = lggr_name

            fmtrs = [self.formatters['high'], self.formatters['low']]

            for zipped in self.zip(*fmtrs):
                hdlr, level, fmtr, monolog = zipped
                copied_hdlr = copy_hdlr(hdlr)
                self.update_hdlr(copied_hdlr, level, fmtr, monolog, **kwargs)
                logger.addHandler(copied_hdlr)

            logger.setLevel(self.levels['low'])

        return logger

    def get_structured_logger(self, name=None, **kwargs):
        """Retrieve a structured data logger

        Args:
            name (string): The logger name.

            kwargs (dict): Keyword arguments to include in every log message.

        See also:
            :func:`pygogo.copy_hdlr`

            :meth:`pygogo.Gogo.update_hdlr`

            :meth:`pygogo.Gogo.zip`

            :class:`pygogo.utils.StructuredAdapter`

        Returns:
            New instance of :class:`pygogo.utils.StructuredAdapter`

        Examples
            >>> from io import StringIO
            >>> from json import loads

            >>> s = StringIO()
            >>> going = Gogo('structured', low_hdlr=handlers.fileobj_hdlr(s))
            >>> logger = going.get_structured_logger(all='true')
            >>> logger  # doctest: +ELLIPSIS
            <...StructuredAdapter...>
            >>> logger.debug('hello')
            >>> logger.debug('extra', extra={'key': 'value'})
            >>> s.seek(0) or 0
            0
            >>> loads(next(s)) == {'all': 'true', 'message': 'hello'}
            True
            >>> loads(next(s)) == {
            ...     'all': 'true', 'message': 'extra', 'key': 'value'}
            True
        """
        # pylint: disable=dict-items-not-iterating
        values = frozenset(kwargs.items())
        name = name or hashlib.md5(str(values).encode('utf-8')).hexdigest()
        lggr_name = '%s.structured.%s' % (self.name, name)
        logger = logging.getLogger(lggr_name)

        if lggr_name not in self.loggers:
            self.loggers.add(lggr_name)
            formatter = formatters.basic_formatter

            for zipped in self.zip(formatter, formatter):
                hdlr, level, fmtr, monolog = zipped
                copied_hdlr = copy_hdlr(hdlr)
                self.update_hdlr(copied_hdlr, level, fmtr, monolog)
                logger.addHandler(copied_hdlr)

            logger.setLevel(self.levels['low'])

        return utils.StructuredAdapter(logger, kwargs)

def copy_hdlr(hdlr):
    """Safely copy a handler and its associated filters.

    Args:
        hdlr (obj): A :class:`logging.handlers` instance.

    See also:
        :meth:`pygogo.Gogo.get_logger`

        :meth:`pygogo.Gogo.get_structured_logger`

    Returns:
        New instance of :class:`logging.handlers`

    Examples:
        >>> hdlr = logging.StreamHandler(sys.stdout)
        >>> copy_hdlr(hdlr) # doctest: +ELLIPSIS
        <...StreamHandler...>
    """
    copied_hdlr = copy(hdlr)
    copied_hdlr.filters = [copy(f) for f in hdlr.filters]
    return copied_hdlr

logger = Gogo().logger
<|endoftext|>


<commit_before>"""
sentry.coreapi
~~~~~~~~~~~~~~

:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
# TODO: We should make the API a class, and UDP/HTTP just inherit from it
#       This will make it so we can more easily control logging with various
#       metadata (rather than generic log messages which aren't useful).

from datetime import datetime, timedelta
import base64
import logging
import uuid
import zlib

from django.utils.encoding import smart_str

from sentry.app import env
from sentry.conf import settings
from sentry.exceptions import InvalidTimestamp
from sentry.models import Project, ProjectKey, TeamMember, Team, User
from sentry.plugins import plugins
from sentry.tasks.store import preprocess_event
from sentry.utils import is_float, json
from sentry.utils.auth import parse_auth_header
from sentry.utils.imports import import_string
from sentry.utils.strings import decompress

logger = logging.getLogger('sentry.coreapi.errors')

MAX_CULPRIT_LENGTH = 200
MAX_MESSAGE_LENGTH = 5000

INTERFACE_ALIASES = {
    'exception': 'sentry.interfaces.Exception',
    'request': 'sentry.interfaces.Http',
    'user': 'sentry.interfaces.User',
    'stacktrace': 'sentry.interfaces.Stacktrace',
    'template': 'sentry.interfaces.Template',
}

RESERVED_FIELDS = (
    'project',
    'event_id',
    'message',
    'checksum',
    'culprit',
    'level',
    'time_spent',
    'logger',
    'server_name',
    'site',
    'timestamp',
    'extra',
    'modules',
    'tags',
    'platform',
)

class APIError(Exception):
    http_status = 400
    msg = 'Invalid request'

    def __init__(self, msg=None):
        if msg:
            self.msg = msg

    def __str__(self):
        return self.msg or ''

class APIUnauthorized(APIError):
    http_status = 401
    msg = 'Unauthorized'

class APIForbidden(APIError):
    http_status = 403

class APITimestampExpired(APIError):
    http_status = 410

def client_metadata(client=None, exception=None, tags=None, extra=None):
    if not extra:
        extra = {}
    if not tags:
        tags = {}

    extra['client'] = client
    extra['request'] = env.request
    extra['tags'] = tags

    tags['client'] = client
    if exception:
        tags['exc_type'] = type(exception).__name__

    result = {'extra': extra}
    if exception:
        result['exc_info'] = True
    return result

def extract_auth_vars(request):
    if request.META.get('HTTP_X_SENTRY_AUTH', '').startswith('Sentry'):
        return parse_auth_header(request.META['HTTP_X_SENTRY_AUTH'])
    elif request.META.get('HTTP_AUTHORIZATION', '').startswith('Sentry'):
        return parse_auth_header(request.META['HTTP_AUTHORIZATION'])
    else:
        return request.GET

def project_from_auth_vars(auth_vars):
    api_key = auth_vars.get('sentry_key')
    if not api_key:
        raise APIForbidden('Invalid api key')
    try:
        pk = ProjectKey.objects.get_from_cache(public_key=api_key)
    except ProjectKey.DoesNotExist:
        raise APIForbidden('Invalid api key')

    if pk.secret_key != auth_vars.get('sentry_secret', pk.secret_key):
        raise APIForbidden('Invalid api key')

    project = Project.objects.get_from_cache(pk=pk.project_id)

    if pk.user_id:
        try:
            team = Team.objects.get_from_cache(pk=project.team_id)
        except Team.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        try:
            TeamMember.objects.get(team=team, user=pk.user_id, is_active=True)
        except TeamMember.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        # We have to refetch this as it may have been caught
        pk.user = User.objects.get(id=pk.user_id)
        if not pk.user.is_active:
            raise APIUnauthorized('Account is not active')

    return project, pk.user

def project_from_api_key_and_id(api_key, project_id):
    """
    Given a public api key and a project id returns
    a project instance or throws APIUnauthorized.
    """
    try:
        pk = ProjectKey.objects.get_from_cache(public_key=api_key)
    except ProjectKey.DoesNotExist:
        raise APIUnauthorized('Invalid api key')

    if str(project_id).isdigit():
        if str(pk.project_id) != str(project_id):
            raise APIUnauthorized('Invalid project id')
    else:
        if str(pk.project.slug) != str(project_id):
            raise APIUnauthorized('Invalid project id')

    project = Project.objects.get_from_cache(pk=pk.project_id)

    if pk.user:
        team = Team.objects.get_from_cache(pk=project.team_id)

        try:
            tm = TeamMember.objects.get(team=team, user=pk.user, is_active=True)
        except TeamMember.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        # We have to refetch this as it may have been caught
        pk.user = User.objects.get(id=pk.user_id)
        if not pk.user.is_active:
            raise APIUnauthorized('Account is not active')

        tm.project = project

        result = plugins.first('has_perm', tm.user, 'create_event', project)
        if result is False:
            raise APIForbidden('Creation of this event was blocked')

    return project

def project_from_id(request):
    """
    Given a request returns a project instance or throws
    APIUnauthorized.
    """
    if not request.user.is_active:
        raise APIUnauthorized('Account is not active')

    try:
        project = Project.objects.get_from_cache(pk=request.GET['project_id'])
    except Project.DoesNotExist:
        raise APIUnauthorized('Invalid project')

    try:
        team = Team.objects.get_from_cache(pk=project.team_id)
    except Project.DoesNotExist:
        raise APIUnauthorized('Member does not have access to project')

    try:
        TeamMember.objects.get(
            user=request.user,
            team=team,
            is_active=True,
        )
    except TeamMember.DoesNotExist:
        raise APIUnauthorized('Member does not have access to project')

    result = plugins.first('has_perm', request.user, 'create_event', project)
    if result is False:
        raise APIForbidden('Creation of this event was blocked')

    return project

def decode_and_decompress_data(encoded_data):
    try:
        try:
            return decompress(encoded_data)
        except zlib.error:
            return base64.b64decode(encoded_data)
    except Exception, e:
        # This error should be caught as it suggests that there's a
        # bug somewhere in the client's code.
        logger.info(e, **client_metadata(exception=e))
        raise APIForbidden('Bad data decoding request (%s, %s)' % (
            e.__class__.__name__, e))

def safely_load_json_string(json_string):
    try:
        obj = json.loads(json_string)
    except Exception, e:
        # This error should be caught as it suggests that there's a
        # bug somewhere in the client's code.
        logger.info(e, **client_metadata(exception=e))
        raise APIForbidden('Bad data reconstructing object (%s, %s)' % (
            e.__class__.__name__, e))

    # XXX: ensure keys are coerced to strings
    return dict((smart_str(k), v) for k, v in obj.iteritems())

def ensure_valid_project_id(desired_project, data, client=None):
    # Confirm they're using either the master key, or their specified project
    # matches with the signed project.
    if desired_project and data.get('project'):
        if str(data.get('project')) not in [str(desired_project.id), desired_project.slug]:
            logger.info('Project ID mismatch: %s != %s', desired_project.id, desired_project.slug,
                **client_metadata(client))
            raise APIForbidden('Invalid credentials')
        data['project'] = desired_project.id
    elif not desired_project:
        data['project'] = 1
    elif not data.get('project'):
        data['project'] = desired_project.id

def process_data_timestamp(data):
    if is_float(data['timestamp']):
        try:
            data['timestamp'] = datetime.fromtimestamp(float(data['timestamp']))
        except Exception:
            raise InvalidTimestamp('Invalid value for timestamp: %r' % data['timestamp'])
    elif not isinstance(data['timestamp'], datetime):
        if '.' in data['timestamp']:
            format = '%Y-%m-%dT%H:%M:%S.%f'
        else:
            format = '%Y-%m-%dT%H:%M:%S'
        if 'Z' in data['timestamp']:
            # support UTC market, but not other timestamps
            format += 'Z'
        try:
            data['timestamp'] = datetime.strptime(data['timestamp'], format)
        except Exception:
            raise InvalidTimestamp('Invalid value for timestamp: %r' % data['timestamp'])

    if data['timestamp'] > datetime.now() + timedelta(minutes=1):
        raise InvalidTimestamp('Invalid value for timestamp (in future): %r' % data['timestamp'])

    return data

def validate_data(project, data, client=None):
    ensure_valid_project_id(project, data, client=client)

    if not data.get('message'):
        data['message'] = '<no message value>'
    elif not isinstance(data['message'], basestring):
        raise APIError('Invalid value for message')
    elif len(data['message']) > MAX_MESSAGE_LENGTH:
        logger.info('Truncated value for message due to length (%d chars)', len(data['message']),
            **client_metadata(client))
        data['message'] = data['message'][:MAX_MESSAGE_LENGTH]

    if data.get('culprit') and len(data['culprit']) > MAX_CULPRIT_LENGTH:
        logger.info('Truncated value for culprit due to length (%d chars)', len(data['culprit']),
            **client_metadata(client))
        data['culprit'] = data['culprit'][:MAX_CULPRIT_LENGTH]

    if not data.get('event_id'):
        data['event_id'] = uuid.uuid4().hex
    if len(data['event_id']) > 32:
        logger.info('Discarded value for event_id due to length (%d chars)', len(data['event_id']),
            **client_metadata(client))
        data['event_id'] = uuid.uuid4().hex

    if 'timestamp' in data:
        try:
            process_data_timestamp(data)
        except InvalidTimestamp, e:
            # Log the error, remove the timestamp, and continue
            logger.info('Discarded invalid value for timestamp: %r', data['timestamp'],
                **client_metadata(client, exception=e))
            del data['timestamp']

    if data.get('modules') and type(data['modules']) != dict:
        logger.info('Discarded invalid type for modules: %s', type(data['modules']),
            **client_metadata(client))
        del data['modules']

    if data.get('extra') and type(data['extra']) != dict:
        logger.info('Discarded invalid type for extra: %s', type(data['extra']),
            **client_metadata(client))
        del data['extra']

    for k in data.keys():
        if k in RESERVED_FIELDS:
            continue

        if not data[k]:
            logger.info('Ignored empty interface value: %s', k, **client_metadata(client))
            del data[k]
            continue

        import_path = INTERFACE_ALIASES.get(k, k)

        if '.' not in import_path:
            logger.info('Ignored unknown attribute: %s', k, **client_metadata(client))
            del data[k]
            continue

        try:
            interface = import_string(import_path)
        except (ImportError, AttributeError), e:
            logger.info('Invalid unknown attribute: %s', k, **client_metadata(client, exception=e))
            del data[k]
            continue

        value = data.pop(k)
        try:
            inst = interface(**value)
            inst.validate()
            data[import_path] = inst.serialize()
        except Exception, e:
            if isinstance(e, AssertionError):
                log = logger.info
            else:
                log = logger.error
            log('Discarded invalid value for interface: %s', k,
                **client_metadata(client, exception=e, extra={'value': value}))

    level = data.get('level') or settings.DEFAULT_LOG_LEVEL
    if isinstance(level, basestring) and not level.isdigit():
        # assume it's something like 'warning'
        try:
            data['level'] = settings.LOG_LEVEL_REVERSE_MAP[level]
        except KeyError, e:
            logger.info('Discarded invalid logger value: %s', level, **client_metadata(client, exception=e))
            data['level'] = settings.LOG_LEVEL_REVERSE_MAP.get(settings.DEFAULT_LOG_LEVEL,
                settings.DEFAULT_LOG_LEVEL)

    return data

def insert_data_to_database(data):
    preprocess_event.delay(data=data)
<commit_msg>Pull user from cache<commit_after>"""
sentry.coreapi
~~~~~~~~~~~~~~

:copyright: (c) 2010-2013 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
# TODO: We should make the API a class, and UDP/HTTP just inherit from it
#       This will make it so we can more easily control logging with various
#       metadata (rather than generic log messages which aren't useful).

from datetime import datetime, timedelta
import base64
import logging
import uuid
import zlib

from django.utils.encoding import smart_str

from sentry.app import env
from sentry.conf import settings
from sentry.exceptions import InvalidTimestamp
from sentry.models import Project, ProjectKey, TeamMember, Team, User
from sentry.plugins import plugins
from sentry.tasks.store import preprocess_event
from sentry.utils import is_float, json
from sentry.utils.auth import parse_auth_header
from sentry.utils.imports import import_string
from sentry.utils.strings import decompress

logger = logging.getLogger('sentry.coreapi.errors')

MAX_CULPRIT_LENGTH = 200
MAX_MESSAGE_LENGTH = 5000

INTERFACE_ALIASES = {
    'exception': 'sentry.interfaces.Exception',
    'request': 'sentry.interfaces.Http',
    'user': 'sentry.interfaces.User',
    'stacktrace': 'sentry.interfaces.Stacktrace',
    'template': 'sentry.interfaces.Template',
}

RESERVED_FIELDS = (
    'project',
    'event_id',
    'message',
    'checksum',
    'culprit',
    'level',
    'time_spent',
    'logger',
    'server_name',
    'site',
    'timestamp',
    'extra',
    'modules',
    'tags',
    'platform',
)

class APIError(Exception):
    http_status = 400
    msg = 'Invalid request'

    def __init__(self, msg=None):
        if msg:
            self.msg = msg

    def __str__(self):
        return self.msg or ''

class APIUnauthorized(APIError):
    http_status = 401
    msg = 'Unauthorized'

class APIForbidden(APIError):
    http_status = 403

class APITimestampExpired(APIError):
    http_status = 410

def client_metadata(client=None, exception=None, tags=None, extra=None):
    if not extra:
        extra = {}
    if not tags:
        tags = {}

    extra['client'] = client
    extra['request'] = env.request
    extra['tags'] = tags

    tags['client'] = client
    if exception:
        tags['exc_type'] = type(exception).__name__

    result = {'extra': extra}
    if exception:
        result['exc_info'] = True
    return result

def extract_auth_vars(request):
    if request.META.get('HTTP_X_SENTRY_AUTH', '').startswith('Sentry'):
        return parse_auth_header(request.META['HTTP_X_SENTRY_AUTH'])
    elif request.META.get('HTTP_AUTHORIZATION', '').startswith('Sentry'):
        return parse_auth_header(request.META['HTTP_AUTHORIZATION'])
    else:
        return request.GET

def project_from_auth_vars(auth_vars):
    api_key = auth_vars.get('sentry_key')
    if not api_key:
        raise APIForbidden('Invalid api key')
    try:
        pk = ProjectKey.objects.get_from_cache(public_key=api_key)
    except ProjectKey.DoesNotExist:
        raise APIForbidden('Invalid api key')

    if pk.secret_key != auth_vars.get('sentry_secret', pk.secret_key):
        raise APIForbidden('Invalid api key')

    project = Project.objects.get_from_cache(pk=pk.project_id)

    if pk.user_id:
        try:
            team = Team.objects.get_from_cache(pk=project.team_id)
        except Team.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        try:
            TeamMember.objects.get(team=team, user=pk.user_id, is_active=True)
        except TeamMember.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        # We have to refetch this as it may have been caught
        pk.user = User.objects.get(id=pk.user_id)
        if not pk.user.is_active:
            raise APIUnauthorized('Account is not active')

    return project, pk.user

def project_from_api_key_and_id(api_key, project_id):
    """
    Given a public api key and a project id returns
    a project instance or throws APIUnauthorized.
    """
    try:
        pk = ProjectKey.objects.get_from_cache(public_key=api_key)
    except ProjectKey.DoesNotExist:
        raise APIUnauthorized('Invalid api key')

    if str(project_id).isdigit():
        if str(pk.project_id) != str(project_id):
            raise APIUnauthorized('Invalid project id')
    else:
        if str(pk.project.slug) != str(project_id):
            raise APIUnauthorized('Invalid project id')

    project = Project.objects.get_from_cache(pk=pk.project_id)

    if pk.user:
        team = Team.objects.get_from_cache(pk=project.team_id)

        try:
            tm = TeamMember.objects.get(team=team, user=pk.user, is_active=True)
        except TeamMember.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        # We have to refetch this as it may have been caught
        pk.user = User.objects.get_from_cache(id=pk.user_id)
        if not pk.user.is_active:
            raise APIUnauthorized('Account is not active')

        tm.project = project

        result = plugins.first('has_perm', tm.user, 'create_event', project)
        if result is False:
            raise APIForbidden('Creation of this event was blocked')

    return project

def project_from_id(request):
    """
    Given a request returns a project instance or throws
    APIUnauthorized.
    """
    if not request.user.is_active:
        raise APIUnauthorized('Account is not active')

    try:
        project = Project.objects.get_from_cache(pk=request.GET['project_id'])
    except Project.DoesNotExist:
        raise APIUnauthorized('Invalid project')

    try:
        team = Team.objects.get_from_cache(pk=project.team_id)
    except Project.DoesNotExist:
        raise APIUnauthorized('Member does not have access to project')

    try:
        TeamMember.objects.get(
            user=request.user,
            team=team,
            is_active=True,
        )
    except TeamMember.DoesNotExist:
        raise APIUnauthorized('Member does not have access to project')

    result = plugins.first('has_perm', request.user, 'create_event', project)
    if result is False:
        raise APIForbidden('Creation of this event was blocked')

    return project

def decode_and_decompress_data(encoded_data):
    try:
        try:
            return decompress(encoded_data)
        except zlib.error:
            return base64.b64decode(encoded_data)
    except Exception, e:
        # This error should be caught as it suggests that there's a
        # bug somewhere in the client's code.
        logger.info(e, **client_metadata(exception=e))
        raise APIForbidden('Bad data decoding request (%s, %s)' % (
            e.__class__.__name__, e))

def safely_load_json_string(json_string):
    try:
        obj = json.loads(json_string)
    except Exception, e:
        # This error should be caught as it suggests that there's a
        # bug somewhere in the client's code.
        logger.info(e, **client_metadata(exception=e))
        raise APIForbidden('Bad data reconstructing object (%s, %s)' % (
            e.__class__.__name__, e))

    # XXX: ensure keys are coerced to strings
    return dict((smart_str(k), v) for k, v in obj.iteritems())

def ensure_valid_project_id(desired_project, data, client=None):
    # Confirm they're using either the master key, or their specified project
    # matches with the signed project.
    if desired_project and data.get('project'):
        if str(data.get('project')) not in [str(desired_project.id), desired_project.slug]:
            logger.info('Project ID mismatch: %s != %s', desired_project.id, desired_project.slug,
                **client_metadata(client))
            raise APIForbidden('Invalid credentials')
        data['project'] = desired_project.id
    elif not desired_project:
        data['project'] = 1
    elif not data.get('project'):
        data['project'] = desired_project.id

def process_data_timestamp(data):
    if is_float(data['timestamp']):
        try:
            data['timestamp'] = datetime.fromtimestamp(float(data['timestamp']))
        except Exception:
            raise InvalidTimestamp('Invalid value for timestamp: %r' % data['timestamp'])
    elif not isinstance(data['timestamp'], datetime):
        if '.' in data['timestamp']:
            format = '%Y-%m-%dT%H:%M:%S.%f'
        else:
            format = '%Y-%m-%dT%H:%M:%S'
        if 'Z' in data['timestamp']:
            # support UTC market, but not other timestamps
            format += 'Z'
        try:
            data['timestamp'] = datetime.strptime(data['timestamp'], format)
        except Exception:
            raise InvalidTimestamp('Invalid value for timestamp: %r' % data['timestamp'])

    if data['timestamp'] > datetime.now() + timedelta(minutes=1):
        raise InvalidTimestamp('Invalid value for timestamp (in future): %r' % data['timestamp'])

    return data

def validate_data(project, data, client=None):
    ensure_valid_project_id(project, data, client=client)

    if not data.get('message'):
        data['message'] = '<no message value>'
    elif not isinstance(data['message'], basestring):
        raise APIError('Invalid value for message')
    elif len(data['message']) > MAX_MESSAGE_LENGTH:
        logger.info('Truncated value for message due to length (%d chars)', len(data['message']),
            **client_metadata(client))
        data['message'] = data['message'][:MAX_MESSAGE_LENGTH]

    if data.get('culprit') and len(data['culprit']) > MAX_CULPRIT_LENGTH:
        logger.info('Truncated value for culprit due to length (%d chars)', len(data['culprit']),
            **client_metadata(client))
        data['culprit'] = data['culprit'][:MAX_CULPRIT_LENGTH]

    if not data.get('event_id'):
        data['event_id'] = uuid.uuid4().hex
    if len(data['event_id']) > 32:
        logger.info('Discarded value for event_id due to length (%d chars)', len(data['event_id']),
            **client_metadata(client))
        data['event_id'] = uuid.uuid4().hex

    if 'timestamp' in data:
        try:
            process_data_timestamp(data)
        except InvalidTimestamp, e:
            # Log the error, remove the timestamp, and continue
            logger.info('Discarded invalid value for timestamp: %r', data['timestamp'],
                **client_metadata(client, exception=e))
            del data['timestamp']

    if data.get('modules') and type(data['modules']) != dict:
        logger.info('Discarded invalid type for modules: %s', type(data['modules']),
            **client_metadata(client))
        del data['modules']

    if data.get('extra') and type(data['extra']) != dict:
        logger.info('Discarded invalid type for extra: %s', type(data['extra']),
            **client_metadata(client))
        del data['extra']

    for k in data.keys():
        if k in RESERVED_FIELDS:
            continue

        if not data[k]:
            logger.info('Ignored empty interface value: %s', k, **client_metadata(client))
            del data[k]
            continue

        import_path = INTERFACE_ALIASES.get(k, k)

        if '.' not in import_path:
            logger.info('Ignored unknown attribute: %s', k, **client_metadata(client))
            del data[k]
            continue

        try:
            interface = import_string(import_path)
        except (ImportError, AttributeError), e:
            logger.info('Invalid unknown attribute: %s', k, **client_metadata(client, exception=e))
            del data[k]
            continue

        value = data.pop(k)
        try:
            inst = interface(**value)
            inst.validate()
            data[import_path] = inst.serialize()
        except Exception, e:
            if isinstance(e, AssertionError):
                log = logger.info
            else:
                log = logger.error
            log('Discarded invalid value for interface: %s', k,
                **client_metadata(client, exception=e, extra={'value': value}))

    level = data.get('level') or settings.DEFAULT_LOG_LEVEL
    if isinstance(level, basestring) and not level.isdigit():
        # assume it's something like 'warning'
        try:
            data['level'] = settings.LOG_LEVEL_REVERSE_MAP[level]
        except KeyError, e:
            logger.info('Discarded invalid logger value: %s', level, **client_metadata(client, exception=e))
            data['level'] = settings.LOG_LEVEL_REVERSE_MAP.get(settings.DEFAULT_LOG_LEVEL,
                settings.DEFAULT_LOG_LEVEL)

    return data

def insert_data_to_database(data):
    preprocess_event.delay(data=data)
<|endoftext|>


<commit_before>from __future__ import absolute_import
import random

from .jokes_en import jokes_en
from .jokes_de import jokes_de
from .jokes_es import jokes_es

all_jokes = {
    'en': jokes_en,
    'de': jokes_de,
    'es': jokes_es,
}

class LanguageNotFoundError(Exception):
    pass

class CategoryNotFoundError(Exception):
    pass

def _get_jokes(language='en', category='neutral', singular=True):
    if language not in all_jokes:
        raise LanguageNotFoundError('No such language %s' % language)

    jokes = all_jokes[language]

    if category not in jokes:
        raise CategoryNotFound('No such category %s' % category)

    jokes = jokes[category]

    if singular:
        return random.choice(jokes)
    else:
        return jokes

def get_joke(language='en', category='neutral'):
    """
    Parameters
    ----------
    category: str
        Choices: 'neutral', 'explicit', 'chuck', 'all'
    lang: str
        Choices: 'en', 'de', 'es'

    Returns
    -------
    joke: str
    """

    return _get_jokes(language, category, singular=True)

def get_jokes(language='en', category='neutral'):
    """
    Parameters
    ----------
    category: str
        Choices: 'neutral', 'explicit', 'chuck', 'all'
    lang: str
        Choices: 'en', 'de', 'es'

    Returns
    -------
    jokes: list
    """

    return _get_jokes(language, category, singular=False)
<commit_msg>Fix exception type<commit_after>from __future__ import absolute_import
import random

from .jokes_en import jokes_en
from .jokes_de import jokes_de
from .jokes_es import jokes_es

all_jokes = {
    'en': jokes_en,
    'de': jokes_de,
    'es': jokes_es,
}

class LanguageNotFoundError(Exception):
    pass

class CategoryNotFoundError(Exception):
    pass

def _get_jokes(language='en', category='neutral', singular=True):
    if language not in all_jokes:
        raise LanguageNotFoundError('No such language %s' % language)

    jokes = all_jokes[language]

    if category not in jokes:
        raise CategoryNotFoundError('No such category %s' % category)

    jokes = jokes[category]

    if singular:
        return random.choice(jokes)
    else:
        return jokes

def get_joke(language='en', category='neutral'):
    """
    Parameters
    ----------
    category: str
        Choices: 'neutral', 'explicit', 'chuck', 'all'
    lang: str
        Choices: 'en', 'de', 'es'

    Returns
    -------
    joke: str
    """

    return _get_jokes(language, category, singular=True)

def get_jokes(language='en', category='neutral'):
    """
    Parameters
    ----------
    category: str
        Choices: 'neutral', 'explicit', 'chuck', 'all'
    lang: str
        Choices: 'en', 'de', 'es'

    Returns
    -------
    jokes: list
    """

    return _get_jokes(language, category, singular=False)
<|endoftext|>


<commit_before>"""
sentry.coreapi
~~~~~~~~~~~~~~

:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
# TODO: We should make the API a class, and UDP/HTTP just inherit from it
#       This will make it so we can more easily control logging with various
#       metadata (rather than generic log messages which arent useful).

from datetime import datetime, timedelta
import base64
import logging
import uuid
import zlib

from django.contrib.auth.models import User
from django.utils.encoding import smart_str

from sentry.conf import settings
from sentry.exceptions import InvalidInterface, InvalidData, InvalidTimestamp
from sentry.models import Project, ProjectKey, TeamMember, Team
from sentry.plugins import plugins
from sentry.tasks.store import store_event
from sentry.utils import is_float, json
from sentry.utils.auth import parse_auth_header
from sentry.utils.imports import import_string
from sentry.utils.queue import maybe_delay

logger = logging.getLogger('sentry.errors.coreapi')

MAX_MESSAGE_LENGTH = 1000

RESERVED_FIELDS = (
    'project',
    'event_id',
    'message',
    'checksum',
    'culprit',
    'level',
    'time_spent',
    'logger',
    'server_name',
    'site',
    'timestamp',
    'extra',
    'modules',
    'tags',
    'platform',
)

class APIError(Exception):
    http_status = 400
    msg = 'Invalid request'

    def __init__(self, msg=None):
        if msg:
            self.msg = msg

    def __str__(self):
        return self.msg or ''

class APIUnauthorized(APIError):
    http_status = 401
    msg = 'Unauthorized'

class APIForbidden(APIError):
    http_status = 403

class APITimestampExpired(APIError):
    http_status = 410

def extract_auth_vars(request):
    if request.META.get('HTTP_X_SENTRY_AUTH', '').startswith('Sentry'):
        return parse_auth_header(request.META['HTTP_X_SENTRY_AUTH'])
    elif request.META.get('HTTP_AUTHORIZATION', '').startswith('Sentry'):
        return parse_auth_header(request.META['HTTP_AUTHORIZATION'])
    else:
        return request.GET

def project_from_auth_vars(auth_vars):
    api_key = auth_vars.get('sentry_key')
    if not api_key:
        raise APIForbidden('Invalid api key')
    try:
        pk = ProjectKey.objects.get_from_cache(public_key=api_key)
    except ProjectKey.DoesNotExist:
        raise APIForbidden('Invalid api key')

    if pk.secret_key != auth_vars.get('sentry_secret', pk.secret_key):
        raise APIForbidden('Invalid api key')

    project = Project.objects.get_from_cache(pk=pk.project_id)

    if pk.user:
        try:
            team = Team.objects.get_from_cache(pk=project.team_id)
        except Team.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        try:
            TeamMember.objects.get(team=team, user=pk.user, is_active=True)
        except TeamMember.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        # We have to refetch this as it may have been catched
        pk.user = User.objects.get(id=pk.user_id)
        if not pk.user.is_active:
            raise APIUnauthorized('Account is not active')

    return project, pk.user

def project_from_api_key_and_id(api_key, project_id):
    """
    Given a public api key and a project id returns
    a project instance or throws APIUnauthorized.
    """
    try:
        pk = ProjectKey.objects.get_from_cache(public_key=api_key)
    except ProjectKey.DoesNotExist:
        raise APIUnauthorized('Invalid api key')

    if str(project_id).isdigit():
        if str(pk.project_id) != str(project_id):
            raise APIUnauthorized('Invalid project id')
    else:
        if str(pk.project.slug) != str(project_id):
            raise APIUnauthorized('Invalid project id')

    project = Project.objects.get_from_cache(pk=pk.project_id)

    if pk.user:
        team = Team.objects.get_from_cache(pk=project.team_id)

        try:
            tm = TeamMember.objects.get(team=team, user=pk.user, is_active=True)
        except TeamMember.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        # We have to refetch this as it may have been catched
        pk.user = User.objects.get(id=pk.user_id)
        if not pk.user.is_active:
            raise APIUnauthorized('Account is not active')

        tm.project = project

        result = plugins.first('has_perm', tm.user, 'create_event', project)
        if result is False:
            raise APIForbidden('Creation of this event was blocked')

    return project

def project_from_id(request):
    """
    Given a request returns a project instance or throws
    APIUnauthorized.
    """
    if not request.user.is_active:
        raise APIUnauthorized('Account is not active')

    try:
        project = Project.objects.get_from_cache(pk=request.GET['project_id'])
    except Project.DoesNotExist:
        raise APIUnauthorized('Invalid project')

    try:
        team = Team.objects.get_from_cache(pk=project.team_id)
    except Project.DoesNotExist:
        raise APIUnauthorized('Member does not have access to project')

    try:
        TeamMember.objects.get(
            user=request.user,
            team=team,
            is_active=True,
        )
    except TeamMember.DoesNotExist:
        raise APIUnauthorized('Member does not have access to project')

    result = plugins.first('has_perm', request.user, 'create_event', project)
    if result is False:
        raise APIForbidden('Creation of this event was blocked')

    return project

def decode_and_decompress_data(encoded_data):
    try:
        try:
            return base64.b64decode(encoded_data).decode('zlib')
        except zlib.error:
            return base64.b64decode(encoded_data)
    except Exception, e:
        # This error should be caught as it suggests that there's a
        # bug somewhere in the client's code.
        logger.exception('Bad data received')
        raise APIForbidden('Bad data decoding request (%s, %s)' % (
            e.__class__.__name__, e))

def safely_load_json_string(json_string):
    try:
        obj = json.loads(json_string)
    except Exception, e:
        # This error should be caught as it suggests that there's a
        # bug somewhere in the client's code.
        logger.exception('Bad data received')
        raise APIForbidden('Bad data reconstructing object (%s, %s)' % (
            e.__class__.__name__, e))

    # XXX: ensure keys are coerced to strings
    return dict((smart_str(k), v) for k, v in obj.iteritems())

def ensure_valid_project_id(desired_project, data):
    # Confirm they're using either the master key, or their specified project
    # matches with the signed project.
    if desired_project and data.get('project'):
        if str(data.get('project')) not in [str(desired_project.id), desired_project.slug]:
            raise APIForbidden('Invalid credentials')
        data['project'] = desired_project.id
    elif not desired_project:
        data['project'] = 1
    elif not data.get('project'):
        data['project'] = desired_project.id

def process_data_timestamp(data):
    if is_float(data['timestamp']):
        try:
            data['timestamp'] = datetime.fromtimestamp(float(data['timestamp']))
        except Exception:
            raise InvalidTimestamp('Invalid value for timestamp: %r' % data['timestamp'])
    elif not isinstance(data['timestamp'], datetime):
        if '.' in data['timestamp']:
            format = '%Y-%m-%dT%H:%M:%S.%f'
        else:
            format = '%Y-%m-%dT%H:%M:%S'
        if 'Z' in data['timestamp']:
            # support UTC market, but not other timestamps
            format += 'Z'
        try:
            data['timestamp'] = datetime.strptime(data['timestamp'], format)
        except Exception:
            raise InvalidTimestamp('Invalid value for timestamp: %r' % data['timestamp'])

    if data['timestamp'] > datetime.now() + timedelta(minutes=1):
        raise InvalidTimestamp('Invalid value for timestamp (in future): %r' % data['timestamp'])

    return data

def validate_data(project, data, client=None):
    ensure_valid_project_id(project, data)

    if not data.get('message'):
        data['message'] = '<no message value>'
    elif len(data['message']) > MAX_MESSAGE_LENGTH:
        raise InvalidData('Value \'message\' is too long. At %d chars, max is %s.' % (
            len(data['message']), MAX_MESSAGE_LENGTH))

    if not data.get('event_id'):
        data['event_id'] = uuid.uuid4().hex
    elif len(data['event_id']) > 32:
        raise InvalidData('Invalid value for \'event_id\': must be a 32 character identifier')

    if 'timestamp' in data:
        try:
            process_data_timestamp(data)
        except InvalidTimestamp:
            # Log the error, remove the timestamp, and continue
            logger.error('Client %r passed an invalid value for timestamp %r',
                client or '<unknown client>',
                data['timestamp'],
            )
            del data['timestamp']

    if data.get('modules') and type(data['modules']) != dict:
        raise InvalidData('Invalid type for \'modules\': must be a mapping')

    for k, v in data.items():
        if k in RESERVED_FIELDS:
            continue

        if '.' not in k:
            logger.error('Ignoring unknown attribute %r passed by client %r',
                k,
                client or '<unknown client>',
            )
            del data[k]
            continue

        try:
            interface = import_string(k)
        except (ImportError, AttributeError), e:
            raise InvalidInterface('%r is not a valid interface name: %s' % (k, e))

        try:
            data[k] = interface(**v).serialize()
        except Exception, e:
            raise InvalidData('Unable to validate interface, %r: %s' % (k, e))

    level = data.get('level') or settings.DEFAULT_LOG_LEVEL
    if isinstance(level, basestring) and not level.isdigit():
        # assume it's something like 'warning'
        try:
            data['level'] = settings.LOG_LEVEL_REVERSE_MAP[level]
        except KeyError:
            raise InvalidData('Invalid logging level specified: %r' % level)

    return data

def insert_data_to_database(data):
    maybe_delay(store_event, data=data)
<commit_msg>Validate culprit length<commit_after>"""
sentry.coreapi
~~~~~~~~~~~~~~

:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
# TODO: We should make the API a class, and UDP/HTTP just inherit from it
#       This will make it so we can more easily control logging with various
#       metadata (rather than generic log messages which arent useful).

from datetime import datetime, timedelta
import base64
import logging
import uuid
import zlib

from django.contrib.auth.models import User
from django.utils.encoding import smart_str

from sentry.conf import settings
from sentry.exceptions import InvalidInterface, InvalidData, InvalidTimestamp
from sentry.models import Project, ProjectKey, TeamMember, Team
from sentry.plugins import plugins
from sentry.tasks.store import store_event
from sentry.utils import is_float, json
from sentry.utils.auth import parse_auth_header
from sentry.utils.imports import import_string
from sentry.utils.queue import maybe_delay

logger = logging.getLogger('sentry.errors.coreapi')

MAX_CULPRIT_LENGTH = 200
MAX_MESSAGE_LENGTH = 1000

RESERVED_FIELDS = (
    'project',
    'event_id',
    'message',
    'checksum',
    'culprit',
    'level',
    'time_spent',
    'logger',
    'server_name',
    'site',
    'timestamp',
    'extra',
    'modules',
    'tags',
    'platform',
)

class APIError(Exception):
    http_status = 400
    msg = 'Invalid request'

    def __init__(self, msg=None):
        if msg:
            self.msg = msg

    def __str__(self):
        return self.msg or ''

class APIUnauthorized(APIError):
    http_status = 401
    msg = 'Unauthorized'

class APIForbidden(APIError):
    http_status = 403

class APITimestampExpired(APIError):
    http_status = 410

def extract_auth_vars(request):
    if request.META.get('HTTP_X_SENTRY_AUTH', '').startswith('Sentry'):
        return parse_auth_header(request.META['HTTP_X_SENTRY_AUTH'])
    elif request.META.get('HTTP_AUTHORIZATION', '').startswith('Sentry'):
        return parse_auth_header(request.META['HTTP_AUTHORIZATION'])
    else:
        return request.GET

def project_from_auth_vars(auth_vars):
    api_key = auth_vars.get('sentry_key')
    if not api_key:
        raise APIForbidden('Invalid api key')
    try:
        pk = ProjectKey.objects.get_from_cache(public_key=api_key)
    except ProjectKey.DoesNotExist:
        raise APIForbidden('Invalid api key')

    if pk.secret_key != auth_vars.get('sentry_secret', pk.secret_key):
        raise APIForbidden('Invalid api key')

    project = Project.objects.get_from_cache(pk=pk.project_id)

    if pk.user:
        try:
            team = Team.objects.get_from_cache(pk=project.team_id)
        except Team.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        try:
            TeamMember.objects.get(team=team, user=pk.user, is_active=True)
        except TeamMember.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        # We have to refetch this as it may have been catched
        pk.user = User.objects.get(id=pk.user_id)
        if not pk.user.is_active:
            raise APIUnauthorized('Account is not active')

    return project, pk.user

def project_from_api_key_and_id(api_key, project_id):
    """
    Given a public api key and a project id returns
    a project instance or throws APIUnauthorized.
    """
    try:
        pk = ProjectKey.objects.get_from_cache(public_key=api_key)
    except ProjectKey.DoesNotExist:
        raise APIUnauthorized('Invalid api key')

    if str(project_id).isdigit():
        if str(pk.project_id) != str(project_id):
            raise APIUnauthorized('Invalid project id')
    else:
        if str(pk.project.slug) != str(project_id):
            raise APIUnauthorized('Invalid project id')

    project = Project.objects.get_from_cache(pk=pk.project_id)

    if pk.user:
        team = Team.objects.get_from_cache(pk=project.team_id)

        try:
            tm = TeamMember.objects.get(team=team, user=pk.user, is_active=True)
        except TeamMember.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        # We have to refetch this as it may have been catched
        pk.user = User.objects.get(id=pk.user_id)
        if not pk.user.is_active:
            raise APIUnauthorized('Account is not active')

        tm.project = project

        result = plugins.first('has_perm', tm.user, 'create_event', project)
        if result is False:
            raise APIForbidden('Creation of this event was blocked')

    return project

def project_from_id(request):
    """
    Given a request returns a project instance or throws
    APIUnauthorized.
    """
    if not request.user.is_active:
        raise APIUnauthorized('Account is not active')

    try:
        project = Project.objects.get_from_cache(pk=request.GET['project_id'])
    except Project.DoesNotExist:
        raise APIUnauthorized('Invalid project')

    try:
        team = Team.objects.get_from_cache(pk=project.team_id)
    except Project.DoesNotExist:
        raise APIUnauthorized('Member does not have access to project')

    try:
        TeamMember.objects.get(
            user=request.user,
            team=team,
            is_active=True,
        )
    except TeamMember.DoesNotExist:
        raise APIUnauthorized('Member does not have access to project')

    result = plugins.first('has_perm', request.user, 'create_event', project)
    if result is False:
        raise APIForbidden('Creation of this event was blocked')

    return project

def decode_and_decompress_data(encoded_data):
    try:
        try:
            return base64.b64decode(encoded_data).decode('zlib')
        except zlib.error:
            return base64.b64decode(encoded_data)
    except Exception, e:
        # This error should be caught as it suggests that there's a
        # bug somewhere in the client's code.
        logger.exception('Bad data received')
        raise APIForbidden('Bad data decoding request (%s, %s)' % (
            e.__class__.__name__, e))

def safely_load_json_string(json_string):
    try:
        obj = json.loads(json_string)
    except Exception, e:
        # This error should be caught as it suggests that there's a
        # bug somewhere in the client's code.
        logger.exception('Bad data received')
        raise APIForbidden('Bad data reconstructing object (%s, %s)' % (
            e.__class__.__name__, e))

    # XXX: ensure keys are coerced to strings
    return dict((smart_str(k), v) for k, v in obj.iteritems())

def ensure_valid_project_id(desired_project, data):
    # Confirm they're using either the master key, or their specified project
    # matches with the signed project.
    if desired_project and data.get('project'):
        if str(data.get('project')) not in [str(desired_project.id), desired_project.slug]:
            raise APIForbidden('Invalid credentials')
        data['project'] = desired_project.id
    elif not desired_project:
        data['project'] = 1
    elif not data.get('project'):
        data['project'] = desired_project.id

def process_data_timestamp(data):
    if is_float(data['timestamp']):
        try:
            data['timestamp'] = datetime.fromtimestamp(float(data['timestamp']))
        except Exception:
            raise InvalidTimestamp('Invalid value for timestamp: %r' % data['timestamp'])
    elif not isinstance(data['timestamp'], datetime):
        if '.' in data['timestamp']:
            format = '%Y-%m-%dT%H:%M:%S.%f'
        else:
            format = '%Y-%m-%dT%H:%M:%S'
        if 'Z' in data['timestamp']:
            # support UTC market, but not other timestamps
            format += 'Z'
        try:
            data['timestamp'] = datetime.strptime(data['timestamp'], format)
        except Exception:
            raise InvalidTimestamp('Invalid value for timestamp: %r' % data['timestamp'])

    if data['timestamp'] > datetime.now() + timedelta(minutes=1):
        raise InvalidTimestamp('Invalid value for timestamp (in future): %r' % data['timestamp'])

    return data

def validate_data(project, data, client=None):
    ensure_valid_project_id(project, data)

    if not data.get('message'):
        data['message'] = '<no message value>'
    elif len(data['message']) > MAX_MESSAGE_LENGTH:
        raise InvalidData('Value \'message\' is too long. Input is %d chars, max is %s.' % (
            len(data['message']), MAX_MESSAGE_LENGTH))

    if data.get('culprit') and len(data['culprit']) > 200:
        raise InvalidData('Value \'culprit\' is too long. Input is %d chars, max is %s.' % (
            len(data['culprit']), MAX_CULPRIT_LENGTH))

    if not data.get('event_id'):
        data['event_id'] = uuid.uuid4().hex
    elif len(data['event_id']) > 32:
        raise InvalidData('Invalid value for \'event_id\': must be a 32 character identifier')

    if 'timestamp' in data:
        try:
            process_data_timestamp(data)
        except InvalidTimestamp:
            # Log the error, remove the timestamp, and continue
            logger.error('Client %r passed an invalid value for timestamp %r',
                client or '<unknown client>',
                data['timestamp'],
            )
            del data['timestamp']

    if data.get('modules') and type(data['modules']) != dict:
        raise InvalidData('Invalid type for \'modules\': must be a mapping')

    for k, v in data.items():
        if k in RESERVED_FIELDS:
            continue

        if '.' not in k:
            logger.error('Ignoring unknown attribute %r passed by client %r',
                k,
                client or '<unknown client>',
            )
            del data[k]
            continue

        try:
            interface = import_string(k)
        except (ImportError, AttributeError), e:
            raise InvalidInterface('%r is not a valid interface name: %s' % (k, e))

        try:
            data[k] = interface(**v).serialize()
        except Exception, e:
            raise InvalidData('Unable to validate interface, %r: %s' % (k, e))

    level = data.get('level') or settings.DEFAULT_LOG_LEVEL
    if isinstance(level, basestring) and not level.isdigit():
        # assume it's something like 'warning'
        try:
            data['level'] = settings.LOG_LEVEL_REVERSE_MAP[level]
        except KeyError:
            raise InvalidData('Invalid logging level specified: %r' % level)

    return data

def insert_data_to_database(data):
    maybe_delay(store_event, data=data)
<|endoftext|>


<commit_before>from __future__ import division
"""
Author: Keith Bourgoin
"""
__license__ = """
Copyright 2015 Parse.ly, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
__all__ = ["Cluster"]
import logging
import random
import time
import weakref

from .broker import Broker
from .exceptions import (ConsumerCoordinatorNotAvailable,
                         UnknownTopicOrPartition)
from .protocol import ConsumerMetadataRequest, ConsumerMetadataResponse
from .topic import Topic

logger = logging.getLogger(__name__)

class TopicDict(dict):
    """Dictionary which will attempt to auto-create unknown topics."""

    def __init__(self, cluster, *args, **kwargs):
        super(TopicDict, self).__init__(*args, **kwargs)
        self._cluster = weakref.proxy(cluster)

    def __missing__(self, key):
        logger.info('Topic %s not found. Attempting to auto-create.', key)
        if self._create_topic(key):
            return self[key]
        else:
            raise UnknownTopicOrPartition('Unknown topic: {}'.format(key))

    def _create_topic(self, topic_name):
        """Auto-create a topic.

        Not exposed in the cluster or broker because this is *only*
        auto-creation.  When there's a real API for creating topics,
        with settings and everything, we'll implement that. To expose just
        this now would be disingenuous, since it's features would be hobbled.
        """
        # Auto-creating will take a moment, so we try 5 times.
        for i in xrange(5):
            # Auto-creating is as simple as issuing a metadata request
            # solely for that topic.  The update is just to be sure
            # our `Cluster` knows about it.
            self._cluster.brokers[0].request_metadata(topics=[topic_name])
            self._cluster.update()
            if topic_name in self:
                logger.info('Topic %s successfully auto-created.', topic_name)
                return True
            time.sleep(0.1)

class Cluster(object):
    """
    A Cluster is a high-level abstraction of the collection of brokers and
    topics that makes up a real kafka cluster.
    """
    def __init__(self,
                 hosts,
                 handler,
                 socket_timeout_ms=30 * 1000,
                 offsets_channel_socket_timeout_ms=10 * 1000,
                 socket_receive_buffer_bytes=64 * 1024,
                 exclude_internal_topics=True):
        """Create a new Cluster instance.

        :param hosts: Comma-separated list of kafka hosts to used to connect.
        :type hosts: str
        :param handler: The concurrency handler for network requests.
        :type handler: :class:`pykafka.handlers.Handler`
        :param socket_timeout_ms: The socket timeout (in milliseconds) for
            network requests
        :type socket_timeout_ms: int
        :param offsets_channel_socket_timeout_ms: The socket timeout (in
            milliseconds) when reading responses for offset commit and
            offset fetch requests.
        :type offsets_channel_socket_timeout_ms: int
        :param socket_receive_buffer_bytes: The size (in bytes) of the socket
            receive buffer for network requests.
        :type socket_receive_buffer_bytes: int
        :param exclude_internal_topics: Whether messages from internal topics
            (specifically, the offsets topic) should be exposed to consumers.
        :type exclude_internal_topics: bool
        """
        self._seed_hosts = hosts
        self._socket_timeout_ms = socket_timeout_ms
        self._offsets_channel_socket_timeout_ms = offsets_channel_socket_timeout_ms
        self._handler = handler
        self._brokers = {}
        self._topics = TopicDict(self)
        self._socket_receive_buffer_bytes = socket_receive_buffer_bytes
        self._exclude_internal_topics = exclude_internal_topics
        self.update()

    def __repr__(self):
        return "<{}.{} at {} (hosts={})>".format(
            self.__class__.__module__,
            self.__class__.__name__,
            hex(id(self)),
            self._seed_hosts,
        )

    @property
    def brokers(self):
        """The dict of known brokers for this cluster"""
        return self._brokers

    @property
    def topics(self):
        """The dict of known topics for this cluster"""
        return self._topics

    @property
    def handler(self):
        """The concurrency handler for network requests"""
        return self._handler

    def _get_metadata(self):
        """Get fresh cluster metadata from a broker."""
        # Works either on existing brokers or seed_hosts list
        brokers = [b for b in self.brokers.values() if b.connected]
        if brokers:
            for broker in brokers:
                response = broker.request_metadata()
                if response is not None:
                    return response
        else:  # try seed hosts
            brokers = self._seed_hosts.split(',')
            for broker_str in brokers:
                try:
                    h, p = broker_str.split(':')
                    broker = Broker(-1, h, p, self._handler,
                                    self._socket_timeout_ms,
                                    self._offsets_channel_socket_timeout_ms,
                                    buffer_size=self._socket_receive_buffer_bytes)
                    response = broker.request_metadata()
                    if response is not None:
                        return response
                except:
                    logger.exception('Unable to connect to broker %s',
                                     broker_str)
        # Couldn't connect anywhere. Raise an error.
        raise Exception('Unable to connect to a broker to fetch metadata.')

    def _update_brokers(self, broker_metadata):
        """Update brokers with fresh metadata.

        :param broker_metadata: Metadata for all brokers.
        :type broker_metadata: Dict of `{name: metadata}` where `metadata` is
            :class:`pykafka.protocol.BrokerMetadata` and `name` is str.
        """
        # FIXME: A cluster with no topics returns no brokers in metadata
        # Remove old brokers
        removed = set(self._brokers.keys()) - set(broker_metadata.keys())
        for id_ in removed:
            logger.info('Removing broker %s', self._brokers[id_])
            self._brokers.pop(id_)
        # Add/update current brokers
        for id_, meta in broker_metadata.iteritems():
            if id_ not in self._brokers:
                logger.info('Discovered broker id_: %s:%s', id_, meta.host, meta.port)
                self._brokers[id_] = Broker.from_metadata(
                    meta, self._handler, self._socket_timeout_ms,
                    self._offsets_channel_socket_timeout_ms,
                    buffer_size=self._socket_receive_buffer_bytes
                )
            else:
                broker = self._brokers[id_]
                if meta.host == broker.host and meta.port == broker.port:
                    continue  # no changes
                # TODO: Can brokers update? Seems like a problem if so.
                #       Figure out and implement update/disconnect/reconnect if
                #       needed.
                raise Exception('Broker host/port change detected! %s', broker)

    def _update_topics(self, metadata):
        """Update topics with fresh metadata.

        :param metadata: Metadata for all topics.
        :type metadata: Dict of `{name, metadata}` where `metadata` is
            :class:`pykafka.protocol.TopicMetadata` and `name` is str.
        """
        # Remove old topics
        removed = set(self._topics.keys()) - set(metadata.keys())
        for name in removed:
            logger.info('Removing topic %s', self._topics[name])
            self._topics.pop(name)
        # Add/update partition information
        for name, meta in metadata.iteritems():
            if not self._should_exclude_topic(name):
                if name not in self._topics:
                    self._topics[name] = Topic(self, meta)
                    logger.info('Discovered topic %s', self._topics[name])
                else:
                    self._topics[name].update(meta)

    def _should_exclude_topic(self, topic_name):
        """Should this topic be excluded from the list shown to the client?"""
        if not self._exclude_internal_topics:
            return False
        return topic_name.startswith("__")

    def get_offset_manager(self, consumer_group):
        """Get the broker designated as the offset manager for this consumer group.

        Based on Step 1 at https://cwiki.apache.org/confluence/display/KAFKA/Committing+and+fetching+consumer+offsets+in+Kafka

        :param consumer_group: The name of the consumer group for which to
            find the offset manager.
        :type consumer_group: str
        """
        # arbitrarily choose a broker, since this request can go to any
        broker = self.brokers[random.choice(self.brokers.keys())]
        MAX_RETRIES = 3

        for i in xrange(MAX_RETRIES):
            if i > 0:
                logger.debug("Retrying")
            time.sleep(i)

            req = ConsumerMetadataRequest(consumer_group)
            future = broker.handler.request(req)
            try:
                res = future.get(ConsumerMetadataResponse)
            except ConsumerCoordinatorNotAvailable:
                logger.debug('Error discovering offset manager.')
                if i == MAX_RETRIES - 1:
                    raise
            else:
                coordinator = self.brokers.get(res.coordinator_id, None)
                if coordinator is None:
                    raise Exception('Coordinator broker with id {} not found'.format(res.coordinator_id))
                return coordinator

    def update(self):
        """Update known brokers and topics."""
        metadata = self._get_metadata()
        self._update_brokers(metadata.brokers)
        self._update_topics(metadata.topics)
<commit_msg>string formatting<commit_after>from __future__ import division
"""
Author: Keith Bourgoin
"""
__license__ = """
Copyright 2015 Parse.ly, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
__all__ = ["Cluster"]
import logging
import random
import time
import weakref

from .broker import Broker
from .exceptions import (ConsumerCoordinatorNotAvailable,
                         UnknownTopicOrPartition)
from .protocol import ConsumerMetadataRequest, ConsumerMetadataResponse
from .topic import Topic

logger = logging.getLogger(__name__)

class TopicDict(dict):
    """Dictionary which will attempt to auto-create unknown topics."""

    def __init__(self, cluster, *args, **kwargs):
        super(TopicDict, self).__init__(*args, **kwargs)
        self._cluster = weakref.proxy(cluster)

    def __missing__(self, key):
        logger.info('Topic %s not found. Attempting to auto-create.', key)
        if self._create_topic(key):
            return self[key]
        else:
            raise UnknownTopicOrPartition('Unknown topic: {}'.format(key))

    def _create_topic(self, topic_name):
        """Auto-create a topic.

        Not exposed in the cluster or broker because this is *only*
        auto-creation.  When there's a real API for creating topics,
        with settings and everything, we'll implement that. To expose just
        this now would be disingenuous, since it's features would be hobbled.
        """
        # Auto-creating will take a moment, so we try 5 times.
        for i in xrange(5):
            # Auto-creating is as simple as issuing a metadata request
            # solely for that topic.  The update is just to be sure
            # our `Cluster` knows about it.
            self._cluster.brokers[0].request_metadata(topics=[topic_name])
            self._cluster.update()
            if topic_name in self:
                logger.info('Topic %s successfully auto-created.', topic_name)
                return True
            time.sleep(0.1)

class Cluster(object):
    """
    A Cluster is a high-level abstraction of the collection of brokers and
    topics that makes up a real kafka cluster.
    """
    def __init__(self,
                 hosts,
                 handler,
                 socket_timeout_ms=30 * 1000,
                 offsets_channel_socket_timeout_ms=10 * 1000,
                 socket_receive_buffer_bytes=64 * 1024,
                 exclude_internal_topics=True):
        """Create a new Cluster instance.

        :param hosts: Comma-separated list of kafka hosts to used to connect.
        :type hosts: str
        :param handler: The concurrency handler for network requests.
        :type handler: :class:`pykafka.handlers.Handler`
        :param socket_timeout_ms: The socket timeout (in milliseconds) for
            network requests
        :type socket_timeout_ms: int
        :param offsets_channel_socket_timeout_ms: The socket timeout (in
            milliseconds) when reading responses for offset commit and
            offset fetch requests.
        :type offsets_channel_socket_timeout_ms: int
        :param socket_receive_buffer_bytes: The size (in bytes) of the socket
            receive buffer for network requests.
        :type socket_receive_buffer_bytes: int
        :param exclude_internal_topics: Whether messages from internal topics
            (specifically, the offsets topic) should be exposed to consumers.
        :type exclude_internal_topics: bool
        """
        self._seed_hosts = hosts
        self._socket_timeout_ms = socket_timeout_ms
        self._offsets_channel_socket_timeout_ms = offsets_channel_socket_timeout_ms
        self._handler = handler
        self._brokers = {}
        self._topics = TopicDict(self)
        self._socket_receive_buffer_bytes = socket_receive_buffer_bytes
        self._exclude_internal_topics = exclude_internal_topics
        self.update()

    def __repr__(self):
        return "<{}.{} at {} (hosts={})>".format(
            self.__class__.__module__,
            self.__class__.__name__,
            hex(id(self)),
            self._seed_hosts,
        )

    @property
    def brokers(self):
        """The dict of known brokers for this cluster"""
        return self._brokers

    @property
    def topics(self):
        """The dict of known topics for this cluster"""
        return self._topics

    @property
    def handler(self):
        """The concurrency handler for network requests"""
        return self._handler

    def _get_metadata(self):
        """Get fresh cluster metadata from a broker."""
        # Works either on existing brokers or seed_hosts list
        brokers = [b for b in self.brokers.values() if b.connected]
        if brokers:
            for broker in brokers:
                response = broker.request_metadata()
                if response is not None:
                    return response
        else:  # try seed hosts
            brokers = self._seed_hosts.split(',')
            for broker_str in brokers:
                try:
                    h, p = broker_str.split(':')
                    broker = Broker(-1, h, p, self._handler,
                                    self._socket_timeout_ms,
                                    self._offsets_channel_socket_timeout_ms,
                                    buffer_size=self._socket_receive_buffer_bytes)
                    response = broker.request_metadata()
                    if response is not None:
                        return response
                except:
                    logger.exception('Unable to connect to broker %s',
                                     broker_str)
        # Couldn't connect anywhere. Raise an error.
        raise Exception('Unable to connect to a broker to fetch metadata.')

    def _update_brokers(self, broker_metadata):
        """Update brokers with fresh metadata.

        :param broker_metadata: Metadata for all brokers.
        :type broker_metadata: Dict of `{name: metadata}` where `metadata` is
            :class:`pykafka.protocol.BrokerMetadata` and `name` is str.
        """
        # FIXME: A cluster with no topics returns no brokers in metadata
        # Remove old brokers
        removed = set(self._brokers.keys()) - set(broker_metadata.keys())
        for id_ in removed:
            logger.info('Removing broker %s', self._brokers[id_])
            self._brokers.pop(id_)
        # Add/update current brokers
        for id_, meta in broker_metadata.iteritems():
            if id_ not in self._brokers:
                logger.info('Discovered broker id %s: %s:%s', id_, meta.host, meta.port)
                self._brokers[id_] = Broker.from_metadata(
                    meta, self._handler, self._socket_timeout_ms,
                    self._offsets_channel_socket_timeout_ms,
                    buffer_size=self._socket_receive_buffer_bytes
                )
            else:
                broker = self._brokers[id_]
                if meta.host == broker.host and meta.port == broker.port:
                    continue  # no changes
                # TODO: Can brokers update? Seems like a problem if so.
                #       Figure out and implement update/disconnect/reconnect if
                #       needed.
                raise Exception('Broker host/port change detected! %s', broker)

    def _update_topics(self, metadata):
        """Update topics with fresh metadata.

        :param metadata: Metadata for all topics.
        :type metadata: Dict of `{name, metadata}` where `metadata` is
            :class:`pykafka.protocol.TopicMetadata` and `name` is str.
        """
        # Remove old topics
        removed = set(self._topics.keys()) - set(metadata.keys())
        for name in removed:
            logger.info('Removing topic %s', self._topics[name])
            self._topics.pop(name)
        # Add/update partition information
        for name, meta in metadata.iteritems():
            if not self._should_exclude_topic(name):
                if name not in self._topics:
                    self._topics[name] = Topic(self, meta)
                    logger.info('Discovered topic %s', self._topics[name])
                else:
                    self._topics[name].update(meta)

    def _should_exclude_topic(self, topic_name):
        """Should this topic be excluded from the list shown to the client?"""
        if not self._exclude_internal_topics:
            return False
        return topic_name.startswith("__")

    def get_offset_manager(self, consumer_group):
        """Get the broker designated as the offset manager for this consumer group.

        Based on Step 1 at https://cwiki.apache.org/confluence/display/KAFKA/Committing+and+fetching+consumer+offsets+in+Kafka

        :param consumer_group: The name of the consumer group for which to
            find the offset manager.
        :type consumer_group: str
        """
        # arbitrarily choose a broker, since this request can go to any
        broker = self.brokers[random.choice(self.brokers.keys())]
        MAX_RETRIES = 3

        for i in xrange(MAX_RETRIES):
            if i > 0:
                logger.debug("Retrying")
            time.sleep(i)

            req = ConsumerMetadataRequest(consumer_group)
            future = broker.handler.request(req)
            try:
                res = future.get(ConsumerMetadataResponse)
            except ConsumerCoordinatorNotAvailable:
                logger.debug('Error discovering offset manager.')
                if i == MAX_RETRIES - 1:
                    raise
            else:
                coordinator = self.brokers.get(res.coordinator_id, None)
                if coordinator is None:
                    raise Exception('Coordinator broker with id {} not found'.format(res.coordinator_id))
                return coordinator

    def update(self):
        """Update known brokers and topics."""
        metadata = self._get_metadata()
        self._update_brokers(metadata.brokers)
        self._update_topics(metadata.topics)
<|endoftext|>