<commit_before>import sys

# the code below is shamelessly ripped (and slightly altered)
# from the flickzeug package

# Copyright (c) 2009 by the Flickzeug Team, see AUTHORS for more details.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#
#    * Redistributions in binary form must reproduce the above
#      copyright notice, this list of conditions and the following
#      disclaimer in the documentation and/or other materials provided
#      with the distribution.
#
#    * The names of the contributors may not be used to endorse or
#      promote products derived from this software without specific
#      prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""
    flickzeug.debug.tbtools
    ~~~~~~~~~~~~~~~~~~~~~~~

    This module provides various traceback related utility functions.

    :copyright: (c) 2009 by the Werkzeug Team, see AUTHORS for more details.
    :license: BSD.
"""
import re
import os
import inspect
import itertools
import traceback
import codecs
from errormator_client.client import PY3

_coding_re = re.compile(r'coding[:=]\s*([-\w.]+)')
_line_re = re.compile(r'^(.*?)$(?m)')
_funcdef_re = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
UTF8_COOKIE = '\xef\xbb\xbf'

system_exceptions = (SystemExit, KeyboardInterrupt)
try:
    system_exceptions += (GeneratorExit,)
except NameError:
    pass

class _Missing(object):

    def __repr__(self):
        return 'no value'

    def __reduce__(self):
        return '_missing'

_missing = _Missing()

def truncate_str(input):
    return input if len(input) < 255 else input[:255] + '...'

def serialize_to_unicode(input, treat_as_class=False):
    if hasattr(input, 'iterkeys'):
        dict_method = input.iterkeys
    elif hasattr(input, 'keys'):
        dict_method = input.keys
    else:
        dict_method = None
    if dict_method is not None:
        dict_keys = itertools.takewhile(lambda x:x[0] < 128, enumerate(dict_method()))
        return dict([(k, truncate_str(repr(input[k]))) for i, k in dict_keys])
    elif isinstance(input, (tuple, list, set, frozenset)):
        return [truncate_str(repr(v)) for i, v in
                itertools.takewhile(lambda x:x[0] < 128, enumerate(input))]
    elif isinstance(input, basestring):
        return truncate_str(input)
    else:
        to_return = repr(input)
        return truncate_str(to_return)

def shorten_filename(frame):
    filename = frame.filename
    if frame.module:
        try:
            # shorten filenames
            package = frame.module.split('.', 1)[0]
            base_filename = sys.modules[package].__file__
            to_trunc = base_filename.rsplit(os.sep, 2)[0]
            filename = filename.split(to_trunc, 1)[-1][1:]
        except Exception as e:
            pass
    return filename

class cached_property(object):
    """A decorator that converts a function into a lazy property.  The
    function wrapped is called the first time to retrieve the result
    and then that calculated result is used the next time you access
    the value::

        class Foo(object):

            @cached_property
            def foo(self):
                # calculate something important here
                return 42

    The class has to have a `__dict__` in order for this property to
    work.

    .. versionchanged:: 0.6
       the `writeable` attribute and parameter was deprecated.  If a
       cached property is writeable or not has to be documented now.
       For performance reasons the implementation does not honor the
       writeable setting and will always make the property writeable.
    """

    # implementation detail: this property is implemented as non-data
    # descriptor.  non-data descriptors are only invoked if there is
    # no entry with the same name in the instance's __dict__.
    # this allows us to completely get rid of the access function call
    # overhead.  If one choses to invoke __get__ by hand the property
    # will still work as expected because the lookup logic is replicated
    # in __get__ for manual invocation.

    def __init__(self, func, name=None, doc=None, writeable=False):
        if writeable:
            from warnings import warn
            warn(DeprecationWarning('the writeable argument to the '
                                    'cached property is a noop since 0.6 '
                                    'because the property is writeable '
                                    'by default for performance reasons'))

        self.__name__ = name or func.__name__
        self.__module__ = func.__module__
        self.__doc__ = doc or func.__doc__
        self.func = func

    def __get__(self, obj, type=None):
        if obj is None:
            return self
        value = obj.__dict__.get(self.__name__, _missing)
        if value is _missing:
            value = self.func(obj)
            obj.__dict__[self.__name__] = value
        return value

def get_current_traceback(ignore_system_exceptions=False,
                          show_hidden_frames=False, skip=0):
    """Get the current exception info as `Traceback` object.  Per default
    calling this method will reraise system exceptions such as generator exit,
    system exit or others.  This behavior can be disabled by passing `False`
    to the function as first parameter.
    """
    exc_type, exc_value, tb = sys.exc_info()
    if ignore_system_exceptions and exc_type in system_exceptions:
        raise
    for x in xrange(skip):
        if tb.tb_next is None:
            break
        tb = tb.tb_next
    tb = Traceback(exc_type, exc_value, tb)
    if not show_hidden_frames:
        tb.filter_hidden_frames()
    return tb

class Line(object):
    """Helper for the source renderer."""
    __slots__ = ('lineno', 'code', 'in_frame', 'current')

    def __init__(self, lineno, code):
        self.lineno = lineno
        self.code = code
        self.in_frame = False
        self.current = False

    def classes(self):
        rv = ['line']
        if self.in_frame:
            rv.append('in-frame')
        if self.current:
            rv.append('current')
        return rv
    classes = property(classes)

class Traceback(object):
    """Wraps a traceback."""

    def __init__(self, exc_type, exc_value, tb):
        self.exc_type = exc_type
        self.exc_value = exc_value
        if not isinstance(exc_type, str):
            exception_type = exc_type.__name__
            if exc_type.__module__ not in ('__builtin__', 'exceptions'):
                exception_type = exc_type.__module__ + '.' + exception_type
        else:
            exception_type = exc_type
        self.exception_type = exception_type

        # we only add frames to the list that are not hidden.  This follows
        # the the magic variables as defined by paste.exceptions.collector
        self.frames = []
        while tb:
            self.frames.append(Frame(exc_type, exc_value, tb))
            tb = tb.tb_next

    def filter_hidden_frames(self):
        """Remove the frames according to the paste spec."""
        new_frames = []
        hidden = False
        for frame in self.frames:
            hide = frame.hide
            if hide in ('before', 'before_and_this'):
                new_frames = []
                hidden = False
                if hide == 'before_and_this':
                    continue
            elif hide in ('reset', 'reset_and_this'):
                hidden = False
                if hide == 'reset_and_this':
                    continue
            elif hide in ('after', 'after_and_this'):
                hidden = True
                if hide == 'after_and_this':
                    continue
            elif hide or hidden:
                continue
            new_frames.append(frame)

        # if the last frame is missing something went terrible wrong :(
        if self.frames[-1] in new_frames:
            self.frames[:] = new_frames

    @property
    def is_syntax_error(self):
        """Is it a syntax error?"""
        return isinstance(self.exc_value, SyntaxError)

    @property
    def exception(self):
        """String representation of the exception."""
        buf = traceback.format_exception_only(self.exc_type, self.exc_value)
        if PY3:
            return ''.join(buf).strip()
        else:
            return ''.join(buf).strip().decode('utf-8', 'replace')

    def log(self, logfile=None):
        """Log the ASCII traceback into a file object."""
        if logfile is None:
            logfile = sys.stderr
        tb = self.plaintext.encode('utf-8', 'replace').rstrip() + '\n'
        logfile.write(tb)

    @cached_property
    def plaintext(self):
        result = ['Traceback (most recent call last):']
        for frame in self.frames:
            result.append('File "%s", line %s, in %s' %
                    (shorten_filename(frame), frame.lineno, frame.function_name,))
            result.append('    %s' % frame.current_line.strip())
        result.append('%s' % self.exception)
        return '\n'.join(result)

    id = property(lambda x: id(x))

    def frameinfo(self):
        """Dict representing frame and variables"""
        result = []
        for frame in self.frames:
            entry = {'file':shorten_filename(frame),
                     'line':frame.lineno,
                     'fn':frame.function_name,
                     'cline':frame.current_line.strip(),
                     'vars':[]}

            # in some situations frame.locals might be not something we expect
            if not isinstance(frame.locals, dict):
                entry['vars'].append({'unknown', 'uninspectable'})
                continue
            for k, v in frame.locals.iteritems():
                try:
                    if k == 'self':
                        entry['vars'].append(('self.' + v.__class__.__name__,
                                              serialize_to_unicode(v)))
                    else:
                        entry['vars'].append((k, serialize_to_unicode(v)))
                except Exception as e:
                    entry['vars'].append((k, '<unserializable>'))
            result.append(entry)
        result.append({'file':'',
                     'line':'',
                     'fn':'',
                     'cline':'%s' % self.exception,
                     'vars':{}}
                      )
        return result

class Frame(object):
    """A single frame in a traceback."""

    def __init__(self, exc_type, exc_value, tb):
        self.lineno = tb.tb_lineno
        self.function_name = tb.tb_frame.f_code.co_name
        self.locals = tb.tb_frame.f_locals
        self.globals = tb.tb_frame.f_globals

        fn = inspect.getsourcefile(tb) or inspect.getfile(tb)
        if fn[-4:] in ('.pyo', '.pyc'):
            fn = fn[:-1]
        # if it's a file on the file system resolve the real filename.
# we actually dont want that or it will break filename shortening
#        if os.path.isfile(fn):
#            fn = os.path.realpath(fn)
        self.filename = fn
        self.module = self.globals.get('__name__')
        self.loader = self.globals.get('__loader__')
        self.code = tb.tb_frame.f_code

        # support for paste's traceback extensions
        self.hide = self.locals.get('__traceback_hide__', False)
        info = self.locals.get('__traceback_info__')
        if info is not None:
            try:
                info = unicode(info)
            except UnicodeError:
                info = str(info).decode('utf-8', 'replace')
        self.info = info

    @cached_property
    def sourcelines(self):
        """The sourcecode of the file as list of unicode strings."""
        # get sourcecode from loader or file
        source = None
        if self.loader is not None:
            try:
                if hasattr(self.loader, 'get_source'):
                    source = self.loader.get_source(self.module)
                elif hasattr(self.loader, 'get_source_by_code'):
                    source = self.loader.get_source_by_code(self.code)
            except:
                # we munch the exception so that we don't cause troubles
                # if the loader is broken.
                pass

        if source is None:
            try:
                f = file(self.filename)
            except IOError:
                return []
            try:
                source = f.read()
            finally:
                f.close()

        # already unicode?  return right away
        if isinstance(source, unicode):
            return source.splitlines()

        # yes. it should be ascii, but we don't want to reject too many
        # characters in the debugger if something breaks
        charset = 'utf-8'
        if source.startswith(UTF8_COOKIE):
            source = source[3:]
        else:
            for idx, match in enumerate(_line_re.finditer(source)):
                match = _line_re.search(match.group())
                if match is not None:
                    charset = match.group(1)
                    break
                if idx > 1:
                    break

        # on broken cookies we fall back to utf-8 too
        try:
            codecs.lookup(charset)
        except LookupError:
            charset = 'utf-8'

        return source.decode(charset, 'replace').splitlines()

    @property
    def current_line(self):
        try:
            return self.sourcelines[self.lineno - 1]
        except IndexError:
            return u''

    id = property(lambda x: id(x))
<commit_msg>get rid of overlapping objects in var map<commit_after>import sys

# the code below is shamelessly ripped (and slightly altered)
# from the flickzeug package

# Copyright (c) 2009 by the Flickzeug Team, see AUTHORS for more details.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#
#    * Redistributions in binary form must reproduce the above
#      copyright notice, this list of conditions and the following
#      disclaimer in the documentation and/or other materials provided
#      with the distribution.
#
#    * The names of the contributors may not be used to endorse or
#      promote products derived from this software without specific
#      prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""
    flickzeug.debug.tbtools
    ~~~~~~~~~~~~~~~~~~~~~~~

    This module provides various traceback related utility functions.

    :copyright: (c) 2009 by the Werkzeug Team, see AUTHORS for more details.
    :license: BSD.
"""
import re
import os
import inspect
import itertools
import traceback
import codecs
from errormator_client.client import PY3

_coding_re = re.compile(r'coding[:=]\s*([-\w.]+)')
_line_re = re.compile(r'^(.*?)$(?m)')
_funcdef_re = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
UTF8_COOKIE = '\xef\xbb\xbf'

system_exceptions = (SystemExit, KeyboardInterrupt)
try:
    system_exceptions += (GeneratorExit,)
except NameError:
    pass

class _Missing(object):

    def __repr__(self):
        return 'no value'

    def __reduce__(self):
        return '_missing'

_missing = _Missing()

def truncate_str(input):
    return input if len(input) < 255 else input[:255] + '...'

def serialize_to_unicode(input, treat_as_class=False):
    if hasattr(input, 'iterkeys'):
        dict_method = input.iterkeys
    elif hasattr(input, 'keys'):
        dict_method = input.keys
    else:
        dict_method = None
    if dict_method is not None:
        dict_keys = itertools.takewhile(lambda x:x[0] < 128, enumerate(dict_method()))
        return dict([(k, truncate_str(repr(input[k]))) for i, k in dict_keys])
    elif isinstance(input, (tuple, list, set, frozenset)):
        return [truncate_str(repr(v)) for i, v in
                itertools.takewhile(lambda x:x[0] < 128, enumerate(input))]
    elif isinstance(input, basestring):
        return truncate_str(input)
    else:
        to_return = repr(input)
        return truncate_str(to_return)

def shorten_filename(frame):
    filename = frame.filename
    if frame.module:
        try:
            # shorten filenames
            package = frame.module.split('.', 1)[0]
            base_filename = sys.modules[package].__file__
            to_trunc = base_filename.rsplit(os.sep, 2)[0]
            filename = filename.split(to_trunc, 1)[-1][1:]
        except Exception as e:
            pass
    return filename

class cached_property(object):
    """A decorator that converts a function into a lazy property.  The
    function wrapped is called the first time to retrieve the result
    and then that calculated result is used the next time you access
    the value::

        class Foo(object):

            @cached_property
            def foo(self):
                # calculate something important here
                return 42

    The class has to have a `__dict__` in order for this property to
    work.

    .. versionchanged:: 0.6
       the `writeable` attribute and parameter was deprecated.  If a
       cached property is writeable or not has to be documented now.
       For performance reasons the implementation does not honor the
       writeable setting and will always make the property writeable.
    """

    # implementation detail: this property is implemented as non-data
    # descriptor.  non-data descriptors are only invoked if there is
    # no entry with the same name in the instance's __dict__.
    # this allows us to completely get rid of the access function call
    # overhead.  If one choses to invoke __get__ by hand the property
    # will still work as expected because the lookup logic is replicated
    # in __get__ for manual invocation.

    def __init__(self, func, name=None, doc=None, writeable=False):
        if writeable:
            from warnings import warn
            warn(DeprecationWarning('the writeable argument to the '
                                    'cached property is a noop since 0.6 '
                                    'because the property is writeable '
                                    'by default for performance reasons'))

        self.__name__ = name or func.__name__
        self.__module__ = func.__module__
        self.__doc__ = doc or func.__doc__
        self.func = func

    def __get__(self, obj, type=None):
        if obj is None:
            return self
        value = obj.__dict__.get(self.__name__, _missing)
        if value is _missing:
            value = self.func(obj)
            obj.__dict__[self.__name__] = value
        return value

def get_current_traceback(ignore_system_exceptions=False,
                          show_hidden_frames=False, skip=0):
    """Get the current exception info as `Traceback` object.  Per default
    calling this method will reraise system exceptions such as generator exit,
    system exit or others.  This behavior can be disabled by passing `False`
    to the function as first parameter.
    """
    exc_type, exc_value, tb = sys.exc_info()
    if ignore_system_exceptions and exc_type in system_exceptions:
        raise
    for x in xrange(skip):
        if tb.tb_next is None:
            break
        tb = tb.tb_next
    tb = Traceback(exc_type, exc_value, tb)
    if not show_hidden_frames:
        tb.filter_hidden_frames()
    return tb

class Line(object):
    """Helper for the source renderer."""
    __slots__ = ('lineno', 'code', 'in_frame', 'current')

    def __init__(self, lineno, code):
        self.lineno = lineno
        self.code = code
        self.in_frame = False
        self.current = False

    def classes(self):
        rv = ['line']
        if self.in_frame:
            rv.append('in-frame')
        if self.current:
            rv.append('current')
        return rv
    classes = property(classes)

class Traceback(object):
    """Wraps a traceback."""

    def __init__(self, exc_type, exc_value, tb):
        self.exc_type = exc_type
        self.exc_value = exc_value
        if not isinstance(exc_type, str):
            exception_type = exc_type.__name__
            if exc_type.__module__ not in ('__builtin__', 'exceptions'):
                exception_type = exc_type.__module__ + '.' + exception_type
        else:
            exception_type = exc_type
        self.exception_type = exception_type

        # we only add frames to the list that are not hidden.  This follows
        # the the magic variables as defined by paste.exceptions.collector
        self.frames = []
        while tb:
            self.frames.append(Frame(exc_type, exc_value, tb))
            tb = tb.tb_next

    def filter_hidden_frames(self):
        """Remove the frames according to the paste spec."""
        new_frames = []
        hidden = False
        for frame in self.frames:
            hide = frame.hide
            if hide in ('before', 'before_and_this'):
                new_frames = []
                hidden = False
                if hide == 'before_and_this':
                    continue
            elif hide in ('reset', 'reset_and_this'):
                hidden = False
                if hide == 'reset_and_this':
                    continue
            elif hide in ('after', 'after_and_this'):
                hidden = True
                if hide == 'after_and_this':
                    continue
            elif hide or hidden:
                continue
            new_frames.append(frame)

        # if the last frame is missing something went terrible wrong :(
        if self.frames[-1] in new_frames:
            self.frames[:] = new_frames

    @property
    def is_syntax_error(self):
        """Is it a syntax error?"""
        return isinstance(self.exc_value, SyntaxError)

    @property
    def exception(self):
        """String representation of the exception."""
        buf = traceback.format_exception_only(self.exc_type, self.exc_value)
        if PY3:
            return ''.join(buf).strip()
        else:
            return ''.join(buf).strip().decode('utf-8', 'replace')

    def log(self, logfile=None):
        """Log the ASCII traceback into a file object."""
        if logfile is None:
            logfile = sys.stderr
        tb = self.plaintext.encode('utf-8', 'replace').rstrip() + '\n'
        logfile.write(tb)

    @cached_property
    def plaintext(self):
        result = ['Traceback (most recent call last):']
        for frame in self.frames:
            result.append('File "%s", line %s, in %s' %
                    (shorten_filename(frame), frame.lineno, frame.function_name,))
            result.append('    %s' % frame.current_line.strip())
        result.append('%s' % self.exception)
        return '\n'.join(result)

    id = property(lambda x: id(x))

    def frameinfo(self):
        """Dict representing frame and variables"""
        result = []
        id_list = []
        for frame in self.frames:
            entry = {'file':shorten_filename(frame),
                     'line':frame.lineno,
                     'fn':frame.function_name,
                     'cline':frame.current_line.strip(),
                     'vars':[]}

            # in some situations frame.locals might be not something we expect
            if not isinstance(frame.locals, dict):
                entry['vars'].append({'unknown', 'uninspectable'})
                continue
            for k, v in frame.locals.iteritems():
                if id(v) not in id_list:
                    id_list.append(id(v))
                else:
                    print 'EXISTS'
                    continue
                try:
                    if k == 'self':
                        entry['vars'].append(('self.' + v.__class__.__name__,
                                              serialize_to_unicode(v)))
                    else:
                        entry['vars'].append((k, serialize_to_unicode(v)))
                except Exception as e:
                    entry['vars'].append((k, '<unserializable>'))
            result.append(entry)
        result.append({'file':'',
                     'line':'',
                     'fn':'',
                     'cline':'%s' % self.exception,
                     'vars':{}}
                      )
        return result

class Frame(object):
    """A single frame in a traceback."""

    def __init__(self, exc_type, exc_value, tb):
        self.lineno = tb.tb_lineno
        self.function_name = tb.tb_frame.f_code.co_name
        self.locals = tb.tb_frame.f_locals
        self.globals = tb.tb_frame.f_globals

        fn = inspect.getsourcefile(tb) or inspect.getfile(tb)
        if fn[-4:] in ('.pyo', '.pyc'):
            fn = fn[:-1]
        # if it's a file on the file system resolve the real filename.
# we actually dont want that or it will break filename shortening
#        if os.path.isfile(fn):
#            fn = os.path.realpath(fn)
        self.filename = fn
        self.module = self.globals.get('__name__')
        self.loader = self.globals.get('__loader__')
        self.code = tb.tb_frame.f_code

        # support for paste's traceback extensions
        self.hide = self.locals.get('__traceback_hide__', False)
        info = self.locals.get('__traceback_info__')
        if info is not None:
            try:
                info = unicode(info)
            except UnicodeError:
                info = str(info).decode('utf-8', 'replace')
        self.info = info

    @cached_property
    def sourcelines(self):
        """The sourcecode of the file as list of unicode strings."""
        # get sourcecode from loader or file
        source = None
        if self.loader is not None:
            try:
                if hasattr(self.loader, 'get_source'):
                    source = self.loader.get_source(self.module)
                elif hasattr(self.loader, 'get_source_by_code'):
                    source = self.loader.get_source_by_code(self.code)
            except:
                # we munch the exception so that we don't cause troubles
                # if the loader is broken.
                pass

        if source is None:
            try:
                f = file(self.filename)
            except IOError:
                return []
            try:
                source = f.read()
            finally:
                f.close()

        # already unicode?  return right away
        if isinstance(source, unicode):
            return source.splitlines()

        # yes. it should be ascii, but we don't want to reject too many
        # characters in the debugger if something breaks
        charset = 'utf-8'
        if source.startswith(UTF8_COOKIE):
            source = source[3:]
        else:
            for idx, match in enumerate(_line_re.finditer(source)):
                match = _line_re.search(match.group())
                if match is not None:
                    charset = match.group(1)
                    break
                if idx > 1:
                    break

        # on broken cookies we fall back to utf-8 too
        try:
            codecs.lookup(charset)
        except LookupError:
            charset = 'utf-8'

        return source.decode(charset, 'replace').splitlines()

    @property
    def current_line(self):
        try:
            return self.sourcelines[self.lineno - 1]
        except IndexError:
            return u''

    id = property(lambda x: id(x))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

from __future__ import division, print_function

import pytest as pt
import numpy as np
import mptom.mparray as mp
import mptom.factory as factory
import mptom._qmtools as qm

from numpy.testing import assert_array_almost_equal

# List of test parameters (sites, local_dim) that still allow for treatment of
# full representation
FULL_TEST_PARAMETERS = [(6, 2), (4, 3)]

# List of test parameters (sites, local_dim, bond_dim) for efficient matrix-
# product representation tests
MP_TEST_PARAMETERS = [(6, 2, 4), (4, 3, 5)]

###############################################################################
#                         Basic creation & operations                         #
###############################################################################
@pt.mark.parametrize('nr_sites, local_dim', FULL_TEST_PARAMETERS)
def test_from_full(nr_sites, local_dim):
    psi = factory.random_vec(nr_sites, local_dim)
    mps = mp.MPArray.from_array(psi, 1)
    assert_array_almost_equal(psi, mps.to_array())

    op = factory.random_op(nr_sites, local_dim)
    mpo = mp.MPArray.from_array(op, 2)
    assert_array_almost_equal(op, mpo.to_array())

@pt.mark.parametrize('nr_sites, local_dim', FULL_TEST_PARAMETERS)
def test_conjugations(nr_sites, local_dim):
    op = factory.random_op(nr_sites, local_dim)
    mpo = mp.MPArray.from_array(op, 2)
    assert_array_almost_equal(np.conj(op), mpo.C().to_array())

@pt.mark.parametrize('nr_sites, local_dim', FULL_TEST_PARAMETERS)
def test_transposition(nr_sites, local_dim):
    op = factory.random_op(nr_sites, local_dim)
    mpo = mp.MPArray.from_array(qm.global_to_local(op, nr_sites), 2)

    opT = op.reshape((local_dim**nr_sites,) * 2).T \
        .reshape((local_dim,) * 2 * nr_sites)
    res = qm.local_to_global(mpo.T().to_array(), nr_sites)
    assert_array_almost_equal(opT, res)

###############################################################################
#                            Algebraic operations                             #
###############################################################################
@pt.mark.parametrize('nr_sites, local_dim, bond_dim', MP_TEST_PARAMETERS)
def test_dot(nr_sites, local_dim, bond_dim):
    mpo1 = factory.random_mpa(nr_sites, (local_dim, local_dim), bond_dim)
    op1 = qm.local_to_global(mpo1.to_array(), nr_sites)
    mpo2 = factory.random_mpa(nr_sites, (local_dim, local_dim), bond_dim)
    op2 = qm.local_to_global(mpo2.to_array(), nr_sites)

    # Dotproduct of all 1st physical with 0th physical legs = np.dot
    dot_np = np.tensordot(op1.reshape((local_dim**nr_sites, ) * 2),
                          op2.reshape((local_dim**nr_sites, ) * 2),
                          axes=([1], [0]))
    dot_np = dot_np.reshape(op1.shape)
    dot_mp = mp.dot(mpo1, mpo2, axes=(1, 0)).to_array()
    dot_mp = qm.local_to_global(dot_mp, nr_sites)
    assert_array_almost_equal(dot_np, dot_mp)
    # this should also be the default axes
    dot_mp = mp.dot(mpo1, mpo2).to_array()
    dot_mp = qm.local_to_global(dot_mp, nr_sites)
    assert_array_almost_equal(dot_np, dot_mp)

    # Dotproduct of all 0th physical with 1st physical legs = np.dot
    dot_np = np.tensordot(op1.reshape((local_dim**nr_sites, ) * 2),
                          op2.reshape((local_dim**nr_sites, ) * 2),
                          axes=([0], [1]))
    dot_np = dot_np.reshape(op1.shape)
    dot_mp = mp.dot(mpo1, mpo2, axes=(0, 1)).to_array()
    dot_mp = qm.local_to_global(dot_mp, nr_sites)
    assert_array_almost_equal(dot_np, dot_mp)
    # this should also be the default axes
    dot_mp = mp.dot(mpo1, mpo2, axes=(-2, -1)).to_array()
    dot_mp = qm.local_to_global(dot_mp, nr_sites)
    assert_array_almost_equal(dot_np, dot_mp)

@pt.mark.parametrize('nr_sites, local_dim, bond_dim', MP_TEST_PARAMETERS)
def test_add_and_subtr(nr_sites, local_dim, bond_dim):
    mpo1 = factory.random_mpa(nr_sites, (local_dim, local_dim), bond_dim)
    op1 = qm.local_to_global(mpo1.to_array(), nr_sites)
    mpo2 = factory.random_mpa(nr_sites, (local_dim, local_dim), bond_dim)
    op2 = qm.local_to_global(mpo2.to_array(), nr_sites)

    sum_mp = qm.local_to_global((mpo1 + mpo2).to_array(), nr_sites)
    assert_array_almost_equal(op1 + op2, sum_mp)

    sum_mp = qm.local_to_global((mpo1 - mpo2).to_array(), nr_sites)
    assert_array_almost_equal(op1 - op2, sum_mp)

@pt.mark.parametrize('nr_sites, local_dim, bond_dim', MP_TEST_PARAMETERS)
def test_mult_mpo_scalar(nr_sites, local_dim, bond_dim):
    mpo = factory.random_mpa(nr_sites, (local_dim, local_dim), bond_dim)
    op = qm.local_to_global(mpo.to_array(), nr_sites)

    res = qm.local_to_global((2 * mpo).to_array(), nr_sites)
    assert_array_almost_equal(2 * op, res)

###############################################################################
#                         Normalization & Compression                         #
###############################################################################
def assert_lcannonical(ltens, msg=''):
    ltens = ltens.reshape((np.prod(ltens.shape[:-1]), ltens.shape[-1]))
    prod = ltens.conj().T.dot(ltens)
    assert_array_almost_equal(prod, np.identity(prod.shape[0]),
                                         err_msg=msg)

def assert_rcannonical(ltens, msg=''):
    ltens = ltens.reshape((ltens.shape[0], np.prod(ltens.shape[1:])))
    prod = ltens.dot(ltens.conj().T)
    assert_array_almost_equal(prod, np.identity(prod.shape[0]),
                                         err_msg=msg)

def assert_correct_normalzation(mpo, lnormal_target, rnormal_target):
    lnormal, rnormal = mpo.normal_form

    assert lnormal == lnormal_target, \
        "Wrong lnormal={} != {}".format(lnormal, lnormal_target)
    assert rnormal == rnormal_target, \
        "Wrong rnormal={} != {}".format(rnormal, rnormal_target)

    for n in xrange(lnormal):
        assert_lcannonical(mpo[n], msg="Failure left cannonical (n={}/{})"
                           .format(n, lnormal_target))
    for n in xrange(rnormal, len(mpo)):
        assert_rcannonical(mpo[n], msg="Failure right cannonical (n={}/{})"
                           .format(n, rnormal_target))

@pt.mark.parametrize('nr_sites, local_dim', FULL_TEST_PARAMETERS)
def test_from_full_normalization(nr_sites, local_dim):
    op = factory.random_op(nr_sites, local_dim)
    mpo = mp.MPArray.from_array(op, 2)
    assert_correct_normalzation(mpo, nr_sites - 1, nr_sites)

@pt.mark.parametrize('nr_sites, local_dim, bond_dim', MP_TEST_PARAMETERS)
def test_incremental_normalization(nr_sites, local_dim, bond_dim):
    mpo = factory.random_mpa(nr_sites, (local_dim, local_dim), bond_dim)
    op = qm.local_to_global(mpo.to_array(), nr_sites)
    assert_correct_normalzation(mpo, 0, nr_sites)
    assert_array_almost_equal(op, qm.local_to_global(mpo.to_array(), nr_sites))

    for site in xrange(1, nr_sites):
        mpo.normalize(left=site)
        assert_correct_normalzation(mpo, site, nr_sites)
        assert_array_almost_equal(op,
                                  qm.local_to_global(mpo.to_array(), nr_sites))

    for site in xrange(nr_sites - 1, 0, -1):
        mpo.normalize(right=site)
        assert_correct_normalzation(mpo, site - 1, site)
        assert_array_almost_equal(op,
                                  qm.local_to_global(mpo.to_array(), nr_sites))

@pt.mark.parametrize('nr_sites, local_dim, bond_dim', MP_TEST_PARAMETERS)
def test_jump_normalization(nr_sites, local_dim, bond_dim):
    mpo = factory.random_mpa(nr_sites, (local_dim, local_dim), bond_dim)
    op = qm.local_to_global(mpo.to_array(), nr_sites)
    assert_correct_normalzation(mpo, 0, nr_sites)
    assert_array_almost_equal(op, qm.local_to_global(mpo.to_array(), nr_sites))

    center = nr_sites // 2
    mpo.normalize(left=center - 1, right=center)
    assert_correct_normalzation(mpo, center - 1, center)
    assert_array_almost_equal(op, qm.local_to_global(mpo.to_array(), nr_sites))
<commit_msg>Refactor unit tests<commit_after>#!/usr/bin/env python
# encoding: utf-8

from __future__ import division, print_function

import pytest as pt
import numpy as np
import mptom.mparray as mp
import mptom.factory as factory
import mptom._qmtools as qm

from numpy.testing import assert_array_almost_equal

# List of test parameters (sites, local_dim, bond_dim)
MP_TEST_PARAMETERS = [(6, 2, 4), (4, 3, 5)]

def mpo_to_global(mpo):
    return qm.local_to_global(mpo.to_array(), len(mpo))

###############################################################################
#                         Basic creation & operations                         #
###############################################################################
@pt.mark.parametrize('nr_sites, local_dim, _', MP_TEST_PARAMETERS)
def test_from_full(nr_sites, local_dim, _):
    psi = factory.random_vec(nr_sites, local_dim)
    mps = mp.MPArray.from_array(psi, 1)
    assert_array_almost_equal(psi, mps.to_array())

    op = factory.random_op(nr_sites, local_dim)
    mpo = mp.MPArray.from_array(op, 2)
    assert_array_almost_equal(op, mpo.to_array())

@pt.mark.parametrize('nr_sites, local_dim, _', MP_TEST_PARAMETERS)
def test_conjugations(nr_sites, local_dim, _):
    op = factory.random_op(nr_sites, local_dim)
    mpo = mp.MPArray.from_array(op, 2)
    assert_array_almost_equal(np.conj(op), mpo.C().to_array())

@pt.mark.parametrize('nr_sites, local_dim, _', MP_TEST_PARAMETERS)
def test_transposition(nr_sites, local_dim, _):
    op = factory.random_op(nr_sites, local_dim)
    mpo = mp.MPArray.from_array(qm.global_to_local(op, nr_sites), 2)

    opT = op.reshape((local_dim**nr_sites,) * 2).T \
        .reshape((local_dim,) * 2 * nr_sites)
    assert_array_almost_equal(opT, mpo_to_global(mpo.T()))

###############################################################################
#                            Algebraic operations                             #
###############################################################################
@pt.mark.parametrize('nr_sites, local_dim, bond_dim', MP_TEST_PARAMETERS)
def test_dot(nr_sites, local_dim, bond_dim):
    mpo1 = factory.random_mpa(nr_sites, (local_dim, local_dim), bond_dim)
    op1 = mpo_to_global(mpo1)
    mpo2 = factory.random_mpa(nr_sites, (local_dim, local_dim), bond_dim)
    op2 = mpo_to_global(mpo2)

    # Dotproduct of all 1st physical with 0th physical legs = np.dot
    dot_np = np.tensordot(op1.reshape((local_dim**nr_sites, ) * 2),
                          op2.reshape((local_dim**nr_sites, ) * 2),
                          axes=([1], [0]))
    dot_np = dot_np.reshape(op1.shape)
    dot_mp = mpo_to_global(mp.dot(mpo1, mpo2, axes=(1, 0)))
    assert_array_almost_equal(dot_np, dot_mp)
    # this should also be the default axes
    dot_mp = mpo_to_global(mp.dot(mpo1, mpo2))
    assert_array_almost_equal(dot_np, dot_mp)

    # Dotproduct of all 0th physical with 1st physical legs = np.dot
    dot_np = np.tensordot(op1.reshape((local_dim**nr_sites, ) * 2),
                          op2.reshape((local_dim**nr_sites, ) * 2),
                          axes=([0], [1]))
    dot_np = dot_np.reshape(op1.shape)
    dot_mp = mpo_to_global(mp.dot(mpo1, mpo2, axes=(0, 1)))
    assert_array_almost_equal(dot_np, dot_mp)
    # this should also be the default axes
    dot_mp = mpo_to_global(mp.dot(mpo1, mpo2, axes=(-2, -1)))
    assert_array_almost_equal(dot_np, dot_mp)

@pt.mark.parametrize('nr_sites, local_dim, bond_dim', MP_TEST_PARAMETERS)
def test_add_and_subtr(nr_sites, local_dim, bond_dim):
    mpo1 = factory.random_mpa(nr_sites, (local_dim, local_dim), bond_dim)
    op1 = mpo_to_global(mpo1)
    mpo2 = factory.random_mpa(nr_sites, (local_dim, local_dim), bond_dim)
    op2 = mpo_to_global(mpo2)

    assert_array_almost_equal(op1 + op2, mpo_to_global(mpo1 + mpo2))
    assert_array_almost_equal(op1 - op2, mpo_to_global(mpo1 - mpo2))

@pt.mark.parametrize('nr_sites, local_dim, bond_dim', MP_TEST_PARAMETERS)
def test_mult_mpo_scalar(nr_sites, local_dim, bond_dim):
    mpo = factory.random_mpa(nr_sites, (local_dim, local_dim), bond_dim)
    op = mpo_to_global(mpo)

    assert_array_almost_equal(2 * op, mpo_to_global(2 * mpo))

###############################################################################
#                         Normalization & Compression                         #
###############################################################################
def assert_lcannonical(ltens, msg=''):
    ltens = ltens.reshape((np.prod(ltens.shape[:-1]), ltens.shape[-1]))
    prod = ltens.conj().T.dot(ltens)
    assert_array_almost_equal(prod, np.identity(prod.shape[0]),
                                         err_msg=msg)

def assert_rcannonical(ltens, msg=''):
    ltens = ltens.reshape((ltens.shape[0], np.prod(ltens.shape[1:])))
    prod = ltens.dot(ltens.conj().T)
    assert_array_almost_equal(prod, np.identity(prod.shape[0]),
                                         err_msg=msg)

def assert_correct_normalzation(mpo, lnormal_target, rnormal_target):
    lnormal, rnormal = mpo.normal_form

    assert lnormal == lnormal_target, \
        "Wrong lnormal={} != {}".format(lnormal, lnormal_target)
    assert rnormal == rnormal_target, \
        "Wrong rnormal={} != {}".format(rnormal, rnormal_target)

    for n in xrange(lnormal):
        assert_lcannonical(mpo[n], msg="Failure left cannonical (n={}/{})"
                           .format(n, lnormal_target))
    for n in xrange(rnormal, len(mpo)):
        assert_rcannonical(mpo[n], msg="Failure right cannonical (n={}/{})"
                           .format(n, rnormal_target))

@pt.mark.parametrize('nr_sites, local_dim, _', MP_TEST_PARAMETERS)
def test_from_full_normalization(nr_sites, local_dim, _):
    op = factory.random_op(nr_sites, local_dim)
    mpo = mp.MPArray.from_array(op, 2)
    assert_correct_normalzation(mpo, nr_sites - 1, nr_sites)

# FIXME Add counter to normalization functions
@pt.mark.parametrize('nr_sites, local_dim, bond_dim', MP_TEST_PARAMETERS)
def test_incremental_normalization(nr_sites, local_dim, bond_dim):
    mpo = factory.random_mpa(nr_sites, (local_dim, local_dim), bond_dim)
    op = mpo_to_global(mpo)
    assert_correct_normalzation(mpo, 0, nr_sites)
    assert_array_almost_equal(op, mpo_to_global(mpo))

    for site in xrange(1, nr_sites):
        mpo.normalize(left=site)
        assert_correct_normalzation(mpo, site, nr_sites)
        assert_array_almost_equal(op, mpo_to_global(mpo))

    for site in xrange(nr_sites - 1, 0, -1):
        mpo.normalize(right=site)
        assert_correct_normalzation(mpo, site - 1, site)
        assert_array_almost_equal(op, mpo_to_global(mpo))

# FIXME Add counter to normalization functions
@pt.mark.parametrize('nr_sites, local_dim, bond_dim', MP_TEST_PARAMETERS)
def test_jump_normalization(nr_sites, local_dim, bond_dim):
    mpo = factory.random_mpa(nr_sites, (local_dim, local_dim), bond_dim)
    op = mpo_to_global(mpo)
    assert_correct_normalzation(mpo, 0, nr_sites)
    assert_array_almost_equal(op, mpo_to_global(mpo))

    center = nr_sites // 2
    mpo.normalize(left=center - 1, right=center)
    assert_correct_normalzation(mpo, center - 1, center)
    assert_array_almost_equal(op, mpo_to_global(mpo))

# @pt.mark.parametrize('nr_sites, local_dim, bond_dim', MP_TEST_PARAMETERS)
# def test_(nr_sites, local_dim, bond_dim):
    # mpo = factory.random_mpa(nr_sites, (local_dim, local_dim), bond_dim)
    # zero = factory.zero_mpa(nr_sites, (local_dim, local_dim), bond_dim)
    # mpo_ = mpo + zero

    # assert_array_almost_equal(op, qm.local_to_global(mpo.to_array(), nr_sites))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.db import models
from django.http.response import Http404

class CommonManager(models.Manager):

    def get_by_id(self, id, fields=None):
        """Gets a document by an id.

        :param id: id of the document to retrieve.
        :param fields: a tuple of field names to return.  If None, all fields
            are returned (default).
        """
        criteria = {'id': id}
        return self._get_one(criteria=criteria)

    def get_by_id_or_404(self, id):
        """Gets by a instance instance r raises a 404 is one isn't found."""
        obj = self.get_by_id(id=id)

        if obj:
            return obj

        raise Http404

    def get_by_ids(self, ids):
        """Gets documents by ids.

        :param ids: list of ids of documents to return
        :param fields: a tuple of field names to return.  If None, all fields
            are returned (default).
        """
        return self.filter(id__in=ids)

    def _get_one(self, **criteria):
        """
        Gets a single document based on a specific set of criteria.

        :param criteria: criteria for the query
        :param fields: a tuple of field names to return.  If None, all fields
            are returned (default).
        """
        try:
            return self.get(**criteria)
        except self.model.DoesNotExist:
            return None

    def bulk_create(self, objs, *args, **kwargs):
        """Insert many object at once."""
        self.model.save_prep(instance_or_instances=objs)
        return super(CommonManager, self).bulk_create(*args, **kwargs)

    def delete_by_id(self, id):
        """Deletes a document by id."""
        return self.delete_by_ids(ids=[id])

    def delete_by_ids(self, ids):
        """Delete objects by ids.

        :param ids: list of objects ids to delete.
        :return: True if objects were deleted.  Otherwise, return False if no
                objects were found or the delete was not successful.
        """
        try:
            self.filter(id__in=ids).delete()
            return True
        except self.model.DoesNotExist:
            return False
<commit_msg>Manager updates.<commit_after># -*- coding: utf-8 -*-
from django.db import models
from django.http.response import Http404
from django.template.defaultfilters import slugify
from python_tools.random_utils import random_alphanum_id

class BaseManager(models.Manager):

    def get_or_none(self, **kwargs):
        """Gets a single object based on kwargs or None if one is not found."""
        try:
            return self.get(**kwargs)
        except self.model.DoesNotExist:
            return None

class CommonManager(BaseManager):

    def get_by_id(self, id, fields=None):
        """Gets a document by an id.

        :param id: id of the document to retrieve.
        :param fields: a tuple of field names to return.  If None, all fields
            are returned (default).
        """
        return self.get_or_none(id=id)

    def get_by_id_or_404(self, id):
        """Gets by a instance instance r raises a 404 is one isn't found."""
        obj = self.get_by_id(id=id)

        if obj:
            return obj

        raise Http404

    def get_by_ids(self, ids):
        """Gets documents by ids.

        :param ids: list of ids of documents to return
        :param fields: a tuple of field names to return.  If None, all fields
            are returned (default).
        """
        return self.filter(id__in=ids)

    def bulk_create(self, objs, *args, **kwargs):
        """Insert many object at once."""
        # TODO: This might be making a bad assumption that this is inheriting
        # from AbstractBaseModel
        self.model.save_prep(instance_or_instances=objs)
        return super(CommonManager, self).bulk_create(*args, **kwargs)

    def delete_by_id(self, id):
        """Deletes a document by id."""
        return self.delete_by_ids(ids=[id])

    def delete_by_ids(self, ids):
        """Delete objects by ids.

        :param ids: list of objects ids to delete.
        :return: True if objects were deleted.  Otherwise, return False if no
                objects were found or the delete was not successful.
        """
        try:
            self.filter(id__in=ids).delete()
            return True
        except self.model.DoesNotExist:
            return False

class SlugManager(BaseManager):
    """Manager mixin for slugs."""

    def get_by_slug(self, slug, **kwargs):
        try:
            return self.get(slug=slug, **kwargs)
        except self.model.DoesNotExist:
            return None

    def get_by_slug_or_404(self, slug, **kwargs):
        group = self.get_by_slug(slug, **kwargs)

        if group:
            return group

        raise Http404

    def is_slug_available(self, slug, **kwargs):
        """Checks to see if a slug is available. If the slug is already being used
        this method returns False.  Otherwise, return True.
        """
        try:
            self.get(slug=slug, **kwargs)
            return False
        except self.model.DoesNotExist:
            return True

    def get_next_slug(self, slug, **kwargs):
        """Gets the next available slug.

        :param slug: the slug to slugify
        :param kwargs: additional filter criteria to check for when looking for
            a unique slug.

        Example:

        if the value "my-slug" is already taken, this method will append "-n"
        to the end of the slug until the next available slug is found.

        """
        original_slug = slug = slugify(slug)
        count = 0

        while not self.is_slug_available(slug=slug, **kwargs):
            count += 1
            slug = '{0}-{1}'.format(original_slug, count)

        return slug

class TokenManager(BaseManager):
    """Manager Mixin for tokens."""

    def get_by_token(self, token, **kwargs):
        """Get by token."""
        return self.get_or_none(token=token, **kwargs)

    def get_by_token_or_404(self, token, **kwargs):
        group = self.get_by_token(token, **kwargs)

        if group:
            return group

        raise Http404

    def get_next_token(self, length=25, **kwargs):
        """Gets the next available token.

        :param kwargs: additional filter criteria to check for when looking for
            a unique token.

        """
        while True:
            tokens = [random_alphanum_id(id_len=length) for t in range(5)]
            obj_ids = self.filter(token__in=tokens).values_list('id')
            available = set(tokens).difference(obj_ids)

            if available:
                return available.pop()

class UserManager(models.Manager):
    """Manager Mixin for models that have a user."""

    def get_by_user(self, user, **kwargs):
        return self.get_by_user_id(user_id=user.id, **kwargs)

    def get_by_user_id(self, user_id, **kwargs):
        try:
            return self.filter(user_id=user_id)
        except self.model.DoesNotExist:
            return []
<|endoftext|>


<commit_before>from __future__ import print_function, division, unicode_literals, absolute_import

import base64
import json
import logging
import sys

import sendgrid
from sendgrid.helpers.mail import (
    Mail, From, To, Cc, Bcc, Subject, Substitution,
    CustomArg, SendAt, Content, MimeType, Attachment, FileName,
    FileContent, ReplyTo, Category, IpPoolName,
    TrackingSettings, ClickTracking,
    OpenTracking, OpenTrackingSubstitutionTag,
    Section)
from python_http_client.exceptions import HTTPError

from espwrap.base import MassEmail, batch, MIMETYPE_HTML, MIMETYPE_TEXT
from espwrap.adaptors.sendgrid_common import breakdown_recipients

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)

if sys.version_info > (2,):
    basestring = str

_HTTP_EXC_MSG = 'SendGrid responded with an HTTP-error code.  Email Subject: %s, Status Code: %s, Reason: %s, Body: %s'

class SendGridMassEmail(MassEmail):
    def __init__(self, api_key, *args, **kwargs):
        super(SendGridMassEmail, self).__init__(*args, **kwargs)

        self.client = sendgrid.SendGridAPIClient(api_key)

        self.delimiters = ('-', '-')

    def set_variable_delimiters(self, start='-', end='-'):
        self.delimiters = (start, end)

    def get_variable_delimiters(self, as_dict=False):
        if as_dict:
            return {
                'start': self.delimiters[0],
                'end': self.delimiters[1],
            }
        return self.delimiters

    def add_tags(self, *tags):
        if len(tags) > 10:
            raise Exception('Too many tags, SendGrid limits to 10 per email')

        if len(tags) + len(self.tags) > 10:
            raise Exception('Requested tags would have raised total to above Sendgrid limit of 10')

        return super(SendGridMassEmail, self).add_tags(*tags)

    def message_constructor(self, to_send):
        message = Mail()
        to_emails = []

        # Content (Text)
        if self.body[MIMETYPE_TEXT]:
            content = Content(
                MimeType.text,
                self.body[MIMETYPE_TEXT]
            )
            message.content = content

        # Content (Html)
        if self.body[MIMETYPE_HTML]:
            content = Content(
                MimeType.html,
                self.body[MIMETYPE_HTML]
            )
            message.content = content

        message.subject = Subject(self.subject)

        # Category
        if self.tags:
            for tag in self.tags:
                message.category = Category(tag)

        # Reply
        if self.reply_to_addr:
            message.reply_to = ReplyTo(self.reply_to_addr)

        # Send At
        if self.send_at:
            message.send_at = SendAt(self.send_at)

        # IP Pool
        if self.ip_pool:
            message.ip_pool_name = IpPoolName(self.ip_pool)

        # CC
        if self.cc_list:
            list_cc = [Cc(email) for email in self.cc_list]
            message.cc = list_cc

        # BCC
        if self.bcc_list:
            list_bcc = [Bcc(email) for email in self.cc_list]
            message.bcc = list_bcc

        # Attachment
        if self.attachments:
            for file_name, file_path_or_string in self.attachments.iteritems():
                attachment = Attachment()
                encoded = base64.b64encode(str(file_path_or_string))
                attachment.file_content = FileContent(encoded)
                attachment.file_name = FileName(file_name)
            message.attachment = attachment

        # Webhook Data
        if self.webhook_data:
            for key, val in self.webhook_data.items():
                message.custom_arg = CustomArg(key, val)

        # Metadata
        if self.metadata:
            for key, val in self.metadata.items():
                message.custom_arg = CustomArg(key, val)

        # Template Name
        if self.template_name:
            message.custom_arg = CustomArg('template_name', self.template_name)

        # Tracking
        tracking_settings = TrackingSettings()
        # Opens
        if self.track_opens:
            tracking_settings.open_tracking = OpenTracking(
                self.track_opens,
                OpenTrackingSubstitutionTag("open_tracking")
            )
        # Clicks
        if self.track_clicks:
            tracking_settings.click_tracking = ClickTracking(
                self.track_clicks,
                self.track_clicks
            )
        message.tracking_settings = tracking_settings
        message.from_email = From(email=self.from_addr, name=self.from_name)

        for subgrps in to_send:
            for subgrp in subgrps:
                substitutions = subgrp['merge_vars']
                substitutions = {
                    '{1}{0}{2}'.format(x, *self.delimiters): substitutions[x]
                    for x in substitutions
                }
                email = subgrp['email']
                name = subgrp['name']
                to_emails.append(
                    To(
                        email=email,
                        name=name,
                        substitutions=substitutions,
                        subject=self.generate_subject(email, name))
                )

        message.add_to(to_emails, is_multiple=True)

        # Global Subs
        for key, val in self.global_merge_vars.items():
            new_key = '{1}{0}{2}'.format(key, *self.delimiters)
            formatted_val = val
            if not isinstance(val, basestring):
                formatted_val = str(val)

            message.substitution = Substitution(new_key, formatted_val)

        return message

    def generate_subject(self, email, name):
        """
        Generate email subject for recipient.
        If username contains "." or "+" it is considered an alias so we
        prepend a custom greeting to avoid bounces from providers like Gmail
        :param email: Recipient email
        :param name: Recipient name
        :return subject: sendgrid.helpers.mail.Subject
        """
        try:
            username = email.split('@')[0]
            if '.' in username or '+' in username:
                return Subject('{} [{}]'.format(self.subject.decode('utf-8'), email))
        except Exception as e:
            logger.exception(e)

        return Subject(self.subject)

    def send(self):

        responses=[]

        self.validate()

        grouped_recipients = batch(list(self.recipients), self.partition)

        for grp in grouped_recipients:
            to_send = breakdown_recipients(grp)

            message = self.message_constructor(to_send)

            # message data as dict for examination/validation below
            message_dict = message.get()

            # do not send if custom args >10 KB (sendgrid rule)
            if 'custom_args' in message_dict:
                message_custom_args = len(json.dumps(message_dict['custom_args']))
                custom_args_size = sys.getsizeof(message_custom_args)
                custom_args_kb = 0.001 * float(custom_args_size)
                if custom_args_kb > 10:
                    raise Exception('attempted to send {} KB custom_args, not to exceed 10 KB.'.format(custom_args_kb))

            # do not send if number of recipients >1000 (sendgrid rule)
            num_recips = len(message_dict['personalizations'])
            if num_recips > 1000:
                raise Exception('attempted to send to {} email addresses, not to exceed 1000 addresses.'.format(num_recips))

            """
            send message and append response from this grp to list of returned responses for all grouped_recipients
            """
            try:
                logger.debug(message)
                response = self.client.send(message)
            except HTTPError as e:
                logger.exception(_HTTP_EXC_MSG, self.subject, e.status_code, e.reason, e.body)
            except Exception:
                logger.exception('Unknown exception while sending a SendGridMassEmail.')
            else:
                responses.append(response)

        # return list of all responses for each grp in grouped_recipients
        return responses
<commit_msg>Solve conflicts<commit_after>from __future__ import print_function, division, unicode_literals, absolute_import

import base64
import json
import logging
import sys

import sendgrid
from sendgrid.helpers.mail import (
    Mail, From, To, Cc, Bcc, Subject, Substitution,
    CustomArg, SendAt, Content, MimeType, Attachment, FileName,
    FileContent, ReplyTo, Category, IpPoolName,
    TrackingSettings, ClickTracking,
    OpenTracking, OpenTrackingSubstitutionTag,
    Section)
from python_http_client.exceptions import HTTPError

from espwrap.base import MassEmail, batch, MIMETYPE_HTML, MIMETYPE_TEXT
from espwrap.adaptors.sendgrid_common import breakdown_recipients

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)

if sys.version_info > (2,):
    basestring = str

_HTTP_EXC_MSG = 'SendGrid responded with an HTTP-error code.  Email Subject: %s, Status Code: %s, Reason: %s, Body: %s'

class SendGridMassEmail(MassEmail):
    def __init__(self, api_key, *args, **kwargs):
        super(SendGridMassEmail, self).__init__(*args, **kwargs)

        self.client = sendgrid.SendGridAPIClient(api_key)

        self.delimiters = ('-', '-')

    def set_variable_delimiters(self, start='-', end='-'):
        self.delimiters = (start, end)

    def get_variable_delimiters(self, as_dict=False):
        if as_dict:
            return {
                'start': self.delimiters[0],
                'end': self.delimiters[1],
            }
        return self.delimiters

    def add_tags(self, *tags):
        if len(tags) > 10:
            raise Exception('Too many tags, SendGrid limits to 10 per email')

        if len(tags) + len(self.tags) > 10:
            raise Exception('Requested tags would have raised total to above Sendgrid limit of 10')

        return super(SendGridMassEmail, self).add_tags(*tags)

    def message_constructor(self, to_send):
        message = Mail()
        to_emails = []

        # Content (Text)
        if self.body[MIMETYPE_TEXT]:
            content = Content(
                MimeType.text,
                self.body[MIMETYPE_TEXT]
            )
            message.content = content

        # Content (Html)
        if self.body[MIMETYPE_HTML]:
            content = Content(
                MimeType.html,
                self.body[MIMETYPE_HTML]
            )
            message.content = content

        message.subject = Subject(self.subject)

        # Category
        if self.tags:
            for tag in self.tags:
                message.category = Category(tag)

        # Reply
        if self.reply_to_addr:
            message.reply_to = ReplyTo(self.reply_to_addr)

        # Send At
        if self.send_at:
            message.send_at = SendAt(self.send_at)

        # IP Pool
        if self.ip_pool:
            message.ip_pool_name = IpPoolName(self.ip_pool)

        # CC
        if self.cc_list:
            list_cc = [Cc(email) for email in self.cc_list]
            message.cc = list_cc

        # BCC
        if self.bcc_list:
            list_bcc = [Bcc(email) for email in self.cc_list]
            message.bcc = list_bcc

        # Attachment
        if self.attachments:
            for file_name, file_path_or_string in self.attachments.iteritems():
                attachment = Attachment()
                encoded = base64.b64encode(str(file_path_or_string))
                attachment.file_content = FileContent(encoded)
                attachment.file_name = FileName(file_name)
            message.attachment = attachment

        # Webhook Data
        if self.webhook_data:
            for key, val in self.webhook_data.items():
                message.custom_arg = CustomArg(key, val)

        # Metadata
        if self.metadata:
            for key, val in self.metadata.items():
                message.custom_arg = CustomArg(key, val)

        # Template Name
        if self.template_name:
            message.custom_arg = CustomArg('template_name', self.template_name)

        # Tracking
        tracking_settings = TrackingSettings()
        # Opens
        if self.track_opens:
            tracking_settings.open_tracking = OpenTracking(
                self.track_opens,
                OpenTrackingSubstitutionTag("open_tracking")
            )
        # Clicks
        if self.track_clicks:
            tracking_settings.click_tracking = ClickTracking(
                self.track_clicks,
                self.track_clicks
            )
        message.tracking_settings = tracking_settings
        message.from_email = From(email=self.from_addr, name=self.from_name)

        for subgrps in to_send:
            for subgrp in subgrps:
                substitutions = subgrp['merge_vars']
                substitutions = {
                    '{1}{0}{2}'.format(x, *self.delimiters): substitutions[x]
                    for x in substitutions
                }
                email = subgrp['email']
                name = subgrp['name']
                to_emails.append(
                    To(
                        email=email,
                        name=name,
                        substitutions=substitutions,
                        subject=self.generate_subject(email, name))
                )

        message.add_to(to_emails, is_multiple=True)

        # Global Subs
        for key, val in self.global_merge_vars.items():
            new_key = '{1}{0}{2}'.format(key, *self.delimiters)
            formatted_val = val
            if not isinstance(val, basestring):
                try:
                    formatted_val = str(val)
                except UnicodeEncodeError:
                    formatted_val = val.encode('utf-8')

            message.substitution = Substitution(new_key, formatted_val)

        return message

    def generate_subject(self, email, name):
        """
        Generate email subject for recipient.
        If username contains "." or "+" it is considered an alias so we
        prepend a custom greeting to avoid bounces from providers like Gmail
        :param email: Recipient email
        :param name: Recipient name
        :return subject: sendgrid.helpers.mail.Subject
        """
        try:
            username = email.split('@')[0]
            if '.' in username or '+' in username:
                return Subject('{} [{}]'.format(self.subject.decode('utf-8'), email.decode('utf-8')))
        except Exception as e:
            logger.exception(e)

        return Subject(self.subject)

    def send(self):

        responses=[]

        self.validate()

        grouped_recipients = batch(list(self.recipients), self.partition)

        for grp in grouped_recipients:
            to_send = breakdown_recipients(grp)

            message = self.message_constructor(to_send)

            # message data as dict for examination/validation below
            message_dict = message.get()

            # do not send if custom args >10 KB (sendgrid rule)
            if 'custom_args' in message_dict:
                message_custom_args = len(json.dumps(message_dict['custom_args']))
                custom_args_size = sys.getsizeof(message_custom_args)
                custom_args_kb = 0.001 * float(custom_args_size)
                if custom_args_kb > 10:
                    raise Exception('attempted to send {} KB custom_args, not to exceed 10 KB.'.format(custom_args_kb))

            # do not send if number of recipients >1000 (sendgrid rule)
            num_recips = len(message_dict['personalizations'])
            if num_recips > 1000:
                raise Exception('attempted to send to {} email addresses, not to exceed 1000 addresses.'.format(num_recips))

            """
            send message and append response from this grp to list of returned responses for all grouped_recipients
            """
            try:
                logger.debug(message)
                response = self.client.send(message)
            except HTTPError as e:
                logger.exception(_HTTP_EXC_MSG, self.subject, e.status_code, e.reason, e.body)
            except Exception:
                logger.exception('Unknown exception while sending a SendGridMassEmail.')
            else:
                responses.append(response)

        # return list of all responses for each grp in grouped_recipients
        return responses
<|endoftext|>