<commit_before>from typing import TypeVar, Dict, Generic, Tuple, Callable

from toolz import dicttoolz

from amino import Maybe, may, Just, Nothing
from amino.list import List, Lists
from amino.boolean import Boolean
from amino.tc.base import Implicits, ImplicitsMeta

A = TypeVar('A')
B = TypeVar('B', covariant=True)
C = TypeVar('C')
D = TypeVar('D', covariant=True)

class Map(Generic[A, B], Dict[A, B], Implicits, metaclass=ImplicitsMeta, implicits=True):

    @staticmethod
    def wrap(d: Dict[A, B]) -> 'Map[A, B]':
        return Map(d)

    @may
    def get(self, key):
        return Dict.get(self, key)

    def lift(self, key: str) -> Maybe[B]:
        return Just(Dict.get(self, key)) if key in self else Nothing

    def get_item(self, key):
        return self.get(key) / (lambda a: (key, a))

    def get_all(self, *keys):
        def append(zm, k):
            return zm // (lambda z: self.get(k) / z.cat)
        return List.wrap(keys).fold_left(Just(List()))(append)

    def get_all_map(self, *keys):
        def append(zm, k):
            return zm // (lambda z: self.get_item(k) / z.cat)
        return List.wrap(keys).fold_left(Just(Map()))(append)

    def get_or_else(self, key, default: Callable[[], C]):
        return self.get(key).get_or_else(default)

    def set_if_missing(self, key: A, default: Callable[[], B]):
        if key in self:
            return self
        else:
            return self + (key, default())

    def __str__(self):
        return str(dict(self))

    def cat(self, item: Tuple[A, B]):
        return Map(dicttoolz.assoc(self, *item))

    __add__ = cat

    def merge(self, other: 'Map[A, B]'):
        return Map(dicttoolz.merge(self, other))

    __pow__ = merge

    def __sub__(self, key: A):
        return Map(dicttoolz.dissoc(self, key))

    def find(self, f: Callable[[B], bool]) -> Maybe[Tuple[A, B]]:
        return self.to_list.find(lambda item: f(item[1]))

    def find_key(self, f: Callable[[A], bool]) -> Maybe[Tuple[A, B]]:
        return self.to_list.find(lambda item: f(item[0]))

    def valfilter(self, f: Callable[[B], bool]) -> 'Map[A, B]':
        return Map(dicttoolz.valfilter(f, self))

    def keyfilter(self, f: Callable[[A], bool]) -> 'Map[A, B]':
        return Map(dicttoolz.keyfilter(f, self))

    def filter(self, f: Callable[[Tuple[A, B]], bool]) -> 'Map[A, B]':
        return Map(dicttoolz.itemfilter(f, self))

    def valmap(self, f: Callable[[B], C]) -> 'Map[A, C]':
        return Map(dicttoolz.valmap(f, dict(self)))

    def keymap(self, f: Callable[[A], C]) -> 'Map[C, B]':
        return Map(dicttoolz.keymap(f, dict(self)))

    def map(self, f: Callable[[A, B], Tuple[C, D]]) -> 'Map[C, D]':
        wrap = lambda a: f(*a)
        return Map(dicttoolz.itemmap(wrap, self))

    def flat_map(
            self,
            f: Callable[[A, B], Maybe[Tuple[C, D]]]
    ) -> 'Map[C, D]':
        filtered = List.wrap([f(a, b) for a, b in self.items()])\
            .join
        return Map(filtered)

    def bimap(self, fa: Callable[[A], C], fb: Callable[[B], D]) -> 'Map[C, D]':
        return self.map(lambda a, b: (fa(a), fb(b)))

    def map2(self, f: Callable[[A, B], C]) -> List[C]:
        return List.wrap([f(a, b) for a, b in self.items()])

    def flat_map2(self, f: Callable[[A, B], List[C]]) -> List[C]:
        return self.to_list.flat_map2(f)

    @property
    def to_list(self):
        return List.wrap(self.items())

    @property
    def head(self):
        return self.to_list.head

    @property
    def keys_view(self):
        return Dict.keys(self)

    @property
    def values_view(self):
        return Dict.values(self)

    @property
    def k(self):
        return List(*Dict.keys(self))

    @property
    def v(self) -> List[B]:
        return Lists.wrap(Dict.values(self))

    @property
    def is_empty(self):
        return self.k.is_empty

    empty = is_empty

    def at(self, *keys):
        return self.keyfilter(lambda a: a in keys)

    def values_at(self, *keys):
        return List.wrap(keys) // self.get

    def has_key(self, name):
        return Boolean(name in self)

    contains = has_key

    def add_maybe(self, key: A, value: Maybe[B]):
        return value / (lambda a: self + (key, a)) | self

__all__ = ('Map',)
<commit_msg>`Map.lift_all`<commit_after>from typing import TypeVar, Dict, Generic, Tuple, Callable

from toolz import dicttoolz

from amino import Maybe, may, Just, Nothing
from amino.list import List, Lists
from amino.boolean import Boolean
from amino.tc.base import Implicits, ImplicitsMeta

A = TypeVar('A')
B = TypeVar('B', covariant=True)
C = TypeVar('C')
D = TypeVar('D', covariant=True)

class Map(Generic[A, B], Dict[A, B], Implicits, metaclass=ImplicitsMeta, implicits=True):

    @staticmethod
    def wrap(d: Dict[A, B]) -> 'Map[A, B]':
        return Map(d)

    @may
    def get(self, key):
        return Dict.get(self, key)

    def lift(self, key: str) -> Maybe[B]:
        return Just(Dict.get(self, key)) if key in self else Nothing

    def get_item(self, key):
        return self.get(key) / (lambda a: (key, a))

    def get_all(self, *keys):
        def append(zm, k):
            return zm // (lambda z: self.get(k) / z.cat)
        return List.wrap(keys).fold_left(Just(List()))(append)

    lift_all = get_all

    def get_all_map(self, *keys):
        def append(zm, k):
            return zm // (lambda z: self.get_item(k) / z.cat)
        return List.wrap(keys).fold_left(Just(Map()))(append)

    def get_or_else(self, key, default: Callable[[], C]):
        return self.get(key).get_or_else(default)

    def set_if_missing(self, key: A, default: Callable[[], B]):
        if key in self:
            return self
        else:
            return self + (key, default())

    def __str__(self):
        return str(dict(self))

    def cat(self, item: Tuple[A, B]):
        return Map(dicttoolz.assoc(self, *item))

    __add__ = cat

    def merge(self, other: 'Map[A, B]'):
        return Map(dicttoolz.merge(self, other))

    __pow__ = merge

    def __sub__(self, key: A):
        return Map(dicttoolz.dissoc(self, key))

    def find(self, f: Callable[[B], bool]) -> Maybe[Tuple[A, B]]:
        return self.to_list.find(lambda item: f(item[1]))

    def find_key(self, f: Callable[[A], bool]) -> Maybe[Tuple[A, B]]:
        return self.to_list.find(lambda item: f(item[0]))

    def valfilter(self, f: Callable[[B], bool]) -> 'Map[A, B]':
        return Map(dicttoolz.valfilter(f, self))

    def keyfilter(self, f: Callable[[A], bool]) -> 'Map[A, B]':
        return Map(dicttoolz.keyfilter(f, self))

    def filter(self, f: Callable[[Tuple[A, B]], bool]) -> 'Map[A, B]':
        return Map(dicttoolz.itemfilter(f, self))

    def valmap(self, f: Callable[[B], C]) -> 'Map[A, C]':
        return Map(dicttoolz.valmap(f, dict(self)))

    def keymap(self, f: Callable[[A], C]) -> 'Map[C, B]':
        return Map(dicttoolz.keymap(f, dict(self)))

    def map(self, f: Callable[[A, B], Tuple[C, D]]) -> 'Map[C, D]':
        wrap = lambda a: f(*a)
        return Map(dicttoolz.itemmap(wrap, self))

    def flat_map(
            self,
            f: Callable[[A, B], Maybe[Tuple[C, D]]]
    ) -> 'Map[C, D]':
        filtered = List.wrap([f(a, b) for a, b in self.items()])\
            .join
        return Map(filtered)

    def bimap(self, fa: Callable[[A], C], fb: Callable[[B], D]) -> 'Map[C, D]':
        return self.map(lambda a, b: (fa(a), fb(b)))

    def map2(self, f: Callable[[A, B], C]) -> List[C]:
        return List.wrap([f(a, b) for a, b in self.items()])

    def flat_map2(self, f: Callable[[A, B], List[C]]) -> List[C]:
        return self.to_list.flat_map2(f)

    @property
    def to_list(self):
        return List.wrap(self.items())

    @property
    def head(self):
        return self.to_list.head

    @property
    def keys_view(self):
        return Dict.keys(self)

    @property
    def values_view(self):
        return Dict.values(self)

    @property
    def k(self):
        return List(*Dict.keys(self))

    @property
    def v(self) -> List[B]:
        return Lists.wrap(Dict.values(self))

    @property
    def is_empty(self):
        return self.k.is_empty

    empty = is_empty

    def at(self, *keys):
        return self.keyfilter(lambda a: a in keys)

    def values_at(self, *keys):
        return List.wrap(keys) // self.get

    def has_key(self, name):
        return Boolean(name in self)

    contains = has_key

    def add_maybe(self, key: A, value: Maybe[B]):
        return value / (lambda a: self + (key, a)) | self

__all__ = ('Map',)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    amqp.five
    ~~~~~~~~~~~

    Compatibility implementations of features
    only available in newer Python versions.

"""
from __future__ import absolute_import

import io
import sys

try:
    from collections import Counter
except ImportError:  # pragma: no cover
    from collections import defaultdict

    def Counter():  # noqa
        return defaultdict(int)

try:
    buffer_t = buffer
except NameError:  # pragma: no cover
    # Py3 does not have buffer, only use this for isa checks.

    class buffer_t(object):  # noqa
        pass

bytes_t = bytes

__all__ = ['Counter', 'reload', 'UserList', 'UserDict',
           'Queue', 'Empty', 'Full', 'LifoQueue', 'builtins',
           'zip_longest', 'map', 'zip', 'string', 'string_t', 'bytes_t',
           'long_t', 'text_t', 'int_types', 'module_name_t',
           'range', 'items', 'keys', 'values', 'nextfun', 'reraise',
           'WhateverIO', 'with_metaclass', 'open_fqdn', 'StringIO',
           'THREAD_TIMEOUT_MAX', 'format_d', 'monotonic', 'buffer_t']

#  ############# py3k ########################################################
PY3 = sys.version_info[0] == 3

try:
    reload = reload                         # noqa
except NameError:                           # pragma: no cover
    from imp import reload                  # noqa

try:
    from collections import UserList        # noqa
except ImportError:                         # pragma: no cover
    from UserList import UserList           # noqa

try:
    from collections import UserDict        # noqa
except ImportError:                         # pragma: no cover
    from UserDict import UserDict           # noqa

#  ############# time.monotonic #############################################

if sys.version_info < (3, 3):

    import platform
    SYSTEM = platform.system()

    if SYSTEM == 'Darwin':
        import ctypes
        from ctypes.util import find_library
        libSystem = ctypes.CDLL('libSystem.dylib')
        CoreServices = ctypes.CDLL(find_library('CoreServices'),
                                   use_errno=True)
        mach_absolute_time = libSystem.mach_absolute_time
        mach_absolute_time.restype = ctypes.c_uint64
        absolute_to_nanoseconds = CoreServices.AbsoluteToNanoseconds
        absolute_to_nanoseconds.restype = ctypes.c_uint64
        absolute_to_nanoseconds.argtypes = [ctypes.c_uint64]

        def _monotonic():
            return absolute_to_nanoseconds(mach_absolute_time()) * 1e-9

    elif SYSTEM == 'Linux':
        # from stackoverflow:
        # questions/1205722/how-do-i-get-monotonic-time-durations-in-python
        import ctypes
        import os

        CLOCK_MONOTONIC = 1  # see <linux/time.h>

        class timespec(ctypes.Structure):
            _fields_ = [
                ('tv_sec', ctypes.c_long),
                ('tv_nsec', ctypes.c_long),
            ]

        librt = ctypes.CDLL('librt.so.1', use_errno=True)
        clock_gettime = librt.clock_gettime
        clock_gettime.argtypes = [
            ctypes.c_int, ctypes.POINTER(timespec),
        ]

        def _monotonic():  # noqa
            t = timespec()
            if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(t)) != 0:
                errno_ = ctypes.get_errno()
                raise OSError(errno_, os.strerror(errno_))
            return t.tv_sec + t.tv_nsec * 1e-9
    else:
        from time import time as _monotonic
try:
    from time import monotonic
except ImportError:
    monotonic = _monotonic  # noqa

# ############# Py3 <-> Py2 #################################################

if PY3:  # pragma: no cover
    import builtins

    from queue import Queue, Empty, Full, LifoQueue
    from itertools import zip_longest

    map = map
    zip = zip
    string = str
    string_t = str
    long_t = int
    text_t = str
    range = range
    int_types = (int,)
    module_name_t = str

    open_fqdn = 'builtins.open'

    def items(d):
        return d.items()

    def keys(d):
        return d.keys()

    def values(d):
        return d.values()

    def nextfun(it):
        return it.__next__

    exec_ = getattr(builtins, 'exec')

    def reraise(tp, value, tb=None):
        if value.__traceback__ is not tb:
            raise value.with_traceback(tb)
        raise value

else:
    import __builtin__ as builtins  # noqa
    from Queue import Queue, Empty, Full, LifoQueue  # noqa
    from itertools import (               # noqa
        imap as map,
        izip as zip,
        izip_longest as zip_longest,
    )

    string = unicode                # noqa
    string_t = basestring           # noqa
    text_t = unicode
    long_t = long                   # noqa
    range = xrange
    module_name_t = str
    int_types = (int, long)

    open_fqdn = '__builtin__.open'

    def items(d):                   # noqa
        return d.iteritems()

    def keys(d):                    # noqa
        return d.iterkeys()

    def values(d):                  # noqa
        return d.itervalues()

    def nextfun(it):                # noqa
        return it.next

    def exec_(code, globs=None, locs=None):  # pragma: no cover
        """Execute code in a namespace."""
        if globs is None:
            frame = sys._getframe(1)
            globs = frame.f_globals
            if locs is None:
                locs = frame.f_locals
            del frame
        elif locs is None:
            locs = globs
        exec("""exec code in globs, locs""")

    exec_("""def reraise(tp, value, tb=None): raise tp, value, tb""")

def with_metaclass(Type, skip_attrs={'__dict__', '__weakref__'}):
    """Class decorator to set metaclass.

    Works with both Python 2 and Python 3 and it does not add
    an extra class in the lookup order like ``six.with_metaclass`` does
    (that is -- it copies the original class instead of using inheritance).

    """

    def _clone_with_metaclass(Class):
        attrs = {key: value for key, value in items(vars(Class))
                 if key not in skip_attrs}
        return Type(Class.__name__, Class.__bases__, attrs)

    return _clone_with_metaclass

# ############# threading.TIMEOUT_MAX ########################################
try:
    from threading import TIMEOUT_MAX as THREAD_TIMEOUT_MAX
except ImportError:
    THREAD_TIMEOUT_MAX = 1e10  # noqa

# ############# format(int, ',d') ############################################

if sys.version_info >= (2, 7):  # pragma: no cover
    def format_d(i):
        return format(i, ',d')
else:  # pragma: no cover
    def format_d(i):  # noqa
        s = '%d' % i
        groups = []
        while s and s[-1].isdigit():
            groups.append(s[-3:])
            s = s[:-3]
        return s + ','.join(reversed(groups))

StringIO = io.StringIO
_SIO_write = StringIO.write
_SIO_init = StringIO.__init__

class WhateverIO(StringIO):

    def __init__(self, v=None, *a, **kw):
        _SIO_init(self, v.decode() if isinstance(v, bytes) else v, *a, **kw)

    def write(self, data):
        _SIO_write(self, data.decode() if isinstance(data, bytes) else data)
<commit_msg>Fixes libSystem error on El Capitan (see celery/kombu#522)<commit_after># -*- coding: utf-8 -*-
"""
    amqp.five
    ~~~~~~~~~~~

    Compatibility implementations of features
    only available in newer Python versions.

"""
from __future__ import absolute_import

import io
import sys

try:
    from collections import Counter
except ImportError:  # pragma: no cover
    from collections import defaultdict

    def Counter():  # noqa
        return defaultdict(int)

try:
    buffer_t = buffer
except NameError:  # pragma: no cover
    # Py3 does not have buffer, only use this for isa checks.

    class buffer_t(object):  # noqa
        pass

bytes_t = bytes

__all__ = ['Counter', 'reload', 'UserList', 'UserDict',
           'Queue', 'Empty', 'Full', 'LifoQueue', 'builtins',
           'zip_longest', 'map', 'zip', 'string', 'string_t', 'bytes_t',
           'long_t', 'text_t', 'int_types', 'module_name_t',
           'range', 'items', 'keys', 'values', 'nextfun', 'reraise',
           'WhateverIO', 'with_metaclass', 'open_fqdn', 'StringIO',
           'THREAD_TIMEOUT_MAX', 'format_d', 'monotonic', 'buffer_t']

#  ############# py3k ########################################################
PY3 = sys.version_info[0] == 3

try:
    reload = reload                         # noqa
except NameError:                           # pragma: no cover
    from imp import reload                  # noqa

try:
    from collections import UserList        # noqa
except ImportError:                         # pragma: no cover
    from UserList import UserList           # noqa

try:
    from collections import UserDict        # noqa
except ImportError:                         # pragma: no cover
    from UserDict import UserDict           # noqa

#  ############# time.monotonic #############################################

if sys.version_info < (3, 3):

    import platform
    SYSTEM = platform.system()

    if SYSTEM == 'Darwin':
        import ctypes
        from ctypes.util import find_library
        libSystem = ctypes.CDLL(find_library('libSystem.dylib'))
        CoreServices = ctypes.CDLL(find_library('CoreServices'),
                                   use_errno=True)
        mach_absolute_time = libSystem.mach_absolute_time
        mach_absolute_time.restype = ctypes.c_uint64
        absolute_to_nanoseconds = CoreServices.AbsoluteToNanoseconds
        absolute_to_nanoseconds.restype = ctypes.c_uint64
        absolute_to_nanoseconds.argtypes = [ctypes.c_uint64]

        def _monotonic():
            return absolute_to_nanoseconds(mach_absolute_time()) * 1e-9

    elif SYSTEM == 'Linux':
        # from stackoverflow:
        # questions/1205722/how-do-i-get-monotonic-time-durations-in-python
        import ctypes
        import os

        CLOCK_MONOTONIC = 1  # see <linux/time.h>

        class timespec(ctypes.Structure):
            _fields_ = [
                ('tv_sec', ctypes.c_long),
                ('tv_nsec', ctypes.c_long),
            ]

        librt = ctypes.CDLL('librt.so.1', use_errno=True)
        clock_gettime = librt.clock_gettime
        clock_gettime.argtypes = [
            ctypes.c_int, ctypes.POINTER(timespec),
        ]

        def _monotonic():  # noqa
            t = timespec()
            if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(t)) != 0:
                errno_ = ctypes.get_errno()
                raise OSError(errno_, os.strerror(errno_))
            return t.tv_sec + t.tv_nsec * 1e-9
    else:
        from time import time as _monotonic
try:
    from time import monotonic
except ImportError:
    monotonic = _monotonic  # noqa

# ############# Py3 <-> Py2 #################################################

if PY3:  # pragma: no cover
    import builtins

    from queue import Queue, Empty, Full, LifoQueue
    from itertools import zip_longest

    map = map
    zip = zip
    string = str
    string_t = str
    long_t = int
    text_t = str
    range = range
    int_types = (int,)
    module_name_t = str

    open_fqdn = 'builtins.open'

    def items(d):
        return d.items()

    def keys(d):
        return d.keys()

    def values(d):
        return d.values()

    def nextfun(it):
        return it.__next__

    exec_ = getattr(builtins, 'exec')

    def reraise(tp, value, tb=None):
        if value.__traceback__ is not tb:
            raise value.with_traceback(tb)
        raise value

else:
    import __builtin__ as builtins  # noqa
    from Queue import Queue, Empty, Full, LifoQueue  # noqa
    from itertools import (               # noqa
        imap as map,
        izip as zip,
        izip_longest as zip_longest,
    )

    string = unicode                # noqa
    string_t = basestring           # noqa
    text_t = unicode
    long_t = long                   # noqa
    range = xrange
    module_name_t = str
    int_types = (int, long)

    open_fqdn = '__builtin__.open'

    def items(d):                   # noqa
        return d.iteritems()

    def keys(d):                    # noqa
        return d.iterkeys()

    def values(d):                  # noqa
        return d.itervalues()

    def nextfun(it):                # noqa
        return it.next

    def exec_(code, globs=None, locs=None):  # pragma: no cover
        """Execute code in a namespace."""
        if globs is None:
            frame = sys._getframe(1)
            globs = frame.f_globals
            if locs is None:
                locs = frame.f_locals
            del frame
        elif locs is None:
            locs = globs
        exec("""exec code in globs, locs""")

    exec_("""def reraise(tp, value, tb=None): raise tp, value, tb""")

def with_metaclass(Type, skip_attrs={'__dict__', '__weakref__'}):
    """Class decorator to set metaclass.

    Works with both Python 2 and Python 3 and it does not add
    an extra class in the lookup order like ``six.with_metaclass`` does
    (that is -- it copies the original class instead of using inheritance).

    """

    def _clone_with_metaclass(Class):
        attrs = {key: value for key, value in items(vars(Class))
                 if key not in skip_attrs}
        return Type(Class.__name__, Class.__bases__, attrs)

    return _clone_with_metaclass

# ############# threading.TIMEOUT_MAX ########################################
try:
    from threading import TIMEOUT_MAX as THREAD_TIMEOUT_MAX
except ImportError:
    THREAD_TIMEOUT_MAX = 1e10  # noqa

# ############# format(int, ',d') ############################################

if sys.version_info >= (2, 7):  # pragma: no cover
    def format_d(i):
        return format(i, ',d')
else:  # pragma: no cover
    def format_d(i):  # noqa
        s = '%d' % i
        groups = []
        while s and s[-1].isdigit():
            groups.append(s[-3:])
            s = s[:-3]
        return s + ','.join(reversed(groups))

StringIO = io.StringIO
_SIO_write = StringIO.write
_SIO_init = StringIO.__init__

class WhateverIO(StringIO):

    def __init__(self, v=None, *a, **kw):
        _SIO_init(self, v.decode() if isinstance(v, bytes) else v, *a, **kw)

    def write(self, data):
        _SIO_write(self, data.decode() if isinstance(data, bytes) else data)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    amqp.five
    ~~~~~~~~~~~

    Compatibility implementations of features
    only available in newer Python versions.

"""
from __future__ import absolute_import, unicode_literals

import io
import sys

try:
    from collections import Counter
except ImportError:  # pragma: no cover
    from collections import defaultdict

    def Counter():  # noqa
        return defaultdict(int)

try:
    buffer_t = buffer
except NameError:  # pragma: no cover
    # Py3 does not have buffer, only use this for isa checks.

    class buffer_t(object):  # noqa
        pass

bytes_t = bytes

__all__ = ['Counter', 'reload', 'UserList', 'UserDict',
           'Queue', 'Empty', 'Full', 'LifoQueue', 'builtins', 'array',
           'zip_longest', 'map', 'zip', 'string', 'string_t', 'bytes_t',
           'bytes_if_py2', 'long_t', 'text_t', 'int_types', 'module_name_t',
           'range', 'items', 'keys', 'values', 'nextfun', 'reraise',
           'WhateverIO', 'with_metaclass', 'StringIO',
           'THREAD_TIMEOUT_MAX', 'format_d', 'monotonic', 'buffer_t']

#  ############# py3k ########################################################
PY3 = sys.version_info[0] >= 3
PY2 = sys.version_info[0] < 3

try:
    reload = reload                         # noqa
except NameError:                           # pragma: no cover
    from imp import reload                  # noqa

try:
    from collections import UserList        # noqa
except ImportError:                         # pragma: no cover
    from UserList import UserList           # noqa

try:
    from collections import UserDict        # noqa
except ImportError:                         # pragma: no cover
    from UserDict import UserDict           # noqa

#  ############# time.monotonic #############################################

if sys.version_info < (3, 3):

    import platform
    SYSTEM = platform.system()

    try:
        import ctypes
    except ImportError:  # pragma: no cover
        ctypes = None  # noqa

    if SYSTEM == 'Darwin' and ctypes is not None:
        from ctypes.util import find_library
        libSystem = ctypes.CDLL(find_library('libSystem.dylib'))
        CoreServices = ctypes.CDLL(find_library('CoreServices'),
                                   use_errno=True)
        mach_absolute_time = libSystem.mach_absolute_time
        mach_absolute_time.restype = ctypes.c_uint64
        absolute_to_nanoseconds = CoreServices.AbsoluteToNanoseconds
        absolute_to_nanoseconds.restype = ctypes.c_uint64
        absolute_to_nanoseconds.argtypes = [ctypes.c_uint64]

        def _monotonic():
            return absolute_to_nanoseconds(mach_absolute_time()) * 1e-9

    elif SYSTEM == 'Linux' and ctypes is not None:
        # from stackoverflow:
        # questions/1205722/how-do-i-get-monotonic-time-durations-in-python
        import os

        CLOCK_MONOTONIC = 1  # see <linux/time.h>

        class timespec(ctypes.Structure):
            _fields_ = [
                ('tv_sec', ctypes.c_long),
                ('tv_nsec', ctypes.c_long),
            ]

        librt = ctypes.CDLL('librt.so.1', use_errno=True)
        clock_gettime = librt.clock_gettime
        clock_gettime.argtypes = [
            ctypes.c_int, ctypes.POINTER(timespec),
        ]

        def _monotonic():  # noqa
            t = timespec()
            if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(t)) != 0:
                errno_ = ctypes.get_errno()
                raise OSError(errno_, os.strerror(errno_))
            return t.tv_sec + t.tv_nsec * 1e-9
    else:
        from time import time as _monotonic
try:
    from time import monotonic
except ImportError:
    monotonic = _monotonic  # noqa

# ############# Py3 <-> Py2 #################################################

if PY3:  # pragma: no cover
    import builtins

    from array import array
    from queue import Queue, Empty, Full, LifoQueue
    from itertools import zip_longest

    map = map
    zip = zip
    string = str
    string_t = str
    long_t = int
    text_t = str
    range = range
    int_types = (int,)
    module_name_t = str

    def bytes_if_py2(s):
        return s

    def items(d):
        return d.items()

    def keys(d):
        return d.keys()

    def values(d):
        return d.values()

    def nextfun(it):
        return it.__next__

    exec_ = getattr(builtins, 'exec')

    def reraise(tp, value, tb=None):
        if value.__traceback__ is not tb:
            raise value.with_traceback(tb)
        raise value

else:
    import __builtin__ as builtins  # noqa
    from array import array as _array
    from Queue import Queue, Empty, Full, LifoQueue  # noqa
    from itertools import (               # noqa
        imap as map,
        izip as zip,
        izip_longest as zip_longest,
    )

    string = unicode                # noqa
    string_t = basestring           # noqa
    text_t = unicode
    long_t = long                   # noqa
    range = xrange
    module_name_t = str
    int_types = (int, long)

    def array(typecode, *args, **kwargs):
        if isinstance(typecode, unicode):
            typecode = typecode.encode()
        return _array(typecode, *args, **kwargs)

    def bytes_if_py2(s):
        if isinstance(s, unicode):
            return s.encode()
        return s

    def items(d):                   # noqa
        return d.iteritems()

    def keys(d):                    # noqa
        return d.iterkeys()

    def values(d):                  # noqa
        return d.itervalues()

    def nextfun(it):                # noqa
        return it.next

    def exec_(code, globs=None, locs=None):  # pragma: no cover
        """Execute code in a namespace."""
        if globs is None:
            frame = sys._getframe(1)
            globs = frame.f_globals
            if locs is None:
                locs = frame.f_locals
            del frame
        elif locs is None:
            locs = globs
        exec("""exec code in globs, locs""")

    exec_("""def reraise(tp, value, tb=None): raise tp, value, tb""")

def with_metaclass(Type, skip_attrs={'__dict__', '__weakref__'}):
    """Class decorator to set metaclass.

    Works with both Python 2 and Python 3 and it does not add
    an extra class in the lookup order like ``six.with_metaclass`` does
    (that is -- it copies the original class instead of using inheritance).

    """

    def _clone_with_metaclass(Class):
        attrs = {key: value for key, value in items(vars(Class))
                 if key not in skip_attrs}
        return Type(Class.__name__, Class.__bases__, attrs)

    return _clone_with_metaclass

# ############# threading.TIMEOUT_MAX ########################################
try:
    from threading import TIMEOUT_MAX as THREAD_TIMEOUT_MAX
except ImportError:
    THREAD_TIMEOUT_MAX = 1e10  # noqa

# ############# format(int, ',d') ############################################

if sys.version_info >= (2, 7):  # pragma: no cover
    def format_d(i):
        return format(i, ',d')
else:  # pragma: no cover
    def format_d(i):  # noqa
        s = '%d' % i
        groups = []
        while s and s[-1].isdigit():
            groups.append(s[-3:])
            s = s[:-3]
        return s + ','.join(reversed(groups))

StringIO = io.StringIO
_SIO_write = StringIO.write
_SIO_init = StringIO.__init__

class WhateverIO(StringIO):

    def __init__(self, v=None, *a, **kw):
        _SIO_init(self, v.decode() if isinstance(v, bytes) else v, *a, **kw)

    def write(self, data):
        _SIO_write(self, data.decode() if isinstance(data, bytes) else data)

def python_2_unicode_compatible(cls):
    """A decorator that defines __unicode__ and __str__ methods under Python 2.
    Under Python 3 it does nothing.

    To support Python 2 and 3 with a single code base, define a __str__ method
    returning text and apply this decorator to the class.

    """
    if PY2:
        if '__str__' not in cls.__dict__:
            raise ValueError(
                "@python_2_unicode_compatible cannot be applied "
                "to {0} because it doesn't define __str__().".format(
                    cls.__name__))
        cls.__unicode__ = cls.__str__
        cls.__str__ = lambda self: self.__unicode__().encode('utf-8')
    return cls
<commit_msg>flakes<commit_after># -*- coding: utf-8 -*-
"""
    amqp.five
    ~~~~~~~~~~~

    Compatibility implementations of features
    only available in newer Python versions.

"""
from __future__ import absolute_import, unicode_literals

import io
import sys

try:
    from collections import Counter
except ImportError:  # pragma: no cover
    from collections import defaultdict

    def Counter():  # noqa
        return defaultdict(int)

try:
    buffer_t = buffer
except NameError:  # pragma: no cover
    # Py3 does not have buffer, only use this for isa checks.

    class buffer_t(object):  # noqa
        pass

bytes_t = bytes

__all__ = ['Counter', 'reload', 'UserList', 'UserDict',
           'Queue', 'Empty', 'Full', 'LifoQueue', 'builtins', 'array',
           'zip_longest', 'map', 'zip', 'string', 'string_t', 'bytes_t',
           'bytes_if_py2', 'long_t', 'text_t', 'int_types', 'module_name_t',
           'range', 'items', 'keys', 'values', 'nextfun', 'reraise',
           'WhateverIO', 'with_metaclass', 'StringIO',
           'THREAD_TIMEOUT_MAX', 'format_d', 'monotonic', 'buffer_t']

#  ############# py3k ########################################################
PY3 = sys.version_info[0] >= 3
PY2 = sys.version_info[0] < 3

try:
    reload = reload                         # noqa
except NameError:                           # pragma: no cover
    from imp import reload                  # noqa

try:
    from collections import UserList        # noqa
except ImportError:                         # pragma: no cover
    from UserList import UserList           # noqa

try:
    from collections import UserDict        # noqa
except ImportError:                         # pragma: no cover
    from UserDict import UserDict           # noqa

#  ############# time.monotonic #############################################

if sys.version_info < (3, 3):

    import platform
    SYSTEM = platform.system()

    try:
        import ctypes
    except ImportError:  # pragma: no cover
        ctypes = None  # noqa

    if SYSTEM == 'Darwin' and ctypes is not None:
        from ctypes.util import find_library
        libSystem = ctypes.CDLL(find_library('libSystem.dylib'))
        CoreServices = ctypes.CDLL(find_library('CoreServices'),
                                   use_errno=True)
        mach_absolute_time = libSystem.mach_absolute_time
        mach_absolute_time.restype = ctypes.c_uint64
        absolute_to_nanoseconds = CoreServices.AbsoluteToNanoseconds
        absolute_to_nanoseconds.restype = ctypes.c_uint64
        absolute_to_nanoseconds.argtypes = [ctypes.c_uint64]

        def _monotonic():
            return absolute_to_nanoseconds(mach_absolute_time()) * 1e-9

    elif SYSTEM == 'Linux' and ctypes is not None:
        # from stackoverflow:
        # questions/1205722/how-do-i-get-monotonic-time-durations-in-python
        import os

        CLOCK_MONOTONIC = 1  # see <linux/time.h>

        class timespec(ctypes.Structure):
            _fields_ = [
                ('tv_sec', ctypes.c_long),
                ('tv_nsec', ctypes.c_long),
            ]

        librt = ctypes.CDLL('librt.so.1', use_errno=True)
        clock_gettime = librt.clock_gettime
        clock_gettime.argtypes = [
            ctypes.c_int, ctypes.POINTER(timespec),
        ]

        def _monotonic():  # noqa
            t = timespec()
            if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(t)) != 0:
                errno_ = ctypes.get_errno()
                raise OSError(errno_, os.strerror(errno_))
            return t.tv_sec + t.tv_nsec * 1e-9
    else:
        from time import time as _monotonic
try:
    from time import monotonic
except ImportError:
    monotonic = _monotonic  # noqa

# ############# Py3 <-> Py2 #################################################

if PY3:  # pragma: no cover
    import builtins

    from array import array
    from queue import Queue, Empty, Full, LifoQueue
    from itertools import zip_longest

    map = map
    zip = zip
    string = str
    string_t = str
    long_t = int
    text_t = str
    range = range
    int_types = (int,)
    module_name_t = str

    def bytes_if_py2(s):
        return s

    def items(d):
        return d.items()

    def keys(d):
        return d.keys()

    def values(d):
        return d.values()

    def nextfun(it):
        return it.__next__

    exec_ = getattr(builtins, 'exec')

    def reraise(tp, value, tb=None):
        if value.__traceback__ is not tb:
            raise value.with_traceback(tb)
        raise value

else:
    import __builtin__ as builtins  # noqa
    from array import array as _array
    from Queue import Queue, Empty, Full, LifoQueue  # noqa
    from itertools import (               # noqa
        imap as map,
        izip as zip,
        izip_longest as zip_longest,
    )

    string = unicode                # noqa
    string_t = basestring           # noqa
    text_t = unicode
    long_t = long                   # noqa
    range = xrange
    module_name_t = str
    int_types = (int, long)

    def array(typecode, *args, **kwargs):
        if isinstance(typecode, unicode):
            typecode = typecode.encode()
        return _array(typecode, *args, **kwargs)

    def bytes_if_py2(s):
        if isinstance(s, unicode):
            return s.encode()
        return s

    def items(d):                   # noqa
        return d.iteritems()

    def keys(d):                    # noqa
        return d.iterkeys()

    def values(d):                  # noqa
        return d.itervalues()

    def nextfun(it):                # noqa
        return it.next

    def exec_(code, globs=None, locs=None):  # pragma: no cover
        """Execute code in a namespace."""
        if globs is None:
            frame = sys._getframe(1)
            globs = frame.f_globals
            if locs is None:
                locs = frame.f_locals
            del frame
        elif locs is None:
            locs = globs
        exec("""exec code in globs, locs""")

    exec_("""def reraise(tp, value, tb=None): raise tp, value, tb""")

def with_metaclass(Type, skip_attrs={'__dict__', '__weakref__'}):
    """Class decorator to set metaclass.

    Works with both Python 2 and Python 3 and it does not add
    an extra class in the lookup order like ``six.with_metaclass`` does
    (that is -- it copies the original class instead of using inheritance).

    """

    def _clone_with_metaclass(Class):
        attrs = {key: value for key, value in items(vars(Class))
                 if key not in skip_attrs}
        return Type(Class.__name__, Class.__bases__, attrs)

    return _clone_with_metaclass

# ############# threading.TIMEOUT_MAX ########################################
try:
    from threading import TIMEOUT_MAX as THREAD_TIMEOUT_MAX
except ImportError:
    THREAD_TIMEOUT_MAX = 1e10  # noqa

# ############# format(int, ',d') ############################################

if sys.version_info >= (2, 7):  # pragma: no cover
    def format_d(i):
        return format(i, ',d')
else:  # pragma: no cover
    def format_d(i):  # noqa
        s = '%d' % i
        groups = []
        while s and s[-1].isdigit():
            groups.append(s[-3:])
            s = s[:-3]
        return s + ','.join(reversed(groups))

StringIO = io.StringIO
_SIO_write = StringIO.write
_SIO_init = StringIO.__init__

class WhateverIO(StringIO):

    def __init__(self, v=None, *a, **kw):
        _SIO_init(self, v.decode() if isinstance(v, bytes) else v, *a, **kw)

    def write(self, data):
        _SIO_write(self, data.decode() if isinstance(data, bytes) else data)

def python_2_unicode_compatible(cls):
    """A decorator that defines __unicode__ and __str__ methods under Python 2.
    Under Python 3 it does nothing.

    To support Python 2 and 3 with a single code base, define a __str__ method
    returning text and apply this decorator to the class.

    """
    if PY2:
        if '__str__' not in cls.__dict__:
            raise ValueError(
                "@python_2_unicode_compatible cannot be applied "
                "to {0} because it doesn't define __str__().".format(
                    cls.__name__))
        cls.__unicode__ = cls.__str__
        cls.__str__ = lambda self: self.__unicode__().encode('utf-8')
    return cls
<|endoftext|>


<commit_before>#!/usr/bin/env python

import csv
import re
import os
from datetime import datetime, timedelta
from collections import Counter, defaultdict
import http.client
import json
from math import cos, sin, pi

ipre = re.compile('(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})')

# IP ranges to exclude that aren't caught automatically for bot-like
# behavior
ignore_prefix = [tuple(p.split('.'))
                 for p in ['150.70', '202.46', '75.130.244.15', '64.233',
                           '96.47.226.21', '178.63.97.34', '89.31.57.5',
                           '85.159.113.228', '148.251.77.34',
                           '178.217.187.39']]

# Tests for bot-like behavior
def suspicious_key(key):
    return (key == 'robots.txt' or 'googled' in key)
def suspicious_referrer(referrer):
    return ('semalt' in referrer or 'buttons' in referrer or
            'best-seo' in referrer)
def suspicious_agent(agent):
    return ('bot' in agent or 'Bot' in agent or 'spider' in agent or
            'Google favicon' in agent or 'Disqus' in agent or
            'prismatic' in agent)

log_files = os.listdir('logs/')

ip_hits = Counter()
country_hits = Counter()
region_hits = Counter()
date_hits = Counter()
hour_hits = Counter()
time_hits = []
res_hits = Counter()
referrer_hits = Counter()
agent_hits = Counter()
possible_robots = set()
excluded = set()

class GeoIP():
    def __init__(self):
        self.conn = http.client.HTTPConnection('freegeoip.net', timeout = 10)

        try:
            with open('cache/geoip.json', 'r') as infile:
                self.cache = json.load(infile)
        except:
            self.cache = {}

    def fetch(self, ip):
        if ip in self.cache:
            return self.cache[ip]

        try:
            self.conn.request('GET', '/json/' + ip)

            response = self.conn.getresponse()
            print('Querying GeoIP:', response.status, response.reason)

            data_serialized = response.read().decode('utf-8')
            data = json.loads(data_serialized)

            self.cache[ip] = data

            return data
        except:
            return None

    def dump(self):
        with open('cache/geoip.json', 'w') as outfile:
            json.dump(self.cache, outfile)

geoip = GeoIP()

# First pass to detect non-human access
for infile in log_files:
    with open(os.path.join('logs', infile), 'r') as log:
        log_reader = csv.reader(log, delimiter = ' ', quotechar = '"')

        for line in log_reader:
            remote_ip_str = line[4]

            if suspicious_key(line[8]):
                possible_robots.add(remote_ip_str)
            if suspicious_referrer(line[16]):
                possible_robots.add(remote_ip_str)
            if suspicious_agent(line[17]):
                possible_robots.add(remote_ip_str)

for infile in log_files:
    with open(os.path.join('logs', infile), 'r') as log:
        log_reader = csv.reader(log, delimiter = ' ', quotechar = '"')

        first_in_file = True
        for line in log_reader:
            # Parse remote IP so it can be checked against prefixes
            remote_ip_str = line[4]
            remote_ip = ipre.search(remote_ip_str)
            octets = ipre.search(remote_ip_str).groups()

            # Check IP against manual prefixes to exclude from analysis
            match = False
            for prefix in ignore_prefix:
                l = len(prefix)
                if octets[0:l] == prefix:
                    match = True
            if match:
                excluded.add(remote_ip_str)
                break

            # Check IP against dynamic possible robots to exclude from analysis
            if remote_ip_str in possible_robots:
                break

            # Only analyze GET requests
            if not line[7] == 'WEBSITE.GET.OBJECT':
                continue

            # Only analyze responses that actually served a real page
            if not line[10][0] == '2':
                continue

            key = line[8]
            if key == '-':
                continue
            res_hits[key] += 1

            if first_in_file:
                # As properties of the client, these only get recorded
                # once per connection

                ip_hits[remote_ip_str] += 1

                referrer = line[16]
                if not referrer == '-':
                    referrer_hits[referrer] += 1
                agent = line[17]
                if not agent == '-':
                    agent_hits[agent.split(' ')[0]] += 1

                report = geoip.fetch(remote_ip_str)
                if not report is None:
                    if 'country_name' in report:
                        country_name = report['country_name']
                        if not country_name == '':
                            country_hits[report['country_name']] += 1
                    if 'region_name' in report:
                        region_name = report['region_name']
                        if not region_name == '':
                            region_hits[report['region_name']] += 1

                t = datetime.strptime(line[2][1:], '%d/%b/%Y:%H:%M:%S')
                date_hits[t.strftime("%Y-%m-%d")] += 1
                hour_hits[t.strftime("%H+0000'")] += 1
                time_hits.append(t.hour + t.minute / 60 + t.second / 3600)

                first_in_file = False

# Cache GeoIP data
geoip.dump()

# Basic report to console
print('IPs:', ip_hits.most_common(), '\n')
print('Countries:', country_hits.most_common(), '\n')
print('Regions:', region_hits.most_common(), '\n')
print('Dates:', sorted([(k, date_hits[k]) for k in date_hits]), '\n')
print('Hours:', sorted([(k, hour_hits[k]) for k in hour_hits]), '\n')
print('Referrers:', referrer_hits.most_common(), '\n')
print('User-Agents:', agent_hits.most_common(), '\n')
print('Resources:', res_hits.most_common(), '\n')
print('Redundant manual exclusions:', possible_robots.intersection(excluded))
print('\n')

# Interface to R for analysis and visualization
import rpy2.robjects as robjects
import rpy2.robjects.lib.ggplot2 as ggplot
from rpy2.robjects import IntVector
from rpy2.robjects import FloatVector
from rpy2.robjects import POSIXct
from rpy2.robjects.packages import importr
base = importr('base')
graphics = importr('graphics')
grdevices = importr('grDevices')
circular = importr('circular')
scales = importr('scales')

# Restructure IP hit count data
n = len(ip_hits)
ranks = IntVector(range(1, n+1))
counts = [c for (i, c) in ip_hits.most_common()]
counts_sum = sum(counts)
fracs_arr = [(c / counts_sum) for c in counts]
fracs = FloatVector(fracs_arr)

grdevices.png('analytics_out/ip_rank_abundance.png')
df = robjects.DataFrame({'rank': ranks, 'f': fracs})
pp = ggplot.ggplot(df) + \
    ggplot.aes_string(x = 'rank', y = 'f') + \
    ggplot.geom_point() + \
    ggplot.scale_y_log10(name = 'fraction of hits')
pp.plot()
grdevices.dev_off()

# Restructure date data
# TODO: make this less hacky
start = base.as_Date(min(date_hits.elements()))
end = base.as_Date(max(date_hits.elements()))
dates = base.seq(start, end, by = 'day')
hits_arr = [0] * len(dates)
for date in date_hits:
    hits_arr[int(base.as_Date(date)[0]) - int(start[0])] = date_hits[date]
hits = IntVector(hits_arr)

grdevices.png('analytics_out/hits_by_date.png')
df = robjects.DataFrame({'date': dates, 'hits': hits})
pp = ggplot.ggplot(df) + \
    ggplot.aes_string(x = 'date', y = 'hits') + \
    ggplot.geom_point() + \
    ggplot.geom_smooth(method = 'loess')
pp.plot()
grdevices.dev_off()

# Restructure hour data
hour_arr = []
hits_arr = []
for hour in hour_hits:
    hour_arr.append(datetime.min + timedelta(hours = int(hour[0:2])))
    hits_arr.append(hour_hits[hour])
hour = POSIXct(hour_arr)
hits = IntVector(hits_arr)

grdevices.png('analytics_out/hits_by_time.png')
df = robjects.DataFrame({'hour': POSIXct(hour), 'hits': IntVector(hits)})
pp = ggplot.ggplot(df) + \
    ggplot.aes_string(x = 'hour', y = 'hits') + \
    ggplot.scale_x_datetime(labels = scales.date_format('%H:%M UTC')) + \
    ggplot.geom_bar(stat = 'identity')
pp.plot()
grdevices.dev_off()

# Restructure circular hit time datax
time_hits_vec = FloatVector(time_hits)
hits_circ = circular.circular(time_hits_vec,
                              units = 'hours', template = 'clock24')
hits_density = circular.density_circular(hits_circ, bw = 100)

print('Von Mises fit for hits by time (hours past 00:00 UTC)')
hits_mle = circular.mle_vonmises(hits_circ)
mu = base.cbind(hits_mle.rx('mu'))[0][0]
mu_se = hits_mle.rx('se.mu')[0][0]
kappa = hits_mle.rx('kappa')[0][0]
kappa_se = hits_mle.rx('se.kappa')[0][0]
print('MLE: mu = %0.2f (%0.2f)  kappa = %0.2f (%0.2f)' %
      (mu, mu_se, kappa, kappa_se))
hits_bs = circular.mle_vonmises_bootstrap_ci(hits_circ)
print(hits_bs)

grdevices.png('analytics_out/hits_by_time_von_mises_point.png')
graphics.par(mar = [1,1,1,1])
circular.plot_circular(hits_circ, stack = True,
                       main = 'Hits by time (hours past 00:00 UTC)')
hits_mle = circular.mle_vonmises(hits_circ)
for reps in range(100):
    mle_samp = circular.rvonmises(n = len(hits_circ),
                                  mu = hits_mle.rx2('mu'),
                                  kappa = hits_mle.rx2('kappa'))
    mle_density = circular.density_circular(mle_samp, bw = 100)
    graphics.lines(mle_density, offset = 0.5, shrink = 0.5,
                   col = grdevices.rgb(1, 0, 0, 0.1))
graphics.lines(hits_density, offset = 0.5, shrink = 0.5, col = 'blue')
grdevices.dev_off()

def moment_plot(c, col):
    m_clock = circular.mean_circular(c)
    m_circ = circular.conversion_circular(m_clock, units = 'radians',
                                          rotation = 'counter', zero = 0)
    m = base.cbind(m_circ)[0]
    v = base.cbind(circular.var_circular(c))[0]
    graphics.points((1 - v) * cos(m), (1 - v) * sin(m), col = col)

grdevices.png('analytics_out/hits_by_time_von_mises_bs.png')
graphics.par(mar = [1,1,1,1])
circular.rose_diag(hits_bs.rx('mu'), bins = 24,
                   main = 'Hits by time (hours past 00:00 UTC)')
for reps in range(100):
    hits_circ_bs = base.sample(hits_circ, replace = True)
    bs_density = circular.density_circular(hits_circ_bs, bw = 100)
    graphics.lines(bs_density, offset = 0.5, shrink = 0.5,
                   col = grdevices.rgb(0, 0, 0, 0.1))
    moment_plot(hits_circ_bs, grdevices.rgb(1, 0, 0, 0.1))
    hits_mle = circular.mle_vonmises(hits_circ_bs)
    mle_samp = circular.rvonmises(n = len(hits_circ),
                                  mu = hits_mle.rx2('mu'),
                                  kappa = hits_mle.rx2('kappa'))
    mle_density = circular.density_circular(mle_samp, bw = 100)
    graphics.lines(mle_density, offset = 0.5, shrink = 0.5,
                   col = grdevices.rgb(1, 0, 0, 0.1))
graphics.lines(hits_density, offset = 0.5, shrink = 0.5, col = 'blue')
moment_plot(hits_circ, 'blue')
grdevices.dev_off()
<commit_msg>Rank-abundance plot for all IPs as well as non-bot ones<commit_after>#!/usr/bin/env python

import csv
import re
import os
from datetime import datetime, timedelta
from collections import Counter, defaultdict
import http.client
import json
from math import cos, sin, pi

ipre = re.compile('(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})')

# IP ranges to exclude that aren't caught automatically for bot-like
# behavior
ignore_prefix = [tuple(p.split('.'))
                 for p in ['150.70', '202.46', '75.130.244.15', '64.233',
                           '96.47.226.21', '178.63.97.34', '89.31.57.5',
                           '85.159.113.228', '148.251.77.34',
                           '178.217.187.39']]

# Tests for bot-like behavior
def suspicious_key(key):
    return (key == 'robots.txt' or 'googled' in key)
def suspicious_referrer(referrer):
    return ('semalt' in referrer or 'buttons' in referrer or
            'best-seo' in referrer)
def suspicious_agent(agent):
    return ('bot' in agent or 'Bot' in agent or 'spider' in agent or
            'Google favicon' in agent or 'Disqus' in agent or
            'prismatic' in agent)

log_files = os.listdir('logs/')

ip_hits = Counter()
ip_hits_all = Counter()
country_hits = Counter()
region_hits = Counter()
date_hits = Counter()
hour_hits = Counter()
time_hits = []
res_hits = Counter()
referrer_hits = Counter()
agent_hits = Counter()
possible_robots = set()
excluded = set()

class GeoIP():
    def __init__(self):
        self.conn = http.client.HTTPConnection('freegeoip.net', timeout = 10)

        try:
            with open('cache/geoip.json', 'r') as infile:
                self.cache = json.load(infile)
        except:
            self.cache = {}

    def fetch(self, ip):
        if ip in self.cache:
            return self.cache[ip]

        try:
            self.conn.request('GET', '/json/' + ip)

            response = self.conn.getresponse()
            print('Querying GeoIP:', response.status, response.reason)

            data_serialized = response.read().decode('utf-8')
            data = json.loads(data_serialized)

            self.cache[ip] = data

            return data
        except:
            return None

    def dump(self):
        with open('cache/geoip.json', 'w') as outfile:
            json.dump(self.cache, outfile)

geoip = GeoIP()

# First pass to detect non-human access
for infile in log_files:
    with open(os.path.join('logs', infile), 'r') as log:
        log_reader = csv.reader(log, delimiter = ' ', quotechar = '"')

        for line in log_reader:
            remote_ip_str = line[4]
            ip_hits_all[remote_ip_str] += 1

            if suspicious_key(line[8]):
                possible_robots.add(remote_ip_str)
            if suspicious_referrer(line[16]):
                possible_robots.add(remote_ip_str)
            if suspicious_agent(line[17]):
                possible_robots.add(remote_ip_str)

for infile in log_files:
    with open(os.path.join('logs', infile), 'r') as log:
        log_reader = csv.reader(log, delimiter = ' ', quotechar = '"')

        first_in_file = True
        for line in log_reader:
            # Parse remote IP so it can be checked against prefixes
            remote_ip_str = line[4]
            remote_ip = ipre.search(remote_ip_str)
            octets = ipre.search(remote_ip_str).groups()

            # Check IP against manual prefixes to exclude from analysis
            match = False
            for prefix in ignore_prefix:
                l = len(prefix)
                if octets[0:l] == prefix:
                    match = True
            if match:
                excluded.add(remote_ip_str)
                break

            # Check IP against dynamic possible robots to exclude from analysis
            if remote_ip_str in possible_robots:
                break

            # Only analyze GET requests
            if not line[7] == 'WEBSITE.GET.OBJECT':
                continue

            # Only analyze responses that actually served a real page
            if not line[10][0] == '2':
                continue

            key = line[8]
            if key == '-':
                continue
            res_hits[key] += 1

            if first_in_file:
                # As properties of the client, these only get recorded
                # once per connection

                ip_hits[remote_ip_str] += 1

                referrer = line[16]
                if not referrer == '-':
                    referrer_hits[referrer] += 1
                agent = line[17]
                if not agent == '-':
                    agent_hits[agent.split(' ')[0]] += 1

                report = geoip.fetch(remote_ip_str)
                if not report is None:
                    if 'country_name' in report:
                        country_name = report['country_name']
                        if not country_name == '':
                            country_hits[report['country_name']] += 1
                    if 'region_name' in report:
                        region_name = report['region_name']
                        if not region_name == '':
                            region_hits[report['region_name']] += 1

                t = datetime.strptime(line[2][1:], '%d/%b/%Y:%H:%M:%S')
                date_hits[t.strftime("%Y-%m-%d")] += 1
                hour_hits[t.strftime("%H+0000'")] += 1
                time_hits.append(t.hour + t.minute / 60 + t.second / 3600)

                first_in_file = False

# Cache GeoIP data
geoip.dump()

# Basic report to console
print('IPs:', ip_hits.most_common(), '\n')
print('Countries:', country_hits.most_common(), '\n')
print('Regions:', region_hits.most_common(), '\n')
print('Dates:', sorted([(k, date_hits[k]) for k in date_hits]), '\n')
print('Hours:', sorted([(k, hour_hits[k]) for k in hour_hits]), '\n')
print('Referrers:', referrer_hits.most_common(), '\n')
print('User-Agents:', agent_hits.most_common(), '\n')
print('Resources:', res_hits.most_common(), '\n')
print('Redundant manual exclusions:', possible_robots.intersection(excluded))
print('\n')

# Interface to R for analysis and visualization
import rpy2.robjects as robjects
import rpy2.robjects.lib.ggplot2 as ggplot
from rpy2.robjects import IntVector
from rpy2.robjects import FloatVector
from rpy2.robjects import POSIXct
from rpy2.robjects.packages import importr
base = importr('base')
graphics = importr('graphics')
grdevices = importr('grDevices')
circular = importr('circular')
scales = importr('scales')

# Restructure IP hit count data
def rank_abundance(counter):
    n = len(counter)
    ranks = IntVector(range(1, n+1))
    counts = [c for (i, c) in counter.most_common()]
    counts_sum = sum(counts)
    fracs_arr = [(c / counts_sum) for c in counts]
    fracs = FloatVector(fracs_arr)

    return ranks, fracs

grdevices.png('analytics_out/ip_rank_abundance.png')
ranks, fracs = rank_abundance(ip_hits)
df = robjects.DataFrame({'rank': ranks, 'f': fracs})
pp = ggplot.ggplot(df) + \
    ggplot.aes_string(x = 'rank', y = 'f') + \
    ggplot.geom_point() + \
    ggplot.scale_y_log10(name = 'fraction of hits')
pp.plot()
grdevices.dev_off()

grdevices.png('analytics_out/ip_all_rank_abundance.png')
ranks, fracs = rank_abundance(ip_hits_all)
df = robjects.DataFrame({'rank': ranks, 'f': fracs})
pp = ggplot.ggplot(df) + \
    ggplot.aes_string(x = 'rank', y = 'f') + \
    ggplot.geom_point() + \
    ggplot.scale_y_log10(name = 'fraction of hits')
pp.plot()
grdevices.dev_off()

# Restructure date data
# TODO: make this less hacky
start = base.as_Date(min(date_hits.elements()))
end = base.as_Date(max(date_hits.elements()))
dates = base.seq(start, end, by = 'day')
hits_arr = [0] * len(dates)
for date in date_hits:
    hits_arr[int(base.as_Date(date)[0]) - int(start[0])] = date_hits[date]
hits = IntVector(hits_arr)

grdevices.png('analytics_out/hits_by_date.png')
df = robjects.DataFrame({'date': dates, 'hits': hits})
pp = ggplot.ggplot(df) + \
    ggplot.aes_string(x = 'date', y = 'hits') + \
    ggplot.geom_point() + \
    ggplot.geom_smooth(method = 'loess')
pp.plot()
grdevices.dev_off()

# Restructure hour data
hour_arr = []
hits_arr = []
for hour in hour_hits:
    hour_arr.append(datetime.min + timedelta(hours = int(hour[0:2])))
    hits_arr.append(hour_hits[hour])
hour = POSIXct(hour_arr)
hits = IntVector(hits_arr)

grdevices.png('analytics_out/hits_by_time.png')
df = robjects.DataFrame({'hour': POSIXct(hour), 'hits': IntVector(hits)})
pp = ggplot.ggplot(df) + \
    ggplot.aes_string(x = 'hour', y = 'hits') + \
    ggplot.scale_x_datetime(labels = scales.date_format('%H:%M UTC')) + \
    ggplot.geom_bar(stat = 'identity')
pp.plot()
grdevices.dev_off()

# Restructure circular hit time datax
time_hits_vec = FloatVector(time_hits)
hits_circ = circular.circular(time_hits_vec,
                              units = 'hours', template = 'clock24')
hits_density = circular.density_circular(hits_circ, bw = 100)

print('Von Mises fit for hits by time (hours past 00:00 UTC)')
hits_mle = circular.mle_vonmises(hits_circ)
mu = base.cbind(hits_mle.rx('mu'))[0][0]
mu_se = hits_mle.rx('se.mu')[0][0]
kappa = hits_mle.rx('kappa')[0][0]
kappa_se = hits_mle.rx('se.kappa')[0][0]
print('MLE: mu = %0.2f (%0.2f)  kappa = %0.2f (%0.2f)' %
      (mu, mu_se, kappa, kappa_se))
hits_bs = circular.mle_vonmises_bootstrap_ci(hits_circ)
print(hits_bs)

grdevices.png('analytics_out/hits_by_time_von_mises_point.png')
graphics.par(mar = [1,1,1,1])
circular.plot_circular(hits_circ, stack = True,
                       main = 'Hits by time (hours past 00:00 UTC)')
hits_mle = circular.mle_vonmises(hits_circ)
for reps in range(100):
    mle_samp = circular.rvonmises(n = len(hits_circ),
                                  mu = hits_mle.rx2('mu'),
                                  kappa = hits_mle.rx2('kappa'))
    mle_density = circular.density_circular(mle_samp, bw = 100)
    graphics.lines(mle_density, offset = 0.5, shrink = 0.5,
                   col = grdevices.rgb(1, 0, 0, 0.1))
graphics.lines(hits_density, offset = 0.5, shrink = 0.5, col = 'blue')
grdevices.dev_off()

def moment_plot(c, col):
    m_clock = circular.mean_circular(c)
    m_circ = circular.conversion_circular(m_clock, units = 'radians',
                                          rotation = 'counter', zero = 0)
    m = base.cbind(m_circ)[0]
    v = base.cbind(circular.var_circular(c))[0]
    graphics.points((1 - v) * cos(m), (1 - v) * sin(m), col = col)

grdevices.png('analytics_out/hits_by_time_von_mises_bs.png')
graphics.par(mar = [1,1,1,1])
circular.rose_diag(hits_bs.rx('mu'), bins = 24,
                   main = 'Hits by time (hours past 00:00 UTC)')
for reps in range(100):
    hits_circ_bs = base.sample(hits_circ, replace = True)
    bs_density = circular.density_circular(hits_circ_bs, bw = 100)
    graphics.lines(bs_density, offset = 0.5, shrink = 0.5,
                   col = grdevices.rgb(0, 0, 0, 0.1))
    moment_plot(hits_circ_bs, grdevices.rgb(1, 0, 0, 0.1))
    hits_mle = circular.mle_vonmises(hits_circ_bs)
    mle_samp = circular.rvonmises(n = len(hits_circ),
                                  mu = hits_mle.rx2('mu'),
                                  kappa = hits_mle.rx2('kappa'))
    mle_density = circular.density_circular(mle_samp, bw = 100)
    graphics.lines(mle_density, offset = 0.5, shrink = 0.5,
                   col = grdevices.rgb(1, 0, 0, 0.1))
graphics.lines(hits_density, offset = 0.5, shrink = 0.5, col = 'blue')
moment_plot(hits_circ, 'blue')
grdevices.dev_off()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
image_scraper is used to find all images from a web page
"""
# @author chengdujin
# @contact chengdujin@gmail.com
# @created Jul. 23, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from BeautifulSoup import BeautifulSoup
from cStringIO import StringIO
from data_processor import thumbnail
import Image
import os
import re
import urllib2
import urlparse

# CONSTANTS
from config import IMAGES_LOCAL_DIR
from config import IMAGES_PUBLIC_DIR
from config import MIN_IMAGE_SIZE
from config import TRANSCODED_LOCAL_DIR
from config import UCK_TIMEOUT

if not os.path.exists(IMAGES_LOCAL_DIR):
    os.mkdir(IMAGES_LOCAL_DIR)

def _link_process(link):
    """
    get rid of cdn prefix
    """
    link = link.replace("\/", "/")
    image_url_complex = urllib2.unquote(link.strip())
    if image_url_complex:
        # as the name could be http://xxx.com/yyy--http://zzz.jpg
        # or http://xxx.com/yyy--https://zzz.jpg
        last_http_index = image_url_complex.rfind('http')
        image_url = image_url_complex[last_http_index:]
        # response is the signal of a valid image
        response = None
        try:
            response = urllib2.urlopen(image_url, timeout=UCK_TIMEOUT)
        except urllib2.URLError as k:
            path = re.split('https?://?', image_url)[-1]
            scheme = urlparse.urlparse(image_url).scheme
            image_url = '%s://%s' % (scheme, path)
            try:
                response = urllib2.urlopen(image_url, timeout=UCK_TIMEOUT)
            except urllib2.URLError as k:
                pass
            except Exception as k:
                print '[image_helper._link_process]', str(k)
        if response:
            return image_url

def find_image(link=None):
    """
    find an image from the link
    """
    if not link:
        return None

    link = _link_process(link)
    image_normalized = normalize(link)
    return image_normalized[0] if image_normalized else None

def find_images(content=None):
    """
    find out all images from content and its size info
    """
    if not content:
        return None

    # determine the type of content
    if isinstance(content, str) and content.startswith(TRANSCODED_LOCAL_DIR):
        # then its a file
        f = open(content, 'r')
        content = f.read()

    soup = BeautifulSoup(content.decode('utf-8', 'ignore'))
    images_normalized = []
    images = soup.findAll('img')
    for image in images:
        if image.get('src'):
            image_normalized = find_image(image.get('src'))
            if image_normalized:
                images_normalized.append(image_normalized)
    return images_normalized

def find_biggest_image(images=None):
    """
    find the biggest in resolution from a pile of images
    """
    if not images:
        return None

    biggest = None
    for image in images:
        resolution_max = MIN_IMAGE_SIZE[0] * MIN_IMAGE_SIZE[1]
        resolution_image = int(image['width']) * int(image['height'])
        if resolution_image > resolution_max:
            biggest = image
            resolution_max = resolution_image
    return biggest

def dedupe_images(images):
    """
    remove same images
    image: {'url':xxx, 'width':yyy, 'height':zzz}
    images = [image, image, image]
    """
    if not images:
        return None

    image_urls = []

    def _exists(image):
        """
        return boolean if image exists in list image_urls
        """
        exists = image['url'] in image_urls
        if not exists:
            image_urls.append(image['url'])
            return False
        else:
            return True

    return filter(lambda x: not _exists(x), images)

# TODO: boundary checker
def scale_image(image=None, size_expected=MIN_IMAGE_SIZE,
                resize_by_width=True, crop_by_center=True, relative_path=None):
    """
    resize an image as requested
    resize_by_width: resize image according to its width(True)/height(False)
    crop_by_center: crop image from its center(True) or by point(0, 0)(False)
    """
    if not image or not size_expected or not relative_path:
        return None, None

    width = int(image['width'])
    height = int(image['height'])
    width_expected = size_expected[0]
    height_expected = size_expected[1]

    if width >= width_expected and height >= height_expected:
        if resize_by_width:
            height_new = width_expected * height / width
            width_new = width_expected
        else:
            width_new = height_expected * width / height
            height_new = height_expected

        # larger and equal than is important here
        if width_new >= width_expected and height_new >= height_expected:
            # resize
            size_new = width_new, height_new
            image_data = None
            try:
                image_data = Image.open(StringIO(urllib2.urlopen(image['url'], timeout=UCK_TIMEOUT).read()))
            except Exception as k:
                print '[image_helper.scale_image]', str(k)
                raise k
            image_data.thumbnail(size_new, Image.ANTIALIAS)
            # crop
            if crop_by_center:
                left = (width_new - width_expected) / 2
                top = (height_new - height_expected) / 2
                right = (width_new + width_expected) / 2
                bottom = (height_new + height_expected) / 2
                image_cropped = image_data.crop((left, top, right, bottom))
            else:
                left = 0
                top = 0
                right = width_expected
                bottom = height_expected
                image_cropped = image_data.crop((left, top, right, bottom))
            # storing
            if image_cropped:
                image_web_path = '%s%s.jpg' % (
                    IMAGES_PUBLIC_DIR, relative_path)
                image_local_path = '%s%s.jpg' % (
                    IMAGES_LOCAL_DIR, relative_path)
                image_cropped = image_cropped.convert('RGB')
                image_cropped.save(image_local_path, 'JPEG')
                return {'url': image_web_path, 'width': width_expected, 'height': height_expected}, {'url': image_local_path, 'width': width_expected, 'height': height_expected}
            else:
                return None, None
        else:
            return scale_image(image, size_expected, not resize_by_width, crop_by_center, relative_path)
    else:
        return None, None

def normalize(images):
    """
    for list of images, remove images that don't match with MIN_IMAGE_SIZE;
    for an image, return None if it doesn't matches with MIN_IMAGE_SIZE
    """
    def _check_image(image):
        """
        check an image if it matches with MIN_IMAGE_SIZE
        """
        if not image:
            raise Exception('[image_helper.normalize] ERROR: Method not well formed!')

        try:
            if 'url' in image:
                image_url = image['url']
                if thumbnail.is_valid_image(image_url):
                    return image
            else:
                if thumbnail.is_valid_image(image):
                    width, height = thumbnail.get_image_size(image)
                    return {'url': image, 'width': width, 'height': height}
            return None
        except IOError as k:
            print '[image_helper.normalize]', (k)
            return None

    if isinstance(images, str) or isinstance(images, unicode):
        image = _check_image(images)
        return [image] if image else None
    elif isinstance(images, list):
        images_new = []
        for image in images:
            image_new = _check_image(image)
            if image_new:
                images_new.append(image_new)
        return images_new if images_new else None
<commit_msg>added hparser<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
image_scraper is used to find all images from a web page
"""
# @author chengdujin
# @contact chengdujin@gmail.com
# @created Jul. 23, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from BeautifulSoup import BeautifulSoup
from config import hparser
from cStringIO import StringIO
from data_processor import thumbnail
import Image
import os
import re
import urllib2
import urlparse

# CONSTANTS
from config import IMAGES_LOCAL_DIR
from config import IMAGES_PUBLIC_DIR
from config import MIN_IMAGE_SIZE
from config import TRANSCODED_LOCAL_DIR
from config import UCK_TIMEOUT

if not os.path.exists(IMAGES_LOCAL_DIR):
    os.mkdir(IMAGES_LOCAL_DIR)

def _link_process(link):
    """
    get rid of cdn prefix
    """
    link = link.replace("\/", "/")
    image_url_complex = urllib2.unquote(hparser.unescape(link.strip()))
    if image_url_complex:
        # as the name could be http://xxx.com/yyy--http://zzz.jpg
        # or http://xxx.com/yyy--https://zzz.jpg
        last_http_index = image_url_complex.rfind('http')
        image_url = image_url_complex[last_http_index:]
        # response is the signal of a valid image
        response = None
        try:
            response = urllib2.urlopen(image_url, timeout=UCK_TIMEOUT)
        except urllib2.URLError as k:
            path = re.split('https?://?', image_url)[-1]
            scheme = urlparse.urlparse(image_url).scheme
            image_url = '%s://%s' % (scheme, path)
            try:
                response = urllib2.urlopen(image_url, timeout=UCK_TIMEOUT)
            except urllib2.URLError as k:
                pass
            except Exception as k:
                print '[image_helper._link_process]', str(k)
        if response:
            return image_url

def find_image(link=None):
    """
    find an image from the link
    """
    if not link:
        return None

    link = _link_process(link)
    image_normalized = normalize(link)
    return image_normalized[0] if image_normalized else None

def find_images(content=None):
    """
    find out all images from content and its size info
    """
    if not content:
        return None

    # determine the type of content
    if isinstance(content, str) and content.startswith(TRANSCODED_LOCAL_DIR):
        # then its a file
        f = open(content, 'r')
        content = f.read()

    soup = BeautifulSoup(content.decode('utf-8', 'ignore'))
    images_normalized = []
    images = soup.findAll('img')
    for image in images:
        if image.get('src'):
            image_normalized = find_image(image.get('src'))
            if image_normalized:
                images_normalized.append(image_normalized)
    return images_normalized

def find_biggest_image(images=None):
    """
    find the biggest in resolution from a pile of images
    """
    if not images:
        return None

    biggest = None
    for image in images:
        resolution_max = MIN_IMAGE_SIZE[0] * MIN_IMAGE_SIZE[1]
        resolution_image = int(image['width']) * int(image['height'])
        if resolution_image > resolution_max:
            biggest = image
            resolution_max = resolution_image
    return biggest

def dedupe_images(images):
    """
    remove same images
    image: {'url':xxx, 'width':yyy, 'height':zzz}
    images = [image, image, image]
    """
    if not images:
        return None

    image_urls = []

    def _exists(image):
        """
        return boolean if image exists in list image_urls
        """
        exists = image['url'] in image_urls
        if not exists:
            image_urls.append(image['url'])
            return False
        else:
            return True

    return filter(lambda x: not _exists(x), images)

# TODO: boundary checker
def scale_image(image=None, size_expected=MIN_IMAGE_SIZE,
                resize_by_width=True, crop_by_center=True, relative_path=None):
    """
    resize an image as requested
    resize_by_width: resize image according to its width(True)/height(False)
    crop_by_center: crop image from its center(True) or by point(0, 0)(False)
    """
    if not image or not size_expected or not relative_path:
        return None, None

    width = int(image['width'])
    height = int(image['height'])
    width_expected = size_expected[0]
    height_expected = size_expected[1]

    if width >= width_expected and height >= height_expected:
        if resize_by_width:
            height_new = width_expected * height / width
            width_new = width_expected
        else:
            width_new = height_expected * width / height
            height_new = height_expected

        # larger and equal than is important here
        if width_new >= width_expected and height_new >= height_expected:
            # resize
            size_new = width_new, height_new
            image_data = None
            try:
                image_data = Image.open(StringIO(urllib2.urlopen(image['url'], timeout=UCK_TIMEOUT).read()))
            except Exception as k:
                print '[image_helper.scale_image]', str(k)
                raise k
            image_data.thumbnail(size_new, Image.ANTIALIAS)
            # crop
            if crop_by_center:
                left = (width_new - width_expected) / 2
                top = (height_new - height_expected) / 2
                right = (width_new + width_expected) / 2
                bottom = (height_new + height_expected) / 2
                image_cropped = image_data.crop((left, top, right, bottom))
            else:
                left = 0
                top = 0
                right = width_expected
                bottom = height_expected
                image_cropped = image_data.crop((left, top, right, bottom))
            # storing
            if image_cropped:
                image_web_path = '%s%s.jpg' % (
                    IMAGES_PUBLIC_DIR, relative_path)
                image_local_path = '%s%s.jpg' % (
                    IMAGES_LOCAL_DIR, relative_path)
                image_cropped = image_cropped.convert('RGB')
                image_cropped.save(image_local_path, 'JPEG')
                return {'url': image_web_path, 'width': width_expected, 'height': height_expected}, {'url': image_local_path, 'width': width_expected, 'height': height_expected}
            else:
                return None, None
        else:
            return scale_image(image, size_expected, not resize_by_width, crop_by_center, relative_path)
    else:
        return None, None

def normalize(images):
    """
    for list of images, remove images that don't match with MIN_IMAGE_SIZE;
    for an image, return None if it doesn't matches with MIN_IMAGE_SIZE
    """
    def _check_image(image):
        """
        check an image if it matches with MIN_IMAGE_SIZE
        """
        if not image:
            raise Exception('[image_helper.normalize] ERROR: Method not well formed!')

        try:
            if 'url' in image:
                image_url = image['url']
                if thumbnail.is_valid_image(image_url):
                    return image
            else:
                if thumbnail.is_valid_image(image):
                    width, height = thumbnail.get_image_size(image)
                    return {'url': image, 'width': width, 'height': height}
            return None
        except IOError as k:
            print '[image_helper.normalize]', (k)
            return None

    if isinstance(images, str) or isinstance(images, unicode):
        image = _check_image(images)
        return [image] if image else None
    elif isinstance(images, list):
        images_new = []
        for image in images:
            image_new = _check_image(image)
            if image_new:
                images_new.append(image_new)
        return images_new if images_new else None
<|endoftext|>


<commit_before>from __future__ import absolute_import

import itertools
import os
import re
import traceback
from typing import (
    Any,
    AnyStr,
    Dict,
    List,
    MutableMapping,
    MutableSequence,
    Optional,
    Pattern,
    Tuple,
    Type,
    Union,
)

import six
from future.utils import raise_from
from typing_extensions import Text  # pylint: disable=unused-import

import ruamel.yaml
from ruamel.yaml.comments import CommentedBase, CommentedMap, CommentedSeq

# move to a regular typing import when Python 3.3-3.6 is no longer supported

lineno_re = re.compile(u"^(.*?:[0-9]+:[0-9]+: )(( *)(.*))")

def regex_chunk(lines, regex):
    # type: (List[str], Pattern[str]) -> List[List[str]]
    lst = list(itertools.dropwhile(lambda x: not regex.match(x), lines))
    arr = []
    while lst:
        ret = [lst[0]] + list(
            itertools.takewhile(lambda x: not regex.match(x), lst[1:])
        )
        arr.append(ret)
        lst = list(itertools.dropwhile(lambda x: not regex.match(x), lst[1:]))
    return arr

def chunk_messages(message):  # type: (str) -> List[Tuple[int, str]]
    file_regex = re.compile(r"^(.+:\d+:\d+:)(\s+)(.+)$")
    item_regex = re.compile(r"^\s*\*\s+")
    arr = []
    for chun in regex_chunk(message.splitlines(), file_regex):
        fst = chun[0]
        mat = file_regex.match(fst)
        if mat:
            place = mat.group(1)
            indent = len(mat.group(2))

            lst = [mat.group(3)] + chun[1:]
            if [x for x in lst if item_regex.match(x)]:
                for item in regex_chunk(lst, item_regex):
                    msg = re.sub(item_regex, "", "\n".join(item))
                    arr.append((indent, place + " " + re.sub(r"[\n\s]+", " ", msg)))
            else:
                msg = re.sub(item_regex, "", "\n".join(lst))
                arr.append((indent, place + " " + re.sub(r"[\n\s]+", " ", msg)))
    return arr

def to_one_line_messages(message):  # type: (str) -> str
    ret = []
    max_elem = (0, "")
    for (indent, msg) in chunk_messages(message):
        if indent > max_elem[0]:
            max_elem = (indent, msg)
        else:
            ret.append(max_elem[1])
            max_elem = (indent, msg)
    ret.append(max_elem[1])
    return "\n".join(ret)

def reformat_yaml_exception_message(message):  # type: (str) -> str
    line_regex = re.compile(r'^\s+in "(.+)", line (\d+), column (\d+)$')
    fname_regex = re.compile(r"^file://" + re.escape(os.getcwd()) + "/")
    msgs = message.splitlines()
    ret = []

    if len(msgs) == 3:
        msgs = msgs[1:]
        nblanks = 0
    elif len(msgs) == 4:
        c_msg = msgs[0]
        match = line_regex.match(msgs[1])
        if match:
            c_file, c_line, c_column = match.groups()
            c_file = re.sub(fname_regex, "", c_file)
            ret.append("{}:{}:{}: {}".format(c_file, c_line, c_column, c_msg))

        msgs = msgs[2:]
        nblanks = 2

    p_msg = msgs[0]
    match = line_regex.match(msgs[1])
    if match:
        p_file, p_line, p_column = match.groups()
        p_file = re.sub(fname_regex, "", p_file)
        ret.append(
            "{}:{}:{}:{} {}".format(p_file, p_line, p_column, " " * nblanks, p_msg)
        )
    return "\n".join(ret)

def _add_lc_filename(
    r, source
):  # type: (ruamel.yaml.comments.CommentedBase, AnyStr) -> None
    if isinstance(r, ruamel.yaml.comments.CommentedBase):
        r.lc.filename = source
    if isinstance(r, MutableSequence):
        for d in r:
            _add_lc_filename(d, source)
    elif isinstance(r, MutableMapping):
        for d in six.itervalues(r):
            _add_lc_filename(d, source)

def relname(source):  # type: (Text) -> Text
    if source.startswith("file://"):
        source = source[7:]
        source = os.path.relpath(source)
    return source

def add_lc_filename(
    r, source
):  # type: (ruamel.yaml.comments.CommentedBase, Text) -> None
    _add_lc_filename(r, relname(source))

def reflow(text, maxline, shift=""):  # type: (Text, int, Text) -> Text
    if maxline < 20:
        maxline = 20
    if len(text) > maxline:
        sp = text.rfind(" ", 0, maxline)
        if sp < 1:
            sp = text.find(" ", sp + 1)
            if sp == -1:
                sp = len(text)
        if sp < len(text):
            return "{}\n{}{}".format(
                text[0:sp], shift, reflow(text[sp + 1 :], maxline, shift)
            )
    return text

def indent(
    v, nolead=False, shift=u"  ", bullet=u"  "
):  # type: (Text, bool, Text, Text) -> Text
    if nolead:
        return v.splitlines()[0] + u"\n".join([shift + l for l in v.splitlines()[1:]])
    else:

        def lineno(i, l):  # type: (int, Text) -> Text
            r = lineno_re.match(l)
            if r is not None:
                return r.group(1) + (bullet if i == 0 else shift) + r.group(2)
            else:
                return (bullet if i == 0 else shift) + l

        return u"\n".join([lineno(i, l) for i, l in enumerate(v.splitlines())])

def bullets(textlist, bul):  # type: (List[Text], Text) -> Text
    if len(textlist) == 1:
        return textlist[0]
    else:
        return "\n".join(indent(t, bullet=bul) for t in textlist)

def strip_dup_lineno(text, maxline=None):  # type: (Text, Optional[int]) -> Text
    if maxline is None:
        maxline = int(os.environ.get("COLUMNS", "100"))
    pre = None
    msg = []
    maxno = 0
    for l in text.splitlines():
        g = lineno_re.match(l)
        if not g:
            continue
        maxno = max(maxno, len(g.group(1)))

    for l in text.splitlines():
        g = lineno_re.match(l)
        if not g:
            msg.append(l)
            continue
        if g.group(1) != pre:
            shift = maxno + len(g.group(3))
            g2 = reflow(g.group(2), maxline - shift, " " * shift)
            pre = g.group(1)
            msg.append(pre + " " * (maxno - len(g.group(1))) + g2)
        else:
            g2 = reflow(g.group(2), maxline - maxno, " " * (maxno + len(g.group(3))))
            msg.append(" " * maxno + g2)
    return "\n".join(msg)

def cmap(
    d,  # type: Union[int, float, str, Text, Dict[Text, Any], List[Dict[Text, Any]]]
    lc=None,  # type: Optional[List[int]]
    fn=None,  # type: Optional[Text]
):  # type: (...) -> Union[int, float, str, Text, CommentedMap, CommentedSeq]
    if lc is None:
        lc = [0, 0, 0, 0]
    if fn is None:
        fn = "test"

    if isinstance(d, CommentedMap):
        fn = d.lc.filename if hasattr(d.lc, "filename") else fn
        for k, v in six.iteritems(d):
            if d.lc.data is not None and k in d.lc.data:
                d[k] = cmap(v, lc=d.lc.data[k], fn=fn)
            else:
                d[k] = cmap(v, lc, fn=fn)
        return d
    if isinstance(d, CommentedSeq):
        fn = d.lc.filename if hasattr(d.lc, "filename") else fn
        for k2, v2 in enumerate(d):
            if d.lc.data is not None and k2 in d.lc.data:
                d[k2] = cmap(v2, lc=d.lc.data[k2], fn=fn)
            else:
                d[k2] = cmap(v2, lc, fn=fn)
        return d
    if isinstance(d, MutableMapping):
        cm = CommentedMap()
        for k in sorted(d.keys()):
            v = d[k]
            if isinstance(v, CommentedBase):
                uselc = [v.lc.line, v.lc.col, v.lc.line, v.lc.col]
                vfn = v.lc.filename if hasattr(v.lc, "filename") else fn
            else:
                uselc = lc
                vfn = fn
            cm[k] = cmap(v, lc=uselc, fn=vfn)
            cm.lc.add_kv_line_col(k, uselc)
            cm.lc.filename = fn
        return cm
    if isinstance(d, MutableSequence):
        cs = CommentedSeq()
        for k3, v3 in enumerate(d):
            if isinstance(v3, CommentedBase):
                uselc = [v3.lc.line, v3.lc.col, v3.lc.line, v3.lc.col]
                vfn = v3.lc.filename if hasattr(v3.lc, "filename") else fn
            else:
                uselc = lc
                vfn = fn
            cs.append(cmap(v3, lc=uselc, fn=vfn))
            cs.lc.add_kv_line_col(k3, uselc)
            cs.lc.filename = fn
        return cs
    else:
        return d

class SourceLine(object):
    def __init__(
        self,
        item,  # type: Any
        key=None,  # type: Optional[Any]
        raise_type=six.text_type,  # type: Union[Type[six.text_type], Type[Exception]]
        include_traceback=False,  # type: bool
    ):  # type: (...) -> None
        self.item = item
        self.key = key
        self.raise_type = raise_type
        self.include_traceback = include_traceback

    def __enter__(self):  # type: () -> SourceLine
        return self

    def __exit__(
        self,
        exc_type,  # type: Any
        exc_value,  # type: Any
        tb,  # type: Any
    ):  # type: (...) -> None
        if not exc_value:
            return
        if self.include_traceback and six.PY2:
            # Python2 doesn't actually have chained exceptions, so
            # fake it by injecting the backtrace into the message.
            raise_from(
                self.makeError(
                    "\n".join(traceback.format_exception(exc_type, exc_value, tb))
                ),
                exc_value,
            )
        else:
            raise_from(self.makeError(six.text_type(exc_value)), exc_value)

    def file(self):  # type: () -> Text
        return Text(self.item.lc.filename) if hasattr(self.item.lc, "filename") else ""

    def line(self):  # type: () -> int
        if (
            self.key is None
            or self.item.lc.data is None
            or self.key not in self.item.lc.data
        ):
            return (self.item.lc.line or 0) + 1
        else:
            return (self.item.lc.data[self.key][0] or 0) + 1

    def column(self):  # type: () -> int
        if (
            self.key is None
            or self.item.lc.data is None
            or self.key not in self.item.lc.data
        ):
            return (self.item.lc.col or 0) + 1
        else:
            return (self.item.lc.data[self.key][1] or 0) + 1

    def makeLead(self):  # type: () -> Text
        return "{}:{}:{}".format(self.file(), self.line(), self.column())

    def makeError(self, msg):  # type: (Text) -> Any
        if not isinstance(self.item, ruamel.yaml.comments.CommentedBase):
            return self.raise_type(msg)
        errs = []
        lead = self.makeLead()
        for m in msg.splitlines():
            if bool(lineno_re.match(m)):
                errs.append(m)
            else:
                errs.append("{} {}".format(lead, m))
        return self.raise_type("\n".join(errs))
<commit_msg>Add missing colon<commit_after>from __future__ import absolute_import

import itertools
import os
import re
import traceback
from typing import (
    Any,
    AnyStr,
    Dict,
    List,
    MutableMapping,
    MutableSequence,
    Optional,
    Pattern,
    Tuple,
    Type,
    Union,
)

import six
from future.utils import raise_from
from typing_extensions import Text  # pylint: disable=unused-import

import ruamel.yaml
from ruamel.yaml.comments import CommentedBase, CommentedMap, CommentedSeq

# move to a regular typing import when Python 3.3-3.6 is no longer supported

lineno_re = re.compile(u"^(.*?:[0-9]+:[0-9]+: )(( *)(.*))")

def regex_chunk(lines, regex):
    # type: (List[str], Pattern[str]) -> List[List[str]]
    lst = list(itertools.dropwhile(lambda x: not regex.match(x), lines))
    arr = []
    while lst:
        ret = [lst[0]] + list(
            itertools.takewhile(lambda x: not regex.match(x), lst[1:])
        )
        arr.append(ret)
        lst = list(itertools.dropwhile(lambda x: not regex.match(x), lst[1:]))
    return arr

def chunk_messages(message):  # type: (str) -> List[Tuple[int, str]]
    file_regex = re.compile(r"^(.+:\d+:\d+:)(\s+)(.+)$")
    item_regex = re.compile(r"^\s*\*\s+")
    arr = []
    for chun in regex_chunk(message.splitlines(), file_regex):
        fst = chun[0]
        mat = file_regex.match(fst)
        if mat:
            place = mat.group(1)
            indent = len(mat.group(2))

            lst = [mat.group(3)] + chun[1:]
            if [x for x in lst if item_regex.match(x)]:
                for item in regex_chunk(lst, item_regex):
                    msg = re.sub(item_regex, "", "\n".join(item))
                    arr.append((indent, place + " " + re.sub(r"[\n\s]+", " ", msg)))
            else:
                msg = re.sub(item_regex, "", "\n".join(lst))
                arr.append((indent, place + " " + re.sub(r"[\n\s]+", " ", msg)))
    return arr

def to_one_line_messages(message):  # type: (str) -> str
    ret = []
    max_elem = (0, "")
    for (indent, msg) in chunk_messages(message):
        if indent > max_elem[0]:
            max_elem = (indent, msg)
        else:
            ret.append(max_elem[1])
            max_elem = (indent, msg)
    ret.append(max_elem[1])
    return "\n".join(ret)

def reformat_yaml_exception_message(message):  # type: (str) -> str
    line_regex = re.compile(r'^\s+in "(.+)", line (\d+), column (\d+)$')
    fname_regex = re.compile(r"^file://" + re.escape(os.getcwd()) + "/")
    msgs = message.splitlines()
    ret = []

    if len(msgs) == 3:
        msgs = msgs[1:]
        nblanks = 0
    elif len(msgs) == 4:
        c_msg = msgs[0]
        match = line_regex.match(msgs[1])
        if match:
            c_file, c_line, c_column = match.groups()
            c_file = re.sub(fname_regex, "", c_file)
            ret.append("{}:{}:{}: {}".format(c_file, c_line, c_column, c_msg))

        msgs = msgs[2:]
        nblanks = 2

    p_msg = msgs[0]
    match = line_regex.match(msgs[1])
    if match:
        p_file, p_line, p_column = match.groups()
        p_file = re.sub(fname_regex, "", p_file)
        ret.append(
            "{}:{}:{}:{} {}".format(p_file, p_line, p_column, " " * nblanks, p_msg)
        )
    return "\n".join(ret)

def _add_lc_filename(
    r, source
):  # type: (ruamel.yaml.comments.CommentedBase, AnyStr) -> None
    if isinstance(r, ruamel.yaml.comments.CommentedBase):
        r.lc.filename = source
    if isinstance(r, MutableSequence):
        for d in r:
            _add_lc_filename(d, source)
    elif isinstance(r, MutableMapping):
        for d in six.itervalues(r):
            _add_lc_filename(d, source)

def relname(source):  # type: (Text) -> Text
    if source.startswith("file://"):
        source = source[7:]
        source = os.path.relpath(source)
    return source

def add_lc_filename(
    r, source
):  # type: (ruamel.yaml.comments.CommentedBase, Text) -> None
    _add_lc_filename(r, relname(source))

def reflow(text, maxline, shift=""):  # type: (Text, int, Text) -> Text
    if maxline < 20:
        maxline = 20
    if len(text) > maxline:
        sp = text.rfind(" ", 0, maxline)
        if sp < 1:
            sp = text.find(" ", sp + 1)
            if sp == -1:
                sp = len(text)
        if sp < len(text):
            return "{}\n{}{}".format(
                text[0:sp], shift, reflow(text[sp + 1 :], maxline, shift)
            )
    return text

def indent(
    v, nolead=False, shift=u"  ", bullet=u"  "
):  # type: (Text, bool, Text, Text) -> Text
    if nolead:
        return v.splitlines()[0] + u"\n".join([shift + l for l in v.splitlines()[1:]])
    else:

        def lineno(i, l):  # type: (int, Text) -> Text
            r = lineno_re.match(l)
            if r is not None:
                return r.group(1) + (bullet if i == 0 else shift) + r.group(2)
            else:
                return (bullet if i == 0 else shift) + l

        return u"\n".join([lineno(i, l) for i, l in enumerate(v.splitlines())])

def bullets(textlist, bul):  # type: (List[Text], Text) -> Text
    if len(textlist) == 1:
        return textlist[0]
    else:
        return "\n".join(indent(t, bullet=bul) for t in textlist)

def strip_dup_lineno(text, maxline=None):  # type: (Text, Optional[int]) -> Text
    if maxline is None:
        maxline = int(os.environ.get("COLUMNS", "100"))
    pre = None
    msg = []
    maxno = 0
    for l in text.splitlines():
        g = lineno_re.match(l)
        if not g:
            continue
        maxno = max(maxno, len(g.group(1)))

    for l in text.splitlines():
        g = lineno_re.match(l)
        if not g:
            msg.append(l)
            continue
        if g.group(1) != pre:
            shift = maxno + len(g.group(3))
            g2 = reflow(g.group(2), maxline - shift, " " * shift)
            pre = g.group(1)
            msg.append(pre + " " * (maxno - len(g.group(1))) + g2)
        else:
            g2 = reflow(g.group(2), maxline - maxno, " " * (maxno + len(g.group(3))))
            msg.append(" " * maxno + g2)
    return "\n".join(msg)

def cmap(
    d,  # type: Union[int, float, str, Text, Dict[Text, Any], List[Dict[Text, Any]]]
    lc=None,  # type: Optional[List[int]]
    fn=None,  # type: Optional[Text]
):  # type: (...) -> Union[int, float, str, Text, CommentedMap, CommentedSeq]
    if lc is None:
        lc = [0, 0, 0, 0]
    if fn is None:
        fn = "test"

    if isinstance(d, CommentedMap):
        fn = d.lc.filename if hasattr(d.lc, "filename") else fn
        for k, v in six.iteritems(d):
            if d.lc.data is not None and k in d.lc.data:
                d[k] = cmap(v, lc=d.lc.data[k], fn=fn)
            else:
                d[k] = cmap(v, lc, fn=fn)
        return d
    if isinstance(d, CommentedSeq):
        fn = d.lc.filename if hasattr(d.lc, "filename") else fn
        for k2, v2 in enumerate(d):
            if d.lc.data is not None and k2 in d.lc.data:
                d[k2] = cmap(v2, lc=d.lc.data[k2], fn=fn)
            else:
                d[k2] = cmap(v2, lc, fn=fn)
        return d
    if isinstance(d, MutableMapping):
        cm = CommentedMap()
        for k in sorted(d.keys()):
            v = d[k]
            if isinstance(v, CommentedBase):
                uselc = [v.lc.line, v.lc.col, v.lc.line, v.lc.col]
                vfn = v.lc.filename if hasattr(v.lc, "filename") else fn
            else:
                uselc = lc
                vfn = fn
            cm[k] = cmap(v, lc=uselc, fn=vfn)
            cm.lc.add_kv_line_col(k, uselc)
            cm.lc.filename = fn
        return cm
    if isinstance(d, MutableSequence):
        cs = CommentedSeq()
        for k3, v3 in enumerate(d):
            if isinstance(v3, CommentedBase):
                uselc = [v3.lc.line, v3.lc.col, v3.lc.line, v3.lc.col]
                vfn = v3.lc.filename if hasattr(v3.lc, "filename") else fn
            else:
                uselc = lc
                vfn = fn
            cs.append(cmap(v3, lc=uselc, fn=vfn))
            cs.lc.add_kv_line_col(k3, uselc)
            cs.lc.filename = fn
        return cs
    else:
        return d

class SourceLine(object):
    def __init__(
        self,
        item,  # type: Any
        key=None,  # type: Optional[Any]
        raise_type=six.text_type,  # type: Union[Type[six.text_type], Type[Exception]]
        include_traceback=False,  # type: bool
    ):  # type: (...) -> None
        self.item = item
        self.key = key
        self.raise_type = raise_type
        self.include_traceback = include_traceback

    def __enter__(self):  # type: () -> SourceLine
        return self

    def __exit__(
        self,
        exc_type,  # type: Any
        exc_value,  # type: Any
        tb,  # type: Any
    ):  # type: (...) -> None
        if not exc_value:
            return
        if self.include_traceback and six.PY2:
            # Python2 doesn't actually have chained exceptions, so
            # fake it by injecting the backtrace into the message.
            raise_from(
                self.makeError(
                    "\n".join(traceback.format_exception(exc_type, exc_value, tb))
                ),
                exc_value,
            )
        else:
            raise_from(self.makeError(six.text_type(exc_value)), exc_value)

    def file(self):  # type: () -> Text
        return Text(self.item.lc.filename) if hasattr(self.item.lc, "filename") else ""

    def line(self):  # type: () -> int
        if (
            self.key is None
            or self.item.lc.data is None
            or self.key not in self.item.lc.data
        ):
            return (self.item.lc.line or 0) + 1
        else:
            return (self.item.lc.data[self.key][0] or 0) + 1

    def column(self):  # type: () -> int
        if (
            self.key is None
            or self.item.lc.data is None
            or self.key not in self.item.lc.data
        ):
            return (self.item.lc.col or 0) + 1
        else:
            return (self.item.lc.data[self.key][1] or 0) + 1

    def makeLead(self):  # type: () -> Text
        return "{}:{}:{}:".format(self.file(), self.line(), self.column())

    def makeError(self, msg):  # type: (Text) -> Any
        if not isinstance(self.item, ruamel.yaml.comments.CommentedBase):
            return self.raise_type(msg)
        errs = []
        lead = self.makeLead()
        for m in msg.splitlines():
            if bool(lineno_re.match(m)):
                errs.append(m)
            else:
                errs.append("{} {}".format(lead, m))
        return self.raise_type("\n".join(errs))
<|endoftext|>