<commit_before># -*- coding: utf-8 -*-
"""
@author: Federico Cerchiari <federicocerchiari@gmail.com>
"""
import types
from copy import deepcopy
from collections import Mapping

from exceptions import TagError

class TagContainer(object):

    def __init__(self):
        self.childs = []
        self.parent = None

    def __call__(self, *args):
        return self.append(args)

    def _yield_items(self, items):
        for item in items:
            if type(item) in (
                types.ListType,
                types.TupleType,
                types.GeneratorType,
            ):
                for subitem in item:
                    yield subitem
            else:
                yield item

    def __getitem__(self, i):
        return self.childs[i]

    def __iter__(self):
        return iter(self.childs)

    def _insert(self, child, idx=None, prepend=False):
        if prepend:
            idx = 0
        else:
            idx = idx or len(self.childs)
        self.childs.insert(idx, child)
        if isinstance(child, Tag):
            child.parent = self
            child._tab_count = self._tab_count + 1
            child._own_index = self.childs.index(child)

    def after(self, *siblings):
        for i, sibling in enumerate(self._yield_items(*siblings)):
            self.parent._insert(sibling, idx=self._own_index + i + 1)
        return self

    def before(self, *siblings):
        for i, sibling in enumerate(self._yield_items(*siblings)):
            self.parent._insert(sibling, idx=self._own_index - i - 1)
        return self

    def prepend(self, *childs):
        for child in self._yield_items(*reversed(childs)):
            self._insert(child, prepend=True)
        return self

    def prepend_to(self, father):
        return father.prepend(self)

    def append(self, *childs):
        for child in self._yield_items(*childs):
            self._insert(child)
        return self

    def append_to(self, father):
        return father.append(self)

    def remove(self):
        self.parent.pop(i=self._own_index)
        return self

    def pop(self, i=0):
        self.childs.pop(i).parent = None
        return self

    def empty(self):
        for child in self.childs:
            self.childs.remove()
        return self

    def children(self):
        return filter(lambda x: isinstance(x, Tag), self.childs)

    def first(self):
        return self.childs[0]

    def last(self):
        return self.childs[-1]

    @property
    def length(self):
        return len(self.childs)

    def next(self):
        return self.parent.child[self._own_index + 1]

    def prev(self):
        return self.parent.child[self._own_index - 1]

    def prev_all(self):
        return self.parent.child[:self._own_index - 1]

    def parent(self):
        return self.parent

    def siblings(self):
        return filter(lambda x: isinstance(x, Tag), self.parent.childs)

    def slice(self, start, end):
        return self.childs[start:end]

    def has(self, pattern):
        # TODO
        # return pattern in self.childs
        pass

    def find(self, pattern):
        # TODO
        # return filter(pattern, self.childs)
        pass

class TagAttrs(dict):
    MAPPING_ATTRS = ('style', )
    MULTI_VALUES_ATTRS = ('klass', 'typ', )
    SPECIALS = {
        'klass': 'class',
        'typ': 'type'
    }
    FORMAT = {
        'style': lambda x: ' '.join('{}: {};'.format(k, v) for k, v in x.iteritems()),
        'klass': lambda x: ' '.join(x) if len(x) > 1 else x,
        'typ': lambda x: ' '.join(x)
    }

    def __setitem__(self, key, value):
        if key in self.MULTI_VALUES_ATTRS:
            if key not in self:
                super(TagAttrs, self).__setitem__(key, [])
            self[key].append(value)
        if key in self.MAPPING_ATTRS:
            if key not in self:
                super(TagAttrs, self).__setitem__(key, {})
            self[key].update(value)
        else:
            super(TagAttrs, self).__setitem__(key, value)

    def update(self, attrs=None, **kwargs):
        if attrs is not None:
            for k, v in attrs.iteritems() if isinstance(attrs, Mapping) else attrs:
                self[k] = v
        for k, v in kwargs.iteritems():
            self[k] = v

    def render(self):
        return ''.join(' {}="{}"'.format(self.SPECIALS.get(k, k),
                                         self.FORMAT.get(k, lambda x: x)(v))
                       for k, v in self.iteritems() if v)

class Tag(TagContainer):
    _template = '<{tag}{attrs}>{inner}</{tag}>'
    _needed = None
    _void = False

    def __init__(self, **kwargs):
        self._own_index = None
        self.attrs = TagAttrs()
        self.data = {}
        if self._needed and not set(self._needed).issubset(set(kwargs)):
            raise TagError()
        self.attr(**kwargs)
        self._tab_count = 0
        super(Tag, self).__init__()

    def index(self):
        return self._own_index

    def attr(self, attrs=None, **kwargs):
        self.attrs.update(attrs or kwargs)
        return self

    def prop(self, other=None, **kwargs):
        return self.attr(other, **kwargs)

    def remove_attr(self, attr):
        self.attrs.pop(attr, None)
        return self

    def remove_prop(self, attr):
        self.attrs.pop(attr, None)
        return self

    def add_class(self, cssclass):
        self.attrs['klass'].append(cssclass)
        return self

    def remove_class(self, cssclass):
        self.attrs['klass'].remove(cssclass)
        return self

    def clone(self):
        return deepcopy(self)

    def remove(self):
        del self

    def contents(self):
        return self.childs

    def css(self, *props, **kwprops):
        styles = {}
        if props:
            if len(props) == 1 and isinstance(props[0], Mapping):
                styles = props[0]
            elif len(props) == 2:
                styles = dict(*props)
            else:
                raise TagError
        elif kwprops:
            styles = kwprops
        else:
            raise TagError
        return self.attr(attrs={'style': styles})

    def hide(self):
        self.attrs['style']['display'] = None
        return self

    def show(self):
        self.attrs['style'].pop('display')
        return self

    def toggle(self):
        return self.show() if self.attrs['style']['display'] == None else self.hide()

    def data(self, key, value=None):
        if value:
            self.data[key] = value
            return self
        else:
            return self.data[key]

    def has_class(self, csscl):
        return csscl in self.attrs['klass']

    def toggleClass(self, csscl):
        return self.remove_class(csscl) if self.has_class(csscl) else self.add_class(csscl)

    def html(self):
        return self._render_childs()

    def replace_with(self, other):
        if isinstance(other, Tag):
            self = other
        else:
            raise TagError()
        return self

    def wrap(self, other):
        return self.before(other.append(self)).parent.pop(self._own_index)

    def wrap_inner(self, other):
        self.childs

    def text(self):
        return ''.join(child.text() if isinstance(child, Tag) else child for child in self.childs)

    def render(self, pretty=False):
        prettying = '\t' * self._tab_count + '\n'  # TODO: we're ugly, prettifying don't work
        tag_data = {
            'tag': getattr(self, '_{}__tag'.format(self.__class__.__name__)),
            'attrs': self.attrs.render()
        }
        if not self._void:
            tag_data['inner'] = self._render_childs(pretty)
        template = self._template + prettying if pretty else self._template
        return template.format(**tag_data)

    def _render_childs(self, pretty):
        return ''.join(child.render(pretty) if isinstance(child, Tag) else str(child) for child in self.childs)

    def is_a(self, pattern):
        # TODO
        # return true if self == pattern
        pass

class VoidTag(Tag):
    _void = True
    _template = '<{tag}{attrs}/>'

class TagFinder(object):

    def replace_all(other):
        # TODO replace all finded with other
        pass

    def filter(pattern):
        # TODO find thing with pattern
        pass
<commit_msg>Classes docstrings<commit_after># -*- coding: utf-8 -*-
"""
@author: Federico Cerchiari <federicocerchiari@gmail.com>
"""
import types
from copy import deepcopy
from collections import Mapping

from exceptions import TagError

class DOMElement(object):
    """
    Takes care of the tree structure using the "childs" and "parent" attributes
    and manages the DOM manipulation with proper valorization of those two.
    """
    def __init__(self):
        super(DOMElement, self).__init__()
        self.childs = []
        self.parent = None

    def __call__(self, *args):
        return self.append(args)

    def _yield_items(self, items):
        for item in items:
            if type(item) in (
                types.ListType,
                types.TupleType,
                types.GeneratorType,
            ):
                for subitem in item:
                    yield subitem
            else:
                yield item

    def __getitem__(self, i):
        return self.childs[i]

    def __iter__(self):
        return iter(self.childs)

    def _insert(self, child, idx=None, prepend=False):
        if prepend:
            idx = 0
        else:
            idx = idx or len(self.childs)
        self.childs.insert(idx, child)
        if isinstance(child, Tag):
            child.parent = self
            child._tab_count = self._tab_count + 1
            child._own_index = self.childs.index(child)

    def after(self, *siblings):
        for i, sibling in enumerate(self._yield_items(*siblings)):
            self.parent._insert(sibling, idx=self._own_index + i + 1)
        return self

    def before(self, *siblings):
        for i, sibling in enumerate(self._yield_items(*siblings)):
            self.parent._insert(sibling, idx=self._own_index - i - 1)
        return self

    def prepend(self, *childs):
        for child in self._yield_items(*reversed(childs)):
            self._insert(child, prepend=True)
        return self

    def prepend_to(self, father):
        return father.prepend(self)

    def append(self, *childs):
        for child in self._yield_items(*childs):
            self._insert(child)
        return self

    def append_to(self, father):
        return father.append(self)

    def remove(self):
        self.parent.pop(i=self._own_index)
        return self

    def pop(self, i=0):
        self.childs.pop(i).parent = None
        return self

    def empty(self):
        for child in self.childs:
            self.childs.remove()
        return self

    def children(self):
        return filter(lambda x: isinstance(x, Tag), self.childs)

    def first(self):
        return self.childs[0]

    def last(self):
        return self.childs[-1]

    def next(self):
        return self.parent.child[self._own_index + 1]

    def prev(self):
        return self.parent.child[self._own_index - 1]

    def prev_all(self):
        return self.parent.child[:self._own_index - 1]

    def parent(self):
        return self.parent

    def siblings(self):
        return filter(lambda x: isinstance(x, Tag), self.parent.childs)

    def slice(self, start, end):
        return self.childs[start:end]

    def clone(self):
        return deepcopy(self)

    def replace_with(self, other):
        if isinstance(other, Tag):
            self = other
        else:
            raise TagError()
        return self

    def has(self, pattern):
        # TODO
        # return pattern in self.childs
        pass

    def find(self, pattern):
        # TODO
        # return filter(pattern, self.childs)
        pass

    def wrap(self, other):
        return self.before(other.append(self)).parent.pop(self._own_index)

    def wrap_inner(self, other):
        self.childs

class TagAttrs(dict):
    """
    Html tag attributes container, a subclass of dict with __setitiem__ and update overload.
    Manages the manipulation and render of tag attributes, using the dict api, with few exceptions:
    - space separated multiple value keys
        i.e. the class atrribute, an update on this key will add the value to the list
    - mapping type attributes
        i.e. style attribute, an udpate will trigger the dict.update method

    TagAttrs.render formats all the attributes in the proper html format.
    """
    MAPPING_ATTRS = ('style', )
    MULTI_VALUES_ATTRS = ('klass', 'typ', )
    SPECIALS = {
        'klass': 'class',
        'typ': 'type'
    }
    FORMAT = {
        'style': lambda x: ' '.join('{}: {};'.format(k, v) for k, v in x.iteritems()),
        'klass': lambda x: ' '.join(x) if len(x) > 1 else x,
        'typ': lambda x: ' '.join(x)
    }

    def __setitem__(self, key, value):
        if key in self.MULTI_VALUES_ATTRS:
            if key not in self:
                super(TagAttrs, self).__setitem__(key, [])
            self[key].append(value)
        if key in self.MAPPING_ATTRS:
            if key not in self:
                super(TagAttrs, self).__setitem__(key, {})
            self[key].update(value)
        else:
            super(TagAttrs, self).__setitem__(key, value)

    def update(self, attrs=None, **kwargs):
        if attrs is not None:
            for k, v in attrs.iteritems() if isinstance(attrs, Mapping) else attrs:
                self[k] = v
        for k, v in kwargs.iteritems():
            self[k] = v

    def render(self):
        return ''.join(' {}="{}"'.format(self.SPECIALS.get(k, k),
                                         self.FORMAT.get(k, lambda x: x)(v))
                       for k, v in self.iteritems() if v)

class Tag(DOMElement):
    """
    Provides an api for tag inner manipulation and for rendering.
    """
    _template = '<{tag}{attrs}>{inner}</{tag}>'
    _needed = None
    _void = False

    def __init__(self, **kwargs):
        self._own_index = None
        self.attrs = TagAttrs()
        self.data = {}
        if self._needed and not set(self._needed).issubset(set(kwargs)):
            raise TagError()
        self.attr(**kwargs)
        self._tab_count = 0
        super(Tag, self).__init__()

    @property
    def length(self):
        return len(self.childs)

    def index(self):
        return self._own_index

    def attr(self, attrs=None, **kwargs):
        self.attrs.update(attrs or kwargs)
        return self

    def prop(self, other=None, **kwargs):
        return self.attr(other, **kwargs)

    def remove_attr(self, attr):
        self.attrs.pop(attr, None)
        return self

    def remove_prop(self, attr):
        self.attrs.pop(attr, None)
        return self

    def add_class(self, cssclass):
        self.attrs['klass'].append(cssclass)
        return self

    def remove_class(self, cssclass):
        self.attrs['klass'].remove(cssclass)
        return self

    def contents(self):
        return self.childs

    def css(self, *props, **kwprops):
        styles = {}
        if props:
            if len(props) == 1 and isinstance(props[0], Mapping):
                styles = props[0]
            elif len(props) == 2:
                styles = dict(*props)
            else:
                raise TagError
        elif kwprops:
            styles = kwprops
        else:
            raise TagError
        return self.attr(attrs={'style': styles})

    def hide(self):
        self.attrs['style']['display'] = None
        return self

    def show(self):
        self.attrs['style'].pop('display')
        return self

    def toggle(self):
        return self.show() if self.attrs['style']['display'] == None else self.hide()

    def data(self, key, value=None):
        if value:
            self.data[key] = value
            return self
        else:
            return self.data[key]

    def has_class(self, csscl):
        return csscl in self.attrs['klass']

    def toggleClass(self, csscl):
        return self.remove_class(csscl) if self.has_class(csscl) else self.add_class(csscl)

    def html(self):
        return self._render_childs()

    def text(self):
        return ''.join(child.text() if isinstance(child, Tag) else child for child in self.childs)

    def render(self, pretty=False):
        prettying = '\t' * self._tab_count + '\n'  # TODO: we're ugly, prettifying don't work
        tag_data = {
            'tag': getattr(self, '_{}__tag'.format(self.__class__.__name__)),
            'attrs': self.attrs.render()
        }
        if not self._void:
            tag_data['inner'] = self._render_childs(pretty)
        template = self._template + prettying if pretty else self._template
        return template.format(**tag_data)

    def _render_childs(self, pretty):
        return ''.join(child.render(pretty) if isinstance(child, Tag) else str(child) for child in self.childs)

    def is_a(self, pattern):
        # TODO
        # return true if self == pattern
        pass

class VoidTag(Tag):
    _void = True
    _template = '<{tag}{attrs}/>'

class DOMFinder(object):
    """
    Provides methods and api for DOM navigation and tag finding.
    Emulates the jQuery $('selector') statement.
    """
    def _digest_pattern(self, pattern):
        # TODO
        # transforms a string (or a Tag) into a filter for tag searching
        pass

    def replace_all(other):
        # TODO replace all finded with other
        pass

    def filter(pattern):
        # TODO find thing with pattern
        pass
<|endoftext|>


<commit_before><commit_msg>Begin calibration with JCT statistics<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
A pure-Python implementation of Tokyo Tyrant protocol.
Python 2.4+ is required.

More information about Tokyo Cabinet:
    http://1978th.net/tokyocabinet/

More information about Tokyo Tyrant:
    http://1978th.net/tokyotyrant/

Usage example (note the automatically managed support for table database)::

    >>> import pyrant
    >>> TEST_HOST, TEST_PORT = '127.0.0.1', 1983
    >>> t = pyrant.Tyrant(host=TEST_HOST, port=TEST_PORT)    # default port is 1978
    >>> if t.table_enabled:
    ...     t['key'] = {'name': 'foo'}
    ...     print t['key']['name']
    ... else:
    ...     t['key'] = 'foo'
    ...     print t['key']
    foo
    >>> del t['key']
    >>> print t['key']
    Traceback (most recent call last):
        ...
    KeyError: 'key'

"""

import itertools as _itertools  # EXPLAIN why is it necessary to rename module?
import uuid

# pyrant
import exceptions
import protocol
import query
import utils

__version__ = '0.6.2'
__all__ = ['Tyrant']

# Constants
DEFAULT_HOST = '127.0.0.1'
DEFAULT_PORT = 1978

class Tyrant(object):
    """A Python dictionary API for Tokyo Tyrant.

    :param host: Tyrant host address
    :param port: Tyrant port number
    :param separator: if set, will be used to get/put lists as values. For table
        databases the separator applies to column values.
    :param literal: if set, returned data is not encoded to Unicode (default is
        False)

    Usage::

        >>> import pyrant
        >>> t = pyrant.Tyrant(host=TEST_HOST, port=TEST_PORT)

        # remove anything that could be left from previous time
        >>> t.clear()

        # make sure there are zero records in the database
        >>> len(t)
        0

    :class:`Tyrant` provides pythonic syntax and data pythonification, while the
    the lower-lever :class:`~pyrant.protocol.TyrantProtocol` closely follows the
    orginal Tokyo Tyrant API and only converts incoming data to strings
    (Unicode). You decide which to use. Generally you would want more pythonic
    API (:class:`Tyrant`) for most cases and the lower-level interface to reduce
    overhead or to fix broken data which cannot be properly converted by means
    of the higher-level API.

    It is also important that Tokyo Cabinet has a great query extension for table
    databases. This extension is supported by :class:`pyrant.query.Query` which
    only requires :class:`~pyrant.protocol.TyrantProtocol` to work and can be
    easily accessed via :attr:`Tyrant.query`::

        >>> t.query
        []

    """

    def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, separator=None,
                 literal=False):
        """
        The pythonic interface for Tokyo Tyrant. Mimics dict API.
        """
        # keep the protocol public just in case anyone needs a specific option
        self.proto = protocol.TyrantProtocol(host, port)

        self.separator = separator
        if not separator and self.table_enabled:
            self.separator = protocol.TABLE_COLUMN_SEP

        self.literal = literal

    def __contains__(self, key):
        try:
            self.proto.vsiz(key)
        except exceptions.TyrantError:
            return False
        else:
            return True

    def __delitem__(self, key):
        try:
            return self.proto.out(key)
        except exceptions.TyrantError:
            raise KeyError(key)

    def __getitem__(self, key):
        try:
            elem = self.proto.get(key, self.literal)
            return utils.to_python(elem, self.db_type, self.separator)
        except exceptions.TyrantError:
            raise KeyError(key)

    def get(self, key, default=None):
        """
        Returns value for `key`. If no record is found, returns `default`.
        """
        try:
            return self[key]
        except KeyError:
            return default

    def __iter__(self):
        return iter(self.iterkeys())

    def __len__(self):
        return self.proto.rnum()

    def __repr__(self):
        return u'<Tyrant %s:%s>' % (self.proto.host, self.proto.port)

    def __setitem__(self, key, value):
        """
        Sets given value for given primary key in the database.
        Additional types conversion is only done if the value is a dictionary.
        """
        if isinstance(value, dict):
            # EXPLAIN why the 'from_python' conversion is necessary, as there is no straight forward way of restoring the python objects. What about limiting the allowed keys and values to string only, an raise exception on any other object type?
            flat = list(_itertools.chain(*((k, utils.from_python(v)) for
                                            k,v in value.iteritems())))
            args = [key] + flat
            self.proto.misc('put', args)  # EXPLAIN why is this hack necessary?
        else:
            if isinstance(value, (list, tuple)):
                assert self.separator, "Separator is not set"
                prepared_value = self.separator.join(value)
            else:
                prepared_value = value
            self.proto.put(key, prepared_value)

    @property
    def db_type(self):
        stats = self.get_stats()
        assert 'type' in stats and stats['type'], ('statistics must provide a '
                                                   'valid database type')
        return stats['type']

    @property
    def db_path(self):
        stats = self.get_stats()
        assert 'path' in stats, 'statistics must provide a database path'
        return stats['path']

    @property
    def table_enabled(self):
        """
        Returns True is current database type is TDB so TDB-specific extensions
        are enabled.
        """
        return self.db_type == protocol.DB_TABLE

    def call_func(self, func, key, value, record_locking=False,
                  global_locking=False):
        """
        Calls specific function.
        """
        # TODO: write better documentation *OR* move this method to lower level
        opts = ((record_locking and protocol.TyrantProtocol.RDBXOLCKREC) |
                (global_locking and protocol.TyrantProtocol.RDBXOLCKGLB))
        return self.proto.ext(func, opts, key, value)

    def clear(self):
        """
        Removes all records from the remote database.
        """
        self.proto.vanish()

    def concat(self, key, value, width=None):
        """
        Concatenates columns of the existing record.
        """
        # TODO: write better documentation, provide example code
        if width is None:
            self.proto.putcat(key, value)
        else:
            self.proto.putshl(key, value, width)

    def generate_key(self):
        """
        Returns a unique primary key for given database. Tries to obtain the
        key using database's built-in function `genuid`. If `genuid` fails to
        provide the key, a UUID is generated instead.
        """
        try:
            return self.proto.genuid()
        except ValueError:
            return uuid.uuid4()

    def get_size(self, key):
        """
        Returns the size of the value for `key`.
        """
        try:
            return self.proto.vsiz(key)
        except exceptions.TyrantError:
            raise KeyError(key)

    def get_stats(self):
        """
        Returns the status message of the database as dictionary.
        """
        return utils.csv_to_dict(self.proto.stat())

    def iterkeys(self):
        """
        Iterates keys using remote operations.
        """
        self.proto.iterinit()
        try:
            while True:
                yield self.proto.iternext()
        except exceptions.TyrantError:
            pass

    def keys(self):
        """
        Returns the list of keys in the database.
        """
        return list(self.iterkeys())

    def itervalues(self):
        for k, v in self.iteritems():
            yield v

    def values(self):
        return list(self.itervalues())

    def iteritems(self):
        """
        Returns a generator with key/value pairs. The data is read from the
        database in chunks to alleviate the issues of a) too many database
        hits, and b) too heavy memory usage when only a part of the list is
        actually used.
        """
        CHUNK_SIZE = 1000
        chunk = []
        for key in self.iterkeys():
            chunk.append(key)
            if CHUNK_SIZE <= len(chunk):
                for k,v in self.multi_get(chunk):
                    yield k,v
                chunk = []
        if chunk:
            for k,v in self.multi_get(chunk):
                yield k,v

    def items(self):
        return list(self.iteritems())

    def has_key(self, key):
        return key in self

    def setdefault(self, key, value):
        """
        >>> t.setdefault('foo', {'one': 'one'})
        {u'one': u'one'}
        >>> t.setdefault('foo', {'two': 'two'})
        {u'one': u'one'}

        """
        if not key in self:
            self[key] = value
        return self[key]

    def update(self, mapping=None, **kwargs):
        """
        Updates given objects from a dict, list of key and value pairs or a
        list of named params. Usage::

            mapping = (
                ('john', dict(name='John')),
            )
            t.update(mapping, mary={'name': 'Mary'})

        See built-in `dict.update` method for more information.
        """
        data = dict(mapping or {}, **kwargs)
        self.multi_set(data)

    def multi_del(self, keys, no_update_log=False):
        """
        Removes given records from the database.
        """
        # TODO: write better documentation: why would user need the no_update_log param?
        opts = (no_update_log and protocol.TyrantProtocol.RDBMONOULOG or 0)
        if not isinstance(keys, (list, tuple)):
            keys = list(keys)

        self.proto.misc('outlist', keys, opts)

    def multi_get(self, keys):
        """
        Returns records that match given keys. Missing keys are silently
        ignored, i.e. the number of results may be lower than the number of
        requested keys. The records are returned as key/value pairs. Usage::

            >>> g = t.multi_get(['foo', 'bar', 'galakteko opasnoste'])
            >>> g
            [('foo', {'one': 'one'}), ('bar', {'two': 'two'})]

        :param keys: the list of keys.

        """
        # TODO: write better documentation: why would user need the no_update_log param?
        assert hasattr(keys, '__iter__'), 'expected iterable, got %s' % keys
        prep_val = lambda v: utils.to_python(v, self.db_type, self.separator)

        keys = list(keys)
        data = self.proto.misc('getlist', keys, 0)
        data_keys = data[::2]
        data_vals = (prep_val(x) for x in data[1::2])
        return zip(data_keys, data_vals)

    def multi_set(self, items, no_update_log=False):
        """
        Stores the given records in the database. The records may be given
        as an iterable sequence. Usage::

           >>> t.multi_set([('foo', {'one': 'one'}), ('bar', {'two': 'two'})])

        which is equevalent with the call::

           >>> t.multi_set({'foo': {'one': 'one'}, 'bar':{'two': 'two'}})

        :param items: the sequence of records to be stored.

        """
        opts = (no_update_log and protocol.TyrantProtocol.RDBMONOULOG or 0)
        ready_pairs = []
        # HACK To allow for items to be given in a sequence, as e.g. the list returned from multi_get.
        if isinstance(items, dict):
            iterator = items.iteritems()
        else:
            iterator = iter(items)
        for key, value in iterator:
            if isinstance(value, dict):
                # make flat list of interleaved key/value pairs
                new_value = []
                for pair in value.items():
                    new_value.extend(pair)  # EXPLAIN why is utils.from_python() not used here?
                value = new_value
            if hasattr(value, '__iter__'):
                assert self.separator, 'Separator is not set'
                value = self.separator.join(value)
            ready_pairs.extend((key, value))

        self.proto.misc('putlist', ready_pairs, opts)

    def prefix_keys(self, prefix, maxkeys=None):
        """
        Get forward matching keys in a database.
        The return value is a list object of the corresponding keys.
        """
        # TODO: write better documentation: describe purpose, provide example code
        if maxkeys is None:
            maxkeys = len(self)

        return self.proto.fwmkeys(prefix, maxkeys)

    def sync(self):
        """
        Synchronizes updated content with the database.

        Tokyo Cabinet is not durable, as data committed to the database is not
        immediately flushed to disk. Unwritten data may lost in the event of
        e.g. a system crash.

        Use `sync` to force a flush of unwritten data to disk, but be aware that
        this also locks the writer process and blocks queries.

        The better approach is to use database replication (copy the data to
        another database instance) and backup often.
        """
        self.proto.sync()

    @property
    def query(self):
        """
        Returns a :class:`~pyrant.Query` object for the database.
        Note: Only available for Table Databases.
        """
        if not self.table_enabled:
            raise TypeError('Query only works with table databases but %s is a '
                            '%s database.' % (self.db_path, self.db_type))
        return query.Query(self.proto, self.db_type, self.literal)
<commit_msg>Added Tyrant.multi_add method. Minor refactoring. Added some todo notes.<commit_after># -*- coding: utf-8 -*-
"""
A pure-Python implementation of Tokyo Tyrant protocol.
Python 2.4+ is required.

More information about Tokyo Cabinet:
    http://1978th.net/tokyocabinet/

More information about Tokyo Tyrant:
    http://1978th.net/tokyotyrant/

Usage example (note the automatically managed support for table database)::

    >>> import pyrant
    >>> TEST_HOST, TEST_PORT = '127.0.0.1', 1983
    >>> t = pyrant.Tyrant(host=TEST_HOST, port=TEST_PORT)    # default port is 1978
    >>> if t.table_enabled:
    ...     t['key'] = {'name': 'foo'}
    ...     print t['key']['name']
    ... else:
    ...     t['key'] = 'foo'
    ...     print t['key']
    foo
    >>> del t['key']
    >>> print t['key']
    Traceback (most recent call last):
        ...
    KeyError: 'key'

"""

import itertools
import uuid

# pyrant
import exceptions
import protocol
import query
import utils

__version__ = '0.6.2'
__all__ = ['Tyrant']

# Constants
DEFAULT_HOST = '127.0.0.1'
DEFAULT_PORT = 1978

class Tyrant(object):
    """A Python dictionary API for Tokyo Tyrant.

    :param host: Tyrant host address
    :param port: Tyrant port number
    :param separator: if set, will be used to get/put lists as values. For table
        databases the separator applies to column values.
    :param literal: if set, returned data is not encoded to Unicode (default is
        False)

    Usage::

        >>> import pyrant
        >>> t = pyrant.Tyrant(host=TEST_HOST, port=TEST_PORT)

        # remove anything that could be left from previous time
        >>> t.clear()

        # make sure there are zero records in the database
        >>> len(t)
        0

    :class:`Tyrant` provides pythonic syntax and data pythonification, while the
    the lower-lever :class:`~pyrant.protocol.TyrantProtocol` closely follows the
    orginal Tokyo Tyrant API and only converts incoming data to strings
    (Unicode). You decide which to use. Generally you would want more pythonic
    API (:class:`Tyrant`) for most cases and the lower-level interface to reduce
    overhead or to fix broken data which cannot be properly converted by means
    of the higher-level API.

    It is also important that Tokyo Cabinet has a great query extension for table
    databases. This extension is supported by :class:`pyrant.query.Query` which
    only requires :class:`~pyrant.protocol.TyrantProtocol` to work and can be
    easily accessed via :attr:`Tyrant.query`::

        >>> t.query
        []

    """

    def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, separator=None,
                 literal=False):
        """
        The pythonic interface for Tokyo Tyrant. Mimics dict API.
        """
        # keep the protocol public just in case anyone needs a specific option
        self.proto = protocol.TyrantProtocol(host, port)

        self.separator = separator
        if not separator and self.table_enabled:
            self.separator = protocol.TABLE_COLUMN_SEP

        self.literal = literal

    def __contains__(self, key):
        try:
            self.proto.vsiz(key)
        except exceptions.TyrantError:
            return False
        else:
            return True

    def __delitem__(self, key):
        try:
            return self.proto.out(key)
        except exceptions.TyrantError:
            raise KeyError(key)

    def __getitem__(self, key):
        if not isinstance(key, (str, unicode)):
            raise TypeError('Primary key must be a string, got %s "%s"'
                            % (type(key).__name__, key))
        try:
            elem = self.proto.get(key, self.literal)
            return utils.to_python(elem, self.db_type, self.separator)
        except exceptions.TyrantError:
            raise KeyError(key)

    def get(self, key, default=None):
        """
        Returns value for `key`. If no record is found, returns `default`.
        """
        try:
            return self[key]
        except KeyError:
            return default

    def __iter__(self):
        return iter(self.iterkeys())

    def __len__(self):
        return self.proto.rnum()

    def __repr__(self):
        return u'<Tyrant %s:%s>' % (self.proto.host, self.proto.port)

    def __setitem__(self, key, value):
        """
        Sets given value for given primary key in the database.
        Additional types conversion is only done if the value is a dictionary.
        """
        if isinstance(value, dict):
            # EXPLAIN why the 'from_python' conversion is necessary, as there is no straight forward way of restoring the python objects. What about limiting the allowed keys and values to string only, an raise exception on any other object type?
            flat = list(itertools.chain(*((k, utils.from_python(v)) for
                                           k,v in value.iteritems())))
            args = [key] + flat
            self.proto.misc('put', args)  # EXPLAIN why is this hack necessary?
        else:
            if isinstance(value, (list, tuple)):
                assert self.separator, "Separator is not set"
                prepared_value = self.separator.join(value)
            else:
                prepared_value = value
            self.proto.put(key, prepared_value)

    @property
    def db_type(self):
        stats = self.get_stats()
        assert 'type' in stats and stats['type'], ('statistics must provide a '
                                                   'valid database type')
        return stats['type']

    @property
    def db_path(self):
        stats = self.get_stats()
        assert 'path' in stats, 'statistics must provide a database path'
        return stats['path']

    @property
    def table_enabled(self):
        """
        Returns True is current database type is TDB so TDB-specific extensions
        are enabled.
        """
        return self.db_type == protocol.DB_TABLE

    def call_func(self, func, key, value, record_locking=False,
                  global_locking=False):
        """
        Calls specific function.
        """
        # TODO: write better documentation *OR* move this method to lower level
        opts = ((record_locking and protocol.TyrantProtocol.RDBXOLCKREC) |
                (global_locking and protocol.TyrantProtocol.RDBXOLCKGLB))
        return self.proto.ext(func, opts, key, value)

    def clear(self):
        """
        Removes all records from the remote database.
        """
        self.proto.vanish()

    def concat(self, key, value, width=None):
        """
        Concatenates columns of the existing record.
        """
        # TODO: write better documentation, provide example code
        if width is None:
            self.proto.putcat(key, value)
        else:
            self.proto.putshl(key, value, width)

    def generate_key(self):
        """
        Returns a unique primary key for given database. Tries to obtain the
        key using database's built-in function `genuid`. If `genuid` fails to
        provide the key, a UUID is generated instead.
        """
        try:
            return self.proto.genuid()
        except ValueError:
            return uuid.uuid4()

    def get_size(self, key):
        """
        Returns the size of the value for `key`.
        """
        try:
            return self.proto.vsiz(key)
        except exceptions.TyrantError:
            raise KeyError(key)

    def get_stats(self):
        """
        Returns the status message of the database as dictionary.
        """
        return utils.csv_to_dict(self.proto.stat())

    def iterkeys(self):
        """
        Iterates keys using remote operations.
        """
        self.proto.iterinit()
        try:
            while True:
                yield self.proto.iternext()
        except exceptions.TyrantError:
            pass

    def keys(self):
        """
        Returns the list of keys in the database.
        """
        return list(self.iterkeys())

    def itervalues(self):
        for k, v in self.iteritems():
            yield v

    def values(self):
        return list(self.itervalues())

    def iteritems(self):
        """
        Returns a generator with key/value pairs. The data is read from the
        database in chunks to alleviate the issues of a) too many database
        hits, and b) too heavy memory usage when only a part of the list is
        actually used.
        """
        CHUNK_SIZE = 1000
        chunk = []
        for key in self.iterkeys():
            chunk.append(key)
            if CHUNK_SIZE <= len(chunk):
                for k,v in self.multi_get(chunk):
                    yield k,v
                chunk = []
        if chunk:
            for k,v in self.multi_get(chunk):
                yield k,v

    def items(self):
        return list(self.iteritems())

    def has_key(self, key):
        return key in self

    def setdefault(self, key, value):
        """
        >>> t.setdefault('foo', {'one': 'one'})
        {u'one': u'one'}
        >>> t.setdefault('foo', {'two': 'two'})
        {u'one': u'one'}

        """
        if not key in self:
            self[key] = value
        return self[key]

    def update(self, mapping=None, **kwargs):
        """
        Updates given objects from a dict, list of key and value pairs or a
        list of named params. Usage::

            mapping = (
                ('john', dict(name='John')),
            )
            t.update(mapping, mary={'name': 'Mary'})

        See built-in `dict.update` method for more information.
        """
        data = dict(mapping or {}, **kwargs)
        self.multi_set(data)

    def multi_add(self, values, chunk_size=1000, no_update_log=False):
        """
        Adds given values as new records and returns a list of automatically
        generated primary keys for these records. Wrapper for
        :meth:`~pyrant.Tyrant.multi_set`.

        :param values: any iterable; in fact, you can pass a generator and it
            will be processed in chunks in order to save on resources and
            cut down the database hit at the same time.
        :param chunk_size: size of chunks in which the data will be fed to the
            database. If set to zero, data is fed to database in one chunk.

        """
        keys = self._multi_add(values=values, chunk_size=chunk_size,
                               no_update_log=no_update_log)
        return list(keys)

    def _multi_add(self, values, chunk_size=1000, no_update_log=False):
        assert hasattr(values, '__iter__'), 'values must be an iterable'
        chunk = []
        for value in values:
            'value', value
            key = self.generate_key()
            yield key
            chunk.append((key, value))
            if 0 < chunk_size < len(chunk):
                # chunk has grown big, need to feed the database
                self.multi_set(chunk, no_update_log=no_update_log)
                chunk = []
        else:
            # feed the remnants
            self.multi_set(chunk, no_update_log=no_update_log)

    def multi_del(self, keys, no_update_log=False):
        """
        Removes given records from the database.
        """
        # TODO: write better documentation: why would user need the no_update_log param?
        opts = (no_update_log and protocol.TyrantProtocol.RDBMONOULOG or 0)
        if not isinstance(keys, (list, tuple)):
            keys = list(keys)

        self.proto.misc('outlist', keys, opts)

    def multi_get(self, keys):
        """
        Returns records that match given keys. Missing keys are silently
        ignored, i.e. the number of results may be lower than the number of
        requested keys. The records are returned as key/value pairs. Usage::

            >>> g = t.multi_get(['foo', 'bar', 'galakteko opasnoste'])
            >>> g
            [('foo', {'one': 'one'}), ('bar', {'two': 'two'})]

        :param keys: the list of keys.

        """
        # TODO: write better documentation: why would user need the no_update_log param?
        assert hasattr(keys, '__iter__'), 'expected iterable, got %s' % keys
        prep_val = lambda v: utils.to_python(v, self.db_type, self.separator)

        keys = list(keys)
        data = self.proto.misc('getlist', keys, 0)
        data_keys = data[::2]
        data_vals = (prep_val(x) for x in data[1::2])
        return zip(data_keys, data_vals)

    def multi_set(self, items, no_update_log=False):
        """
        Stores the given records in the database. The records may be given
        as an iterable sequence. Usage::

           >>> t.multi_set([('foo', {'one': 'one'}), ('bar', {'two': 'two'})])

        which is equevalent with the call::

           >>> t.multi_set({'foo': {'one': 'one'}, 'bar':{'two': 'two'}})

        :param items: the sequence of records to be stored.

        """
        # TODO: add here and in other places (in this module and protocol)
        # notes on the no-update-log param:
        #
        # One thing worth noting is  that you can choose replication option
        # when you call from TT. By default, all your operation will be written
        # to update logging files. If you call with RDBMONOULOG option, then it
        # won't record update log nor replication. If you are doing get or
        # search operation which does not require replication, you should call
        # with RDBMONOULOG.
        # If you want to initialise or optimise, but not replicate tcrdbsync,
        # tcrdboptimize, tcrdbvanish, then you should call it with RDBMONOULOG,
        # too.
        #  You can do quite a lot if you call "misc" from Lua extensions.
        #
        # source:
        # http://tokyocabinetwiki.pbworks.com/37_hidden_features_of_the_misc_method

        opts = (no_update_log and protocol.TyrantProtocol.RDBMONOULOG or 0)
        ready_pairs = []
        # HACK To allow for items to be given in a sequence, as e.g. the list returned from multi_get.
        if isinstance(items, dict):
            iterator = items.iteritems()
        else:
            iterator = iter(items)
        for key, value in iterator:
            if isinstance(value, dict):
                # make flat list of interleaved key/value pairs
                new_value = []
                for pair in value.items():
                    new_value.extend(pair)  # EXPLAIN why is utils.from_python() not used here?
                value = new_value
            if hasattr(value, '__iter__'):
                assert self.separator, 'Separator is not set'
                strings = (str(x) for x in value)
                value = self.separator.join(strings)
            ready_pairs.extend((key, value))

        self.proto.misc('putlist', ready_pairs, opts)

    def prefix_keys(self, prefix, maxkeys=None):
        """
        Get forward matching keys in a database.
        The return value is a list object of the corresponding keys.
        """
        # TODO: write better documentation: describe purpose, provide example code
        if maxkeys is None:
            maxkeys = len(self)

        return self.proto.fwmkeys(prefix, maxkeys)

    def sync(self):
        """
        Synchronizes updated content with the database.

        Tokyo Cabinet is not durable, as data committed to the database is not
        immediately flushed to disk. Unwritten data may lost in the event of
        e.g. a system crash.

        Use `sync` to force a flush of unwritten data to disk, but be aware that
        this also locks the writer process and blocks queries.

        The better approach is to use database replication (copy the data to
        another database instance) and backup often.
        """
        self.proto.sync()

    @property
    def query(self):
        """
        Returns a :class:`~pyrant.Query` object for the database.
        Note: Only available for Table Databases.
        """
        if not self.table_enabled:
            raise TypeError('Query only works with table databases but %s is a '
                            '%s database.' % (self.db_path, self.db_type))
        return query.Query(self.proto, self.db_type, self.literal)
<|endoftext|>


<commit_before>"""
Google Cloud Messaging
Previously known as C2DM
Documentation is available on the Android Developer website:
https://developer.android.com/google/gcm/index.html
"""

import json

try:
	from urllib.request import Request, urlopen
	from urllib.parse import urlencode
except ImportError:
	# Python 2 support
	from urllib2 import Request, urlopen
	from urllib import urlencode

from django.core.exceptions import ImproperlyConfigured
from . import NotificationError
from .settings import PUSH_NOTIFICATIONS_SETTINGS as SETTINGS

class GCMError(NotificationError):
	pass

def _chunks(l, n):
	"""
	Yield successive chunks from list \a l with a minimum size \a n
	"""
	for i in range(0, len(l), n):
		yield l[i:i + n]

def _gcm_send(setting_type, data, content_type):
	key_tuple = SETTINGS.get("GCM_API_KEY")

	if not key_tuple:
		raise ImproperlyConfigured('You need to set PUSH_NOTIFICATIONS_SETTINGS["GCM_API_KEY"] to send messages through GCM.')

	key = None

	for i, (x, y) in enumerate(key_tuple):
		if setting_type is None:
			key = y
			break
		elif x == setting_type:
			key = y

	if not key:
		raise ImproperlyConfigured('You need to set PUSH_NOTIFICATIONS_SETTINGS["GCM_API_KEY"] to send messages through GCM.')

	headers = {
		"Content-Type": content_type,
		"Authorization": "key=%s" % (key),
		"Content-Length": str(len(data)),
	}

	request = Request(SETTINGS["GCM_POST_URL"], data, headers)
	return urlopen(request).read()

def _gcm_send_plain(setting_type, registration_id, data, collapse_key=None, delay_while_idle=False, time_to_live=0):
	"""
	Sends a GCM notification to a single registration_id.
	This will send the notification as form data.
	If sending multiple notifications, it is more efficient to use
	gcm_send_bulk_message() with a list of registration_ids
	"""

	values = {"registration_id": registration_id}

	if collapse_key:
		values["collapse_key"] = collapse_key

	if delay_while_idle:
		values["delay_while_idle"] = int(delay_while_idle)

	if time_to_live:
		values["time_to_live"] = time_to_live

	for k, v in data.items():
		values["data.%s" % (k)] = v.encode("utf-8")

	data = urlencode(sorted(values.items())).encode("utf-8")  # sorted items for tests

	result = _gcm_send(setting_type, data, "application/x-www-form-urlencoded;charset=UTF-8")
	if result.startswith("Error="):
		raise GCMError(result)
	return result

def _gcm_send_json(setting_type, registration_ids, data, collapse_key=None, delay_while_idle=False, time_to_live=0):
	"""
	Sends a GCM notification to one or more registration_ids. The registration_ids
	needs to be a list.
	This will send the notification as json data.
	"""

	values = {"registration_ids": registration_ids}

	if data is not None:
		values["data"] = data

	if collapse_key:
		values["collapse_key"] = collapse_key

	if delay_while_idle:
		values["delay_while_idle"] = delay_while_idle

	if time_to_live:
		values["time_to_live"] = time_to_live

	data = json.dumps(values, separators=(",", ":"), sort_keys=True).encode("utf-8")  # keys sorted for tests

	result = json.loads(_gcm_send(setting_type, data, "application/json"))
	if result["failure"]:
		raise GCMError(result)
	return result

def gcm_send_message(setting_type, registration_id, data, collapse_key=None, delay_while_idle=False, time_to_live=0):
	"""
	Sends a GCM notification to a single registration_id.

	This will send the notification as form data if possible, otherwise it will
	fall back to json data.

	If sending multiple notifications, it is more efficient to use
	gcm_send_bulk_message() with a list of registration_ids
	"""

	args = data, collapse_key, delay_while_idle, time_to_live

	try:
		_gcm_send_plain(setting_type, registration_id, *args)
	except AttributeError:
		_gcm_send_json(setting_type, [registration_id], *args)

def gcm_send_bulk_message(setting_type, registration_ids, data, collapse_key=None, delay_while_idle=False, time_to_live=0):
	"""
	Sends a GCM notification to one or more registration_ids. The registration_ids
	needs to be a list.
	This will send the notification as json data.
	"""

	args = data, collapse_key, delay_while_idle, time_to_live

	# GCM only allows up to 1000 reg ids per bulk message
	# https://developer.android.com/google/gcm/gcm.html#request
	max_recipients = SETTINGS.get("GCM_MAX_RECIPIENTS")
	if len(registration_ids) > max_recipients:
		ret = []
		for chunk in _chunks(registration_ids, max_recipients):
			ret.append(_gcm_send_json(chunk, *args))
		return ret

	return _gcm_send_json(setting_type, registration_ids, *args)
<commit_msg>gcm: Fix bulk_message notification with multiple settings.<commit_after>"""
Google Cloud Messaging
Previously known as C2DM
Documentation is available on the Android Developer website:
https://developer.android.com/google/gcm/index.html
"""

import json

try:
	from urllib.request import Request, urlopen
	from urllib.parse import urlencode
except ImportError:
	# Python 2 support
	from urllib2 import Request, urlopen
	from urllib import urlencode

from django.core.exceptions import ImproperlyConfigured
from . import NotificationError
from .settings import PUSH_NOTIFICATIONS_SETTINGS as SETTINGS

class GCMError(NotificationError):
	pass

def _chunks(l, n):
	"""
	Yield successive chunks from list \a l with a minimum size \a n
	"""
	for i in range(0, len(l), n):
		yield l[i:i + n]

def _gcm_send(setting_type, data, content_type):
	key_tuple = SETTINGS.get("GCM_API_KEY")

	if not key_tuple:
		raise ImproperlyConfigured('You need to set PUSH_NOTIFICATIONS_SETTINGS["GCM_API_KEY"] to send messages through GCM.')

	key = None
	if setting_type.__class__ == list:
		for sett_type in setting_type:
			for i, (x, y) in enumerate(key_tuple):
				if sett_type is None:
					key = y
					break
				elif x == sett_type:
					key = y

			if not key:
				raise ImproperlyConfigured('You need to set PUSH_NOTIFICATIONS_SETTINGS["GCM_API_KEY"] to send messages through GCM.')

			headers = {
				"Content-Type": content_type,
				"Authorization": "key=%s" % (key),
				"Content-Length": str(len(data)),
			}

			request = Request(SETTINGS["GCM_POST_URL"], data, headers)
			return urlopen(request).read()
	else:
		sett_type = setting_type
		for i, (x, y) in enumerate(key_tuple):
			if sett_type is None:
				key = y
				break
			elif x == sett_type:
				key = y

		if not key:
			raise ImproperlyConfigured('You need to set PUSH_NOTIFICATIONS_SETTINGS["GCM_API_KEY"] to send messages through GCM.')

		headers = {
			"Content-Type": content_type,
			"Authorization": "key=%s" % (key),
			"Content-Length": str(len(data)),
		}

		request = Request(SETTINGS["GCM_POST_URL"], data, headers)
		return urlopen(request).read()

def _gcm_send_plain(setting_type, registration_id, data, collapse_key=None, delay_while_idle=False, time_to_live=0):
	"""
	Sends a GCM notification to a single registration_id.
	This will send the notification as form data.
	If sending multiple notifications, it is more efficient to use
	gcm_send_bulk_message() with a list of registration_ids
	"""

	values = {"registration_id": registration_id}

	if collapse_key:
		values["collapse_key"] = collapse_key

	if delay_while_idle:
		values["delay_while_idle"] = int(delay_while_idle)

	if time_to_live:
		values["time_to_live"] = time_to_live

	for k, v in data.items():
		values["data.%s" % (k)] = v.encode("utf-8")

	data = urlencode(sorted(values.items())).encode("utf-8")  # sorted items for tests

	result = _gcm_send(setting_type, data, "application/x-www-form-urlencoded;charset=UTF-8")
	if result.startswith("Error="):
		raise GCMError(result)
	return result

def _gcm_send_json(setting_type, registration_ids, data, collapse_key=None, delay_while_idle=False, time_to_live=0):
	"""
	Sends a GCM notification to one or more registration_ids. The registration_ids
	needs to be a list.
	This will send the notification as json data.
	"""

	values = {"registration_ids": registration_ids}

	if data is not None:
		values["data"] = data

	if collapse_key:
		values["collapse_key"] = collapse_key

	if delay_while_idle:
		values["delay_while_idle"] = delay_while_idle

	if time_to_live:
		values["time_to_live"] = time_to_live

	data = json.dumps(values, separators=(",", ":"), sort_keys=True).encode("utf-8")  # keys sorted for tests

	result = json.loads(_gcm_send(setting_type, data, "application/json"))
	if result["failure"]:
		raise GCMError(result)
	return result

def gcm_send_message(setting_type, registration_id, data, collapse_key=None, delay_while_idle=False, time_to_live=0):
	"""
	Sends a GCM notification to a single registration_id.

	This will send the notification as form data if possible, otherwise it will
	fall back to json data.

	If sending multiple notifications, it is more efficient to use
	gcm_send_bulk_message() with a list of registration_ids
	"""

	args = data, collapse_key, delay_while_idle, time_to_live

	try:
		_gcm_send_plain(setting_type, registration_id, *args)
	except AttributeError:
		_gcm_send_json([setting_type], [registration_id], *args)

def gcm_send_bulk_message(setting_type, registration_ids, data, collapse_key=None, delay_while_idle=False, time_to_live=0):
	"""
	Sends a GCM notification to one or more registration_ids. The registration_ids
	needs to be a list.
	This will send the notification as json data.
	"""

	args = data, collapse_key, delay_while_idle, time_to_live

	# GCM only allows up to 1000 reg ids per bulk message
	# https://developer.android.com/google/gcm/gcm.html#request
	max_recipients = SETTINGS.get("GCM_MAX_RECIPIENTS")
	if len(registration_ids) > max_recipients:
		ret = []
		for chunk in _chunks(registration_ids, max_recipients):
			ret.append(_gcm_send_json(setting_type, chunk, *args))
		return ret

	return _gcm_send_json(setting_type, registration_ids, *args)
<|endoftext|>


<commit_before>import ast
import json
import os
import shutil

import re
import warnings

import pkg_resources

from pyroSAR._dev_config import ConfigHandler
from spatialist.ancillary import finder

config = ConfigHandler()

class ExamineSnap(object):
    """
    Class to check if ESA SNAP is installed.
    Upon initialization, this class searches for relevant binaries and the accompanying
    relative directory structure, which uniquely identify an ESA SNAP installation on a system.
    First, all relevant file and folder names are read from the pyroSAR config file if it exists
    and their existence is verified.
    If this fails, a system check is performed to find relevant binaries in the system PATH variable and
    additional files and folders relative to them.
    Furthermore, a snap.auxdata.properties file is scanned for auxiliary data URLs and local storage location.
    This is used by SNAP to manage data from e.g. the SRTM mission. In case SNAP is not installed, the respective
    information is read from a default file delivered with pyroSAR. This has the advantage of using the SNAP download
    URLs and local directory structure without having SNAP installed such that it can be adapted by other SAR software.
    """

    def __init__(self):

        # define some attributes which identify SNAP
        self.identifiers = ['path', 'gpt', 'etc', 'auxdata']

        # try reading all necessary attributes from the config file
        # print('reading config..')
        self.__read_config()

        # if SNAP could not be identified from the config attributes, do a system search for it
        if not self.__is_identified():
            # print('identifying SNAP..')
            self.__identify_snap()

        # if the auxdatapath attribute was not yet set, create a default directory
        if not hasattr(self, 'auxdatapath'):
            self.auxdatapath = os.path.join(os.path.expanduser('~'), '.snap', 'auxdata')
            if not os.path.isdir(self.auxdatapath):
                os.makedirs(self.auxdatapath)

        # if the SNAP auxdata properties attribute was not yet identified,
        # point it to the default file delivered with pyroSAR
        if not hasattr(self, 'properties'):
            # print('using default properties file..')
            template = os.path.join('snap', 'data', 'snap.auxdata.properties')
            self.properties = pkg_resources.resource_filename(__name__, template)

        # update the snap properties; this reads the 'properties' file and looks for any changes,
        # which are then updated for the object
        self.__update_snap_properties()

        # update the config file: this scans for config changes and re-writes the config file if any are found
        self.__update_config()

    def __is_identified(self):
        """
        Check if SNAP has been properly identified, i.e. all paths in self.identifiers
        have been detected and confirmed.

        Returns
        -------
        bool
        """
        return sum([hasattr(self, x) for x in self.identifiers]) == len(self.identifiers)

    def __identify_snap(self):
        """
        do a comprehensive search for an ESA SNAP installation

        Returns
        -------
        bool
            has the SNAP properties file been changed?
        """
        # create a list of possible SNAP executables
        defaults = ['snap64.exe', 'snap32.exe', 'snap.exe', 'snap']
        paths = os.environ['PATH'].split(os.path.pathsep)
        options = [os.path.join(path, option) for path in paths for option in defaults]
        options = [x for x in options if os.path.isfile(x)]

        if not hasattr(self, 'path') or not os.path.isfile(self.path):
            executables = options
        else:
            executables = [self.path] + options

        # for each possible SNAP executable, check whether additional files and directories exist relative to it
        # to confirm whether it actually is a ESA SNAP installation or something else like e.g. the Ubuntu App Manager
        for path in executables:
            if os.path.islink(path):
                path = os.path.realpath(path)

            # check whether a directory etc exists relative to the SNAP executable
            etc = os.path.join(os.path.dirname(os.path.dirname(path)), 'etc')
            if not os.path.isdir(etc):
                continue

            # check the content of the etc directory
            auxdata = os.listdir(etc)
            if 'snap.auxdata.properties' not in auxdata:
                continue
            else:
                auxdata_properties = os.path.join(etc, 'snap.auxdata.properties')

            # identify the gpt executable
            gpt_candidates = finder(os.path.dirname(path), ['gpt', 'gpt.exe'])
            if len(gpt_candidates) == 0:
                continue
            else:
                gpt = gpt_candidates[0]

            self.path = path
            self.etc = etc
            self.gpt = gpt
            self.auxdata = auxdata
            self.properties = auxdata_properties
            return

        warnings.warn('SNAP could not be identified. If you have installed it please add the path to the SNAP '
                      'executables (bin subdirectory) to the PATH environment. '
                      'E.g. in the Linux .bashrc file add the following line:\nexport PATH=$PATH:path/to/snap/bin"')

    def __read_config(self):
        """
        This method reads the config.ini to examine the snap paths.
        If the snap paths are not in the config.ini or the paths are wrong they will be automatically created.

        Returns
        -------

        """
        for attr in self.identifiers + ['auxdatapath', 'properties']:
            self.__read_config_attr(attr, 'SNAP')

        snap_properties = {}
        if 'OUTPUT' in config.sections:
            snap_properties = config['OUTPUT']
        if len(snap_properties.keys()) > 0:
            setattr(self, 'snap_properties', snap_properties)

    def __read_config_attr(self, attr, section):
        """
        read an attribute from the config file and set it as an object attribute

        Parameters
        ----------
        attr: str
            the attribute name
        section: str
            the config section to read the attribute from

        Returns
        -------

        """
        if section in config.sections:
            if attr in config[section].keys():
                val = config[section][attr]
                if attr in ['path', 'gpt', 'properties']:
                    exist = os.path.isfile(val)
                elif attr == 'auxdata':
                    val = ast.literal_eval(val)
                    exist = isinstance(val, list)
                else:
                    exist = os.path.isdir(val)
                if exist:
                    # print('setting attribute {}'.format(attr))
                    setattr(self, attr, val)

    def __update_config(self):
        for section in ['SNAP', 'OUTPUT', 'URL']:
            if section not in config.sections:
                # print('creating section {}..'.format(section))
                config.add_section(section)

        for key in self.identifiers + ['auxdatapath', 'properties']:
            if hasattr(self, key):
                self.__update_config_attr(key, getattr(self, key), 'SNAP')

        for key, value in self.snap_properties.items():
            self.__update_config_attr(key, value, 'OUTPUT')

    @staticmethod
    def __update_config_attr(attr, value, section):
        if isinstance(value, list):
            value = json.dumps(value)

        if attr not in config[section].keys() or config[section][attr] != value:
            # print('updating attribute {0}:{1}..'.format(section, attr))
            # print('  {0} -> {1}'.format(repr(config[section][attr]), repr(value)))
            config.set(section, key=attr, value=value, overwrite=True)

    def __update_snap_properties(self):
        """
        Read the snap.auxdata.properties file entries to object attributes

        Returns
        -------

        """
        pattern = r'^(?P<key>[\w\.]*)\s*=\s*(?P<value>.*)\n'

        if not hasattr(self, 'snap_properties'):
            self.snap_properties = {}

        demPath = os.path.join(self.auxdatapath, 'dem')
        landCoverPath = os.path.join(self.auxdatapath, 'LandCover')

        with open(self.properties, 'r') as prop:
            for line in prop:
                if re.search(pattern, line):
                    key, value = re.match(re.compile(pattern), line).groups()
                    value = value \
                        .replace('${AuxDataPath}', self.auxdatapath) \
                        .replace('${demPath}', demPath) \
                        .replace('${landCoverPath}', landCoverPath) \
                        .replace('\\', '/')
                    if not key in self.snap_properties.keys() or self.snap_properties[key] != value:
                        self.snap_properties[key] = value

class ExamineGamma(object):
    def __init__(self):
        home_sys = os.environ.get('GAMMA_HOME')
        if 'GAMMA' in config.sections:
            attr = config['GAMMA']
            for key, value in attr.items():
                setattr(self, key, value)
        if hasattr(self, 'home'):
            if home_sys is not None and self.home != home_sys:
                print('the value of GAMMA_HOME is different to that in the pyroSAR configuration;\n'
                      '  was: {}\n'
                      '  is : {}\n'
                      'resetting the configuration and deleting parsed modules'
                      .format(self.home, home_sys))
                parsed = os.path.join(os.path.dirname(config.file), 'gammaparse')
                shutil.rmtree(parsed)
                self.home = home_sys
        if not hasattr(self, 'home'):
            if home_sys is not None:
                setattr(self, 'home', home_sys)
            else:
                raise RuntimeError('could not read Gamma installation directory')
        self.version = re.search('GAMMA_SOFTWARE-(?P<version>[0-9]{8})',
                                 getattr(self, 'home')).group('version')
        self.__update_config()

    def __update_config(self):
        if 'GAMMA' not in config.sections:
            config.add_section('GAMMA')

        for attr in ['home', 'version']:
            self.__update_config_attr(attr, getattr(self, attr), 'GAMMA')

    @staticmethod
    def __update_config_attr(attr, value, section):
        if isinstance(value, list):
            value = json.dumps(value)

        if attr not in config[section].keys() or config[section][attr] != value:
            config.set(section, key=attr, value=value, overwrite=True)<commit_msg>[ExamineGamma] ignore GAMMA_HOME if path does not exist, issue warning<commit_after>import ast
import json
import os
import shutil

import re
import warnings

import pkg_resources

from pyroSAR._dev_config import ConfigHandler
from spatialist.ancillary import finder

config = ConfigHandler()

class ExamineSnap(object):
    """
    Class to check if ESA SNAP is installed.
    Upon initialization, this class searches for relevant binaries and the accompanying
    relative directory structure, which uniquely identify an ESA SNAP installation on a system.
    First, all relevant file and folder names are read from the pyroSAR config file if it exists
    and their existence is verified.
    If this fails, a system check is performed to find relevant binaries in the system PATH variable and
    additional files and folders relative to them.
    Furthermore, a snap.auxdata.properties file is scanned for auxiliary data URLs and local storage location.
    This is used by SNAP to manage data from e.g. the SRTM mission. In case SNAP is not installed, the respective
    information is read from a default file delivered with pyroSAR. This has the advantage of using the SNAP download
    URLs and local directory structure without having SNAP installed such that it can be adapted by other SAR software.
    """

    def __init__(self):

        # define some attributes which identify SNAP
        self.identifiers = ['path', 'gpt', 'etc', 'auxdata']

        # try reading all necessary attributes from the config file
        # print('reading config..')
        self.__read_config()

        # if SNAP could not be identified from the config attributes, do a system search for it
        if not self.__is_identified():
            # print('identifying SNAP..')
            self.__identify_snap()

        # if the auxdatapath attribute was not yet set, create a default directory
        if not hasattr(self, 'auxdatapath'):
            self.auxdatapath = os.path.join(os.path.expanduser('~'), '.snap', 'auxdata')
            if not os.path.isdir(self.auxdatapath):
                os.makedirs(self.auxdatapath)

        # if the SNAP auxdata properties attribute was not yet identified,
        # point it to the default file delivered with pyroSAR
        if not hasattr(self, 'properties'):
            # print('using default properties file..')
            template = os.path.join('snap', 'data', 'snap.auxdata.properties')
            self.properties = pkg_resources.resource_filename(__name__, template)

        # update the snap properties; this reads the 'properties' file and looks for any changes,
        # which are then updated for the object
        self.__update_snap_properties()

        # update the config file: this scans for config changes and re-writes the config file if any are found
        self.__update_config()

    def __is_identified(self):
        """
        Check if SNAP has been properly identified, i.e. all paths in self.identifiers
        have been detected and confirmed.

        Returns
        -------
        bool
        """
        return sum([hasattr(self, x) for x in self.identifiers]) == len(self.identifiers)

    def __identify_snap(self):
        """
        do a comprehensive search for an ESA SNAP installation

        Returns
        -------
        bool
            has the SNAP properties file been changed?
        """
        # create a list of possible SNAP executables
        defaults = ['snap64.exe', 'snap32.exe', 'snap.exe', 'snap']
        paths = os.environ['PATH'].split(os.path.pathsep)
        options = [os.path.join(path, option) for path in paths for option in defaults]
        options = [x for x in options if os.path.isfile(x)]

        if not hasattr(self, 'path') or not os.path.isfile(self.path):
            executables = options
        else:
            executables = [self.path] + options

        # for each possible SNAP executable, check whether additional files and directories exist relative to it
        # to confirm whether it actually is a ESA SNAP installation or something else like e.g. the Ubuntu App Manager
        for path in executables:
            if os.path.islink(path):
                path = os.path.realpath(path)

            # check whether a directory etc exists relative to the SNAP executable
            etc = os.path.join(os.path.dirname(os.path.dirname(path)), 'etc')
            if not os.path.isdir(etc):
                continue

            # check the content of the etc directory
            auxdata = os.listdir(etc)
            if 'snap.auxdata.properties' not in auxdata:
                continue
            else:
                auxdata_properties = os.path.join(etc, 'snap.auxdata.properties')

            # identify the gpt executable
            gpt_candidates = finder(os.path.dirname(path), ['gpt', 'gpt.exe'])
            if len(gpt_candidates) == 0:
                continue
            else:
                gpt = gpt_candidates[0]

            self.path = path
            self.etc = etc
            self.gpt = gpt
            self.auxdata = auxdata
            self.properties = auxdata_properties
            return

        warnings.warn('SNAP could not be identified. If you have installed it please add the path to the SNAP '
                      'executables (bin subdirectory) to the PATH environment. '
                      'E.g. in the Linux .bashrc file add the following line:\nexport PATH=$PATH:path/to/snap/bin"')

    def __read_config(self):
        """
        This method reads the config.ini to examine the snap paths.
        If the snap paths are not in the config.ini or the paths are wrong they will be automatically created.

        Returns
        -------

        """
        for attr in self.identifiers + ['auxdatapath', 'properties']:
            self.__read_config_attr(attr, 'SNAP')

        snap_properties = {}
        if 'OUTPUT' in config.sections:
            snap_properties = config['OUTPUT']
        if len(snap_properties.keys()) > 0:
            setattr(self, 'snap_properties', snap_properties)

    def __read_config_attr(self, attr, section):
        """
        read an attribute from the config file and set it as an object attribute

        Parameters
        ----------
        attr: str
            the attribute name
        section: str
            the config section to read the attribute from

        Returns
        -------

        """
        if section in config.sections:
            if attr in config[section].keys():
                val = config[section][attr]
                if attr in ['path', 'gpt', 'properties']:
                    exist = os.path.isfile(val)
                elif attr == 'auxdata':
                    val = ast.literal_eval(val)
                    exist = isinstance(val, list)
                else:
                    exist = os.path.isdir(val)
                if exist:
                    # print('setting attribute {}'.format(attr))
                    setattr(self, attr, val)

    def __update_config(self):
        for section in ['SNAP', 'OUTPUT', 'URL']:
            if section not in config.sections:
                # print('creating section {}..'.format(section))
                config.add_section(section)

        for key in self.identifiers + ['auxdatapath', 'properties']:
            if hasattr(self, key):
                self.__update_config_attr(key, getattr(self, key), 'SNAP')

        for key, value in self.snap_properties.items():
            self.__update_config_attr(key, value, 'OUTPUT')

    @staticmethod
    def __update_config_attr(attr, value, section):
        if isinstance(value, list):
            value = json.dumps(value)

        if attr not in config[section].keys() or config[section][attr] != value:
            # print('updating attribute {0}:{1}..'.format(section, attr))
            # print('  {0} -> {1}'.format(repr(config[section][attr]), repr(value)))
            config.set(section, key=attr, value=value, overwrite=True)

    def __update_snap_properties(self):
        """
        Read the snap.auxdata.properties file entries to object attributes

        Returns
        -------

        """
        pattern = r'^(?P<key>[\w\.]*)\s*=\s*(?P<value>.*)\n'

        if not hasattr(self, 'snap_properties'):
            self.snap_properties = {}

        demPath = os.path.join(self.auxdatapath, 'dem')
        landCoverPath = os.path.join(self.auxdatapath, 'LandCover')

        with open(self.properties, 'r') as prop:
            for line in prop:
                if re.search(pattern, line):
                    key, value = re.match(re.compile(pattern), line).groups()
                    value = value \
                        .replace('${AuxDataPath}', self.auxdatapath) \
                        .replace('${demPath}', demPath) \
                        .replace('${landCoverPath}', landCoverPath) \
                        .replace('\\', '/')
                    if not key in self.snap_properties.keys() or self.snap_properties[key] != value:
                        self.snap_properties[key] = value

class ExamineGamma(object):
    def __init__(self):
        home_sys = os.environ.get('GAMMA_HOME')
        if not os.path.isdir(home_sys):
            warnings.warn('found GAMMA_HOME environment variable, but directory does not exist')
            home_sys = None
        if 'GAMMA' in config.sections:
            attr = config['GAMMA']
            for key, value in attr.items():
                setattr(self, key, value)
        if hasattr(self, 'home'):
            if home_sys is not None and self.home != home_sys:
                print('the value of GAMMA_HOME is different to that in the pyroSAR configuration;\n'
                      '  was: {}\n'
                      '  is : {}\n'
                      'resetting the configuration and deleting parsed modules'
                      .format(self.home, home_sys))
                parsed = os.path.join(os.path.dirname(config.file), 'gammaparse')
                shutil.rmtree(parsed)
                self.home = home_sys
        if not hasattr(self, 'home'):
            if home_sys is not None:
                setattr(self, 'home', home_sys)
            else:
                raise RuntimeError('could not read Gamma installation directory')
        self.version = re.search('GAMMA_SOFTWARE-(?P<version>[0-9]{8})',
                                 getattr(self, 'home')).group('version')
        self.__update_config()

    def __update_config(self):
        if 'GAMMA' not in config.sections:
            config.add_section('GAMMA')

        for attr in ['home', 'version']:
            self.__update_config_attr(attr, getattr(self, attr), 'GAMMA')

    @staticmethod
    def __update_config_attr(attr, value, section):
        if isinstance(value, list):
            value = json.dumps(value)

        if attr not in config[section].keys() or config[section][attr] != value:
            config.set(section, key=attr, value=value, overwrite=True)<|endoftext|>