<commit_before>"""
A pythonic python-only implementation of Tokyo Tyrant protocol.
Python 2.4+ is needed.

This library takes a "pythonic" approach to make it more clear and easy
to implement.

More information about Tokyo Cabinet:
    http://tokyocabinet.sourceforge.net/

More information about Tokyo Tyrant:
    http://tokyocabinet.sourceforge.net/tyrantdoc/

This is an usage example:

    >>> import pyrant
    >>> t = pyrant.Tyrant(host='127.0.0.1', port=1978)
    >>> if t.dbtype != pyrant.DBTYPETABLE:
    ...     t['key'] = 'foo'
    ...     print t['key']
    ... else:
    ...     t['key'] = {'name': 'foo'}
    ...     print t['key']['name']
    foo
    >>> del t['key']
    >>> print t['key']
    Traceback (most recent call last):
        ...
    KeyError: 'key'

"""

import itertools as _itertools
from protocol import TyrantProtocol, TyrantError

__version__ = '0.0.2'
__all__ = ['Tyrant', 'TyrantError', 'TyrantProtocol', 'Q']

# Constants
DEFAULT_HOST = '127.0.0.1'
DEFAULT_PORT = 1978
MAX_RESULTS = 1000

# Table Types
DBTYPEBTREE = 'B+ tree'
DBTYPETABLE = 'table'
DBTYPEMEMORY = 'on-memory hash'
DBTYPEHASH = 'hash'

def _parse_elem(elem, dbtype, sep=None):
    if dbtype == DBTYPETABLE:
        # Split element by \x00 which is the column separator
        elems = elem.split('\x00')
        if not elems[0]:
            return None

        return dict((elems[i], elems[i + 1]) \
                        for i in xrange(0, len(elems), 2))
    elif sep and sep in elem:
        return elem.split(sep)

    return elem

class Tyrant(dict):
    """Main class of Tyrant implementation. Acts like a python dictionary,
    so you can query object using normal subscript operations.
    """

    def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, separator=None,
                 literal=False):
        """
        Acts like a python dictionary. Params are:

        host: Tyrant Host address
        port: Tyrant port number
        separator: If this parameter is set, you can put and get lists as
        values.
        literal: If is set string is returned instead of unicode
        """
        # We want to make protocol public just in case anyone need any
        # specific option
        self.proto = TyrantProtocol(host, port)
        self.dbtype = self.get_stats()['type']
        self.separator = separator
        self.literal = literal

    def __contains__(self, key):
        try:
            self.proto.vsiz(key)
        except TyrantError:
            return False
        else:
            return True

    def __delitem__(self, key):
        try:
            return self.proto.out(key)
        except TyrantError:
            raise KeyError(key)

    def __getitem__(self, key):
        try:
            return _parse_elem(self.proto.get(key, self.literal), self.dbtype,
                               self.separator)
        except TyrantError:
            raise KeyError(key)

    def __len__(self):
        return self.proto.rnum()

    def __repr__(self):
        return object.__repr__(self)

    def __setitem__(self, key, value):
        if isinstance(value, dict):
            flat = _itertools.chain([key], *value.iteritems())
            self.proto.misc('put', list(flat))

        elif isinstance(value, (list, tuple)):
            assert self.separator, "Separator is not set"

            flat = self.separator.join(value)
            self.proto.put(key, flat)

        else:
            self.proto.put(key, value)

    def call_func(self, func, key, value, record_locking=False,
                  global_locking=False):
        """Call specific function.
        """
        opts = ((record_locking and TyrantProtocol.RDBXOLCKREC) |
                (global_locking and TyrantProtocol.RDBXOLCKGLB))
        return self.proto.ext(func, opts, key, value)

    def clear(self):
        """Used in order to remove all records of a remote database object"""
        self.proto.vanish()

    def concat(self, key, value, width=None):
        """Concatenate columns of the existing record"""
        if width is None:
            self.proto.putcat(key, value)
        else:
            self.proto.putshl(key, value, width)

    def get_size(self, key):
        """Get the size of the value of a record"""
        try:
            return self.proto.vsiz(key)
        except TyrantError:
            raise KeyError(key)

    def get_stats(self):
        """Get the status string of the database.
        The return value is the status message of the database.The message
        format is a dictionary.
        """
        return dict(l.split('\t', 1) \
                        for l in self.proto.stat().splitlines() if l)

    def iterkeys(self):
        """Iterate keys using remote operations"""
        self.proto.iterinit()
        try:
            while True:
                yield self.proto.iternext()
        except TyrantError:
            pass

    def keys(self):
        """Return the list of keys in database"""
        return list(self.iterkeys())

    def update(self, other, **kwargs):
        """Update/Add given objects into database"""
        self.multi_set(other.iteritems())
        if kwargs:
            self.update(kwargs)

    def multi_del(self, keys, no_update_log=False):
        """Remove given records from database"""
        opts = (no_update_log and TyrantProtocol.RDBMONOULOG or 0)
        if not isinstance(keys, (list, tuple)):
            keys = list(keys)

        self.proto.misc("outlist", keys, opts)

    def multi_get(self, keys, no_update_log=False):
        """Returns a list of records that match given keys"""
        opts = (no_update_log and TyrantProtocol.RDBMONOULOG or 0)
        if not isinstance(keys, (list, tuple)):
            keys = list(keys)

        rval = self.proto.misc("getlist", keys, opts)

        if len(rval) <= len(keys):
            # 1.1.10 protocol, may return invalid results
            if len(rval) < len(keys):
                raise KeyError("Missing a result, unusable response in 1.1.10")

            return rval

        # 1.1.11 protocol returns interleaved key, value list
        d = dict((rval[i], _parse_elem(rval[i + 1], self.dbtype,
                                       self.separator)) \
                    for i in xrange(0, len(rval), 2))
        return d

    def multi_set(self, items, no_update_log=False):
        """Store given records into database"""
        opts = (no_update_log and TyrantProtocol.RDBMONOULOG or 0)
        lst = []
        for k, v in items.iteritems():
            if isinstance(v, (list, tuple)):
                assert self.separator, "Separator is not set"

                v = self.separator.join(v)
            lst.extend((k, v))

        self.proto.misc("putlist", lst, opts)

    def get_int(self, key):
        """Get an integer for given key. Must been added by addint"""
        return self.proto.getint(key)

    def get_double(self, key):
        """Get a double for given key. Must been added by adddouble"""
        return self.proto.getdouble(key)

    def prefix_keys(self, prefix, maxkeys=None):
        """Get forward matching keys in a database.
        The return value is a list object of the corresponding keys.
        """
        if maxkeys is None:
            maxkeys = len(self)

        return self.proto.fwmkeys(prefix, maxkeys)

    def sync(self):
        """Synchronize updated content into database"""
        self.proto.sync()

    def _get_query(self):
        return Query(self.proto, self.dbtype)

    query = property(_get_query)

class Q(object):
    """Condition object. You can | this type to ORs conditions,
    but you cannot use operand "&", to do this just add more Q to your filter
    Example:

        >>> t = Tyrant()
        >>> t.clear()
        >>> t.dbtype == pyrant.DBTYPETABLE
        True
        >>> t['i'] = {'name': 'Reflejo', 'test': 0}
        >>> t['you'] = {'name': 'Fulano', 'test': 1}
        >>> res = t.query.filter(Q(name='Reflejo'), Q(test=0))
        >>> res[0]['i']['name']
        u'Reflejo'
        >>> res = t.query.filter(Q(name='Reflejo') | Q(name='Fulano'))
        >>> len(res)
        2

    """

    def __init__(self, **kwargs):
        assert kwargs, "You need to specify at least one condition"

        for kw, val in kwargs.iteritems():
            nameop = kw.split('__')
            self._op = 's' if isinstance(val, (str, unicode)) else 'n'
            self._op += nameop[1] if len(nameop) > 1 else 'eq'
            self.name = nameop[0]
            self.expr = val

    def __or__(self, q):
        import copy
        assert isinstance(q, Q), "Unsupported operand type(s) for |"

        op = '%s_or' % q._op
        if q._op == self._op and op in TyrantProtocol.conditionsmap:
            qcopy = copy.copy(q)
            qcopy._op = op
            qcopy.expr = "%s,%s" % (q.expr , self.expr)

            return qcopy
        else:
            raise TypeError("Unsoported operand for |. You can only do this "\
                            "on contains or eq")

    def _getop(self):
        return TyrantProtocol.conditionsmap[self._op]

    op = property(_getop)

    def __repr__(self):
        return "%s [%s] %s" % (self.name, self.op, self.expr)

class Query(object):
    """Query table operations. This is a lazy object
    that abstract all queries for tyrant protocol.

    Usage:

    # Insert object into database
    >>> t = Tyrant()
    >>> t.dbtype == pyrant.DBTYPETABLE
    True
    >>> t['i'] = {'name': 'Reflejo', 'test': 0}

    # Now let's see if we can reach it querying tyrant
    >>> res = t.query.filter(name='Reflejo', test=0).order('name')
    >>> print res[0]['i']['name']
    Reflejo

    """

    def __init__(self, proto, dbtype):
        self._conditions = []
        self._order = None
        self._order_t = 0
        self._cache = {}
        self._proto = proto
        self._dbtype = dbtype

    def order(self, name):
        """Define result order. name parameter is the column name.
        You can prefix "-" to order desc.
        If "#" is added just before column name, column are ordered as numbers

        Examples:
            order('-name')
            order('-#ranking')
            order('name')

        """
        if name.startswith('-'):
            if name.startswith('-#'):
                order = (name[2:], TyrantProtocol.RDBQONUMDESC)
            else:
                order = (name[1:], TyrantProtocol.RDBQOSTRDESC)
        elif name.startswith('#'):
            order = (name[1:], TyrantProtocol.RDBQONUMASC)
        else:
            order = (name, TyrantProtocol.RDBQOSTRASC)

        if self._order != order[0] or self._order_t != order[1]:
            # Add another rule means that our naive cache should be empty'ed
            self._cache = {}
            self._order = order[0]
            self._order_t = order[1]

        return self

    def filter(self, *args, **kwargs):
        """Add condition to query. This could be done by Q object or by keyword
        arguments. Keys are:
            __eq: Equals (default) to expression
            __lt: Less than expression
            __le: Less or equal to expression
            __gt: Greater than expression
            __ge: Greater or equal to expression

        Example:
            >>> t = Tyrant()
            >>> t.get_stats()['type']
            u'table'
            >>> t['i'] = {'name': 'Reflejo', 'test': 4}
            >>> t['i2'] = {'name': 'Reflejo', 'test': 4}
            >>> t['i3'] = {'name': 'Reflejo', 'test': 4}

            >>> res = t.query.filter(test__gt=0)[:1]
            >>> len(res) == 1
            True
            >>> print res[0]['i']['name']
            Reflejo

        """
        # Add another rule means that our naive cache should be empty'ed
        self._cache = {}

        # Iterate arguments. Should be instances of Q
        for q in args:
            assert isinstance(q, Q), "Arguments must be instances of Q"
            self._conditions.append(q)

        # Generate Q with arguments as needed
        for name, expr in kwargs.iteritems():
            self._conditions.append(Q(**{name: expr}))

        return self

    def __len__(self):
        return len(self[:])

    def __repr__(self):
        # Do the query using getitem
        return str(self[:])

    def __getitem__(self, k):
        # Retrieve an item or slice from the set of results.
        if not isinstance(k, (slice, int, long)):
            raise TypeError("ResultSet indices must be integers")

        # Check slice integrity
        assert (not isinstance(k, slice) and (k >= 0)) \
            or (isinstance(k, slice) and (k.start is None or k.start >= 0) \
            and (k.stop is None or k.stop >= 0)), \
            "Negative indexing is not supported."

        if isinstance(k, slice):
            offset = k.start or 0
            limit = (k.stop - offset) if k.stop is not None else MAX_RESULTS
        else:
            offset = k
            limit = 1

        cache_key = "%s_%s" % (offset, limit)
        if cache_key in self._cache:
            return self._cache[cache_key]

        conditions = [(c.name, c.op, c.expr) for c in self._conditions]

        # Do the search.
        keys = self._proto.search(conditions, limit, offset,
                                  order_type=self._order_t,
                                  order_field=self._order)

        # Since results are keys, we need to query for actual values
        if isinstance(k, slice):
            ret = [{key: _parse_elem(self._proto.get(key, self.literal),
                                     self._dbtype)} \
                        for key in keys]
        else:
            ret = {
                keys[0]: _parse_elem(self._proto.get(keys[0], self.literal),
                                     self._dbtype)
            }

        self._cache[cache_key] = ret
        return ret
<commit_msg>Add: .get method allowing to get with a default value. Thanks to Carlo Cabanilla<commit_after>"""
A pythonic python-only implementation of Tokyo Tyrant protocol.
Python 2.4+ is needed.

This library takes a "pythonic" approach to make it more clear and easy
to implement.

More information about Tokyo Cabinet:
    http://tokyocabinet.sourceforge.net/

More information about Tokyo Tyrant:
    http://tokyocabinet.sourceforge.net/tyrantdoc/

This is an usage example:

    >>> import pyrant
    >>> t = pyrant.Tyrant(host='127.0.0.1', port=1978)
    >>> if t.dbtype != pyrant.DBTYPETABLE:
    ...     t['key'] = 'foo'
    ...     print t['key']
    ... else:
    ...     t['key'] = {'name': 'foo'}
    ...     print t['key']['name']
    foo
    >>> del t['key']
    >>> print t['key']
    Traceback (most recent call last):
        ...
    KeyError: 'key'

"""

import itertools as _itertools
from protocol import TyrantProtocol, TyrantError

__version__ = '0.0.2'
__all__ = ['Tyrant', 'TyrantError', 'TyrantProtocol', 'Q']

# Constants
DEFAULT_HOST = '127.0.0.1'
DEFAULT_PORT = 1978
MAX_RESULTS = 1000

# Table Types
DBTYPEBTREE = 'B+ tree'
DBTYPETABLE = 'table'
DBTYPEMEMORY = 'on-memory hash'
DBTYPEHASH = 'hash'

def _parse_elem(elem, dbtype, sep=None):
    if dbtype == DBTYPETABLE:
        # Split element by \x00 which is the column separator
        elems = elem.split('\x00')
        if not elems[0]:
            return None

        return dict((elems[i], elems[i + 1]) \
                        for i in xrange(0, len(elems), 2))
    elif sep and sep in elem:
        return elem.split(sep)

    return elem

class Tyrant(dict):
    """Main class of Tyrant implementation. Acts like a python dictionary,
    so you can query object using normal subscript operations.
    """

    def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, separator=None,
                 literal=False):
        """
        Acts like a python dictionary. Params are:

        host: Tyrant Host address
        port: Tyrant port number
        separator: If this parameter is set, you can put and get lists as
        values.
        literal: If is set string is returned instead of unicode
        """
        # We want to make protocol public just in case anyone need any
        # specific option
        self.proto = TyrantProtocol(host, port)
        self.dbtype = self.get_stats()['type']
        self.separator = separator
        self.literal = literal

    def __contains__(self, key):
        try:
            self.proto.vsiz(key)
        except TyrantError:
            return False
        else:
            return True

    def __delitem__(self, key):
        try:
            return self.proto.out(key)
        except TyrantError:
            raise KeyError(key)

    def __getitem__(self, key):
        try:
            return _parse_elem(self.proto.get(key, self.literal), self.dbtype,
                               self.separator)
        except TyrantError:
            raise KeyError(key)

    def get(self, key, default=None):
        """Allow for getting with a default.

           >>> t = Tyrant()
           >>> t['foo'] = {'a': 'z', 'b': 'y'}
           >>> print t.get('foo', {})
           {u'a': u'z', u'b': u'y'}
           >>> print t.get('bar', {})
           {}

        """
        try:
            return self[key]
        except KeyError:
            return default

    def __len__(self):
        return self.proto.rnum()

    def __repr__(self):
        return object.__repr__(self)

    def __setitem__(self, key, value):
        if isinstance(value, dict):
            flat = _itertools.chain([key], *value.iteritems())
            self.proto.misc('put', list(flat))

        elif isinstance(value, (list, tuple)):
            assert self.separator, "Separator is not set"

            flat = self.separator.join(value)
            self.proto.put(key, flat)

        else:
            self.proto.put(key, value)

    def call_func(self, func, key, value, record_locking=False,
                  global_locking=False):
        """Call specific function.
        """
        opts = ((record_locking and TyrantProtocol.RDBXOLCKREC) |
                (global_locking and TyrantProtocol.RDBXOLCKGLB))
        return self.proto.ext(func, opts, key, value)

    def clear(self):
        """Used in order to remove all records of a remote database object"""
        self.proto.vanish()

    def concat(self, key, value, width=None):
        """Concatenate columns of the existing record"""
        if width is None:
            self.proto.putcat(key, value)
        else:
            self.proto.putshl(key, value, width)

    def get_size(self, key):
        """Get the size of the value of a record"""
        try:
            return self.proto.vsiz(key)
        except TyrantError:
            raise KeyError(key)

    def get_stats(self):
        """Get the status string of the database.
        The return value is the status message of the database.The message
        format is a dictionary.
        """
        return dict(l.split('\t', 1) \
                        for l in self.proto.stat().splitlines() if l)

    def iterkeys(self):
        """Iterate keys using remote operations"""
        self.proto.iterinit()
        try:
            while True:
                yield self.proto.iternext()
        except TyrantError:
            pass

    def keys(self):
        """Return the list of keys in database"""
        return list(self.iterkeys())

    def update(self, other, **kwargs):
        """Update/Add given objects into database"""
        self.multi_set(other.iteritems())
        if kwargs:
            self.update(kwargs)

    def multi_del(self, keys, no_update_log=False):
        """Remove given records from database"""
        opts = (no_update_log and TyrantProtocol.RDBMONOULOG or 0)
        if not isinstance(keys, (list, tuple)):
            keys = list(keys)

        self.proto.misc("outlist", keys, opts)

    def multi_get(self, keys, no_update_log=False):
        """Returns a list of records that match given keys"""
        opts = (no_update_log and TyrantProtocol.RDBMONOULOG or 0)
        if not isinstance(keys, (list, tuple)):
            keys = list(keys)

        rval = self.proto.misc("getlist", keys, opts)

        if len(rval) <= len(keys):
            # 1.1.10 protocol, may return invalid results
            if len(rval) < len(keys):
                raise KeyError("Missing a result, unusable response in 1.1.10")

            return rval

        # 1.1.11 protocol returns interleaved key, value list
        d = dict((rval[i], _parse_elem(rval[i + 1], self.dbtype,
                                       self.separator)) \
                    for i in xrange(0, len(rval), 2))
        return d

    def multi_set(self, items, no_update_log=False):
        """Store given records into database"""
        opts = (no_update_log and TyrantProtocol.RDBMONOULOG or 0)
        lst = []
        for k, v in items.iteritems():
            if isinstance(v, (list, tuple)):
                assert self.separator, "Separator is not set"

                v = self.separator.join(v)
            lst.extend((k, v))

        self.proto.misc("putlist", lst, opts)

    def get_int(self, key):
        """Get an integer for given key. Must been added by addint"""
        return self.proto.getint(key)

    def get_double(self, key):
        """Get a double for given key. Must been added by adddouble"""
        return self.proto.getdouble(key)

    def prefix_keys(self, prefix, maxkeys=None):
        """Get forward matching keys in a database.
        The return value is a list object of the corresponding keys.
        """
        if maxkeys is None:
            maxkeys = len(self)

        return self.proto.fwmkeys(prefix, maxkeys)

    def sync(self):
        """Synchronize updated content into database"""
        self.proto.sync()

    def _get_query(self):
        return Query(self.proto, self.dbtype, self.literal)

    query = property(_get_query)

class Q(object):
    """Condition object. You can | this type to ORs conditions,
    but you cannot use operand "&", to do this just add more Q to your filter
    Example:

        >>> t = Tyrant()
        >>> t.clear()
        >>> t.dbtype == pyrant.DBTYPETABLE
        True
        >>> t['i'] = {'name': 'Reflejo', 'test': 0}
        >>> t['you'] = {'name': 'Fulano', 'test': 1}
        >>> res = t.query.filter(Q(name='Reflejo'), Q(test=0))
        >>> res[0]['i']['name']
        u'Reflejo'
        >>> res = t.query.filter(Q(name='Reflejo') | Q(name='Fulano'))
        >>> len(res)
        2

    """

    def __init__(self, **kwargs):
        assert kwargs, "You need to specify at least one condition"

        for kw, val in kwargs.iteritems():
            nameop = kw.split('__')
            self._op = 's' if isinstance(val, (str, unicode)) else 'n'
            self._op += nameop[1] if len(nameop) > 1 else 'eq'
            self.name = nameop[0]
            self.expr = val

    def __or__(self, q):
        import copy
        assert isinstance(q, Q), "Unsupported operand type(s) for |"

        op = '%s_or' % q._op
        if q._op == self._op and op in TyrantProtocol.conditionsmap:
            qcopy = copy.copy(q)
            qcopy._op = op
            qcopy.expr = "%s,%s" % (q.expr , self.expr)

            return qcopy
        else:
            raise TypeError("Unsoported operand for |. You can only do this "\
                            "on contains or eq")

    def _getop(self):
        return TyrantProtocol.conditionsmap[self._op]

    op = property(_getop)

    def __repr__(self):
        return "%s [%s] %s" % (self.name, self.op, self.expr)

class Query(object):
    """Query table operations. This is a lazy object
    that abstract all queries for tyrant protocol.

    Usage:

    # Insert object into database
    >>> t = Tyrant()
    >>> t.dbtype == pyrant.DBTYPETABLE
    True
    >>> t['i'] = {'name': 'Reflejo', 'test': 0}

    # Now let's see if we can reach it querying tyrant
    >>> res = t.query.filter(name='Reflejo', test=0).order('name')
    >>> print res[0]['i']['name']
    Reflejo

    """

    def __init__(self, proto, dbtype, literal=False):
        self._conditions = []
        self._order = None
        self._order_t = 0
        self._cache = {}
        self._proto = proto
        self._dbtype = dbtype
        self.literal = literal

    def order(self, name):
        """Define result order. name parameter is the column name.
        You can prefix "-" to order desc.
        If "#" is added just before column name, column are ordered as numbers

        Examples:
            order('-name')
            order('-#ranking')
            order('name')

        """
        if name.startswith('-'):
            if name.startswith('-#'):
                order = (name[2:], TyrantProtocol.RDBQONUMDESC)
            else:
                order = (name[1:], TyrantProtocol.RDBQOSTRDESC)
        elif name.startswith('#'):
            order = (name[1:], TyrantProtocol.RDBQONUMASC)
        else:
            order = (name, TyrantProtocol.RDBQOSTRASC)

        if self._order != order[0] or self._order_t != order[1]:
            # Add another rule means that our naive cache should be empty'ed
            self._cache = {}
            self._order = order[0]
            self._order_t = order[1]

        return self

    def filter(self, *args, **kwargs):
        """Add condition to query. This could be done by Q object or by keyword
        arguments. Keys are:
            __eq: Equals (default) to expression
            __lt: Less than expression
            __le: Less or equal to expression
            __gt: Greater than expression
            __ge: Greater or equal to expression

        Example:
            >>> t = Tyrant()
            >>> t.get_stats()['type']
            u'table'
            >>> t['i'] = {'name': 'Reflejo', 'test': 4}
            >>> t['i2'] = {'name': 'Reflejo', 'test': 4}
            >>> t['i3'] = {'name': 'Reflejo', 'test': 4}

            >>> res = t.query.filter(test__gt=0)[:1]
            >>> len(res) == 1
            True
            >>> print res[0]['i']['name']
            Reflejo

        """
        # Add another rule means that our naive cache should be empty'ed
        self._cache = {}

        # Iterate arguments. Should be instances of Q
        for q in args:
            assert isinstance(q, Q), "Arguments must be instances of Q"
            self._conditions.append(q)

        # Generate Q with arguments as needed
        for name, expr in kwargs.iteritems():
            self._conditions.append(Q(**{name: expr}))

        return self

    def __len__(self):
        return len(self[:])

    def __repr__(self):
        # Do the query using getitem
        return str(self[:])

    def __getitem__(self, k):
        # Retrieve an item or slice from the set of results.
        if not isinstance(k, (slice, int, long)):
            raise TypeError("ResultSet indices must be integers")

        # Check slice integrity
        assert (not isinstance(k, slice) and (k >= 0)) \
            or (isinstance(k, slice) and (k.start is None or k.start >= 0) \
            and (k.stop is None or k.stop >= 0)), \
            "Negative indexing is not supported."

        if isinstance(k, slice):
            offset = k.start or 0
            limit = (k.stop - offset) if k.stop is not None else MAX_RESULTS
        else:
            offset = k
            limit = 1

        cache_key = "%s_%s" % (offset, limit)
        if cache_key in self._cache:
            return self._cache[cache_key]

        conditions = [(c.name, c.op, c.expr) for c in self._conditions]

        # Do the search.
        keys = self._proto.search(conditions, limit, offset,
                                  order_type=self._order_t,
                                  order_field=self._order)

        # Since results are keys, we need to query for actual values
        if isinstance(k, slice):
            ret = [{key: _parse_elem(self._proto.get(key, self.literal),
                                     self._dbtype)} \
                        for key in keys]
        else:
            ret = {
                keys[0]: _parse_elem(self._proto.get(keys[0], self.literal),
                                     self._dbtype)
            }

        self._cache[cache_key] = ret
        return ret
<|endoftext|>


<commit_before>import random
import re
import time
import os
import traceback
import requests
from modules import ircbot

userdata_file = os.getcwd() + '/user_data/dikantenyvaovao/'
nwikimax = 5

def irc_retrieve(channel=''):
    bot = LiveRecentChangesBot(channel)
    while 1:
        try:
            bot.start()
        except KeyboardInterrupt:
            bot.die()
            return

class IrcBotException(Exception):
    pass

class LiveRecentChangesBot(ircbot.SingleServerIRCBot):
    """IRC client used to track edits on targetted wikis.
    @lang string containing languages of editions to track.
    For example : fr,en,de will track fr.wiktionary, en.wiktionary and de.wiktionary
    @user the username which will be use by the IRC client
    """

    def __init__(self, lang, user="botjagwar-w%x" % random.randint(1, 0xffff)):
        self.channels_list = []
        self.chronometer = 0.0
        self.change_langs(lang)
        self.service_address = "http://localhost:8000"
        try:
            self.errfile = open(userdata_file + 'dikantenyvaovao.exceptions', 'a')
        except IOError:
            self.errfile = open(userdata_file + 'dikantenyvaovao.exceptions', 'w')
        self.iso2languagename = {}
        self.joined = []
        self.langs = []
        self.stats = {'edits': 0.0, 'newentries': 0.0, 'errors': 0.0}
        self.edits = 0
        self.username = user

        self.connect_in_languages()

    def change_langs(self, langstring):
        self.langs = re.split("\,[ ]?", langstring)

    def connect_in_languages(self):
        """mametaka fitohizana amin'ny tsanely irc an'i Wikimedia"""
        print ("\n---------------------\n       PARAMETATRA : ")
        self.langs = ['en', 'fr']
        i = 0
        for language in self.langs:
            if i > nwikimax:
                break
            i += 1
            if language == 'mg':
                continue
            language = language.strip()
            channel = "#%s.wiktionary" % language
            print ("kaodim-piteny:", language, ", tsanely:", channel, " anarana:", self.username)

            irc_bot = ircbot.SingleServerIRCBot.__init__(
                self, [("irc.wikimedia.org", 6667)], self.username, "Bot-Jagwar [IRCbot v2].")
            self.joined.append(language)
            self.channels_list.append(irc_bot)
        print ("Vita ny fampitohizana")

    def on_welcome(self, serv, ev):
        for language in self.joined:
            # print ("Mangataka tonga soa avy amin'i #"+language+".wiktionary")
            serv.join("#" + language + ".wiktionary")

    def on_kick(self, serv, ev):
        for language in self.joined:
            print ("Voadaka. Mangataka tonga soa avy amin'i #" + language + ".wiktionary")
            serv.join("#" + language + ".wiktionary")

    def on_pubmsg(self, serv, ev):
        try:
            ct_time = time.time()
            msg = _prepare_message(ev)
            wikilanguage = _get_origin_wiki(msg)
            pagename = _get_pagename(msg)
            url = self.service_address + "/wiktionary_page/%s" % (wikilanguage)
            data = {u'title': pagename}
            requests.post(url, data=data)
            self.edits += 1
            if not self.edits % 5:
                throughput = 60. * 5. / (float(ct_time) - self.chronometer)
                self.chronometer = ct_time
                print ("Fiovana faha-%d (fiovana %.2f / min)" % (self.edits, throughput))

        except Exception as e:
            print (traceback.format_exc())
            self.stats['errors'] += 1
            errstr = u"\n%s" % e.message
            self.errfile.write(errstr.encode('utf8'))

def _get_origin_wiki(message):
    try:
        lang = re.search('//([a-z|\-]+).wiktionary', message).groups()[0]
        return lang
    except AttributeError:
        lang = 'fr'  # fiteny defaulta
        return lang

def _get_pagename(message):
    message = message[:message.find('http')]
    item = re.search("\[\[(.*)\]\]", message).groups()[0]
    item = unicode(item[3:-3])
    if len(item) > 200:
        print (item)
        raise IrcBotException("Title is too long")
    return item

def _get_message_type(message):
    if "Log/delete" in message:
        return "delete"
    else:
        return "edit"

def _prepare_message(ev):
    try:
        message = ev.arguments()[0].decode('utf8')
    except UnicodeDecodeError:
        message = ev.arguments()[0].decode('latin1')
    return message

if __name__ == '__main__':
    try:
        irc_retrieve()
    finally:
        print "bye"<commit_msg>update & fix service<commit_after>import random
import re
import time
import os
import traceback
import requests
from modules import ircbot

userdata_file = os.getcwd() + '/user_data/dikantenyvaovao/'
nwikimax = 5

def irc_retrieve(channel=''):
    bot = LiveRecentChangesBot(channel)
    while 1:
        try:
            bot.start()
        except KeyboardInterrupt:
            bot.die()
            return

class IrcBotException(Exception):
    pass

class LiveRecentChangesBot(ircbot.SingleServerIRCBot):
    """IRC client used to track edits on targetted wikis.
    @lang string containing languages of editions to track.
    For example : fr,en,de will track fr.wiktionary, en.wiktionary and de.wiktionary
    @user the username which will be use by the IRC client
    """

    def __init__(self, lang, user="botjagwar-w%x" % random.randint(1, 0xffff)):
        self.channels_list = []
        self.chronometer = 0.0
        self.change_langs(lang)
        self.service_address = "http://localhost:8000"
        try:
            self.errfile = open(userdata_file + 'dikantenyvaovao.exceptions', 'a')
        except IOError:
            self.errfile = open(userdata_file + 'dikantenyvaovao.exceptions', 'w')
        self.iso2languagename = {}
        self.joined = []
        self.langs = []
        self.stats = {'edits': 0.0, 'newentries': 0.0, 'errors': 0.0}
        self.edits = 0
        self.username = user

        self.connect_in_languages()

    def change_langs(self, langstring):
        self.langs = re.split("\,[ ]?", langstring)

    def connect_in_languages(self):
        """mametaka fitohizana amin'ny tsanely irc an'i Wikimedia"""
        print ("\n---------------------\n       PARAMETATRA : ")
        self.langs = ['en', 'fr']
        i = 0
        for language in self.langs:
            if i > nwikimax:
                break
            i += 1
            if language == 'mg':
                continue
            language = language.strip()
            channel = "#%s.wiktionary" % language
            print ("kaodim-piteny:", language, ", tsanely:", channel, " anarana:", self.username)

            irc_bot = ircbot.SingleServerIRCBot.__init__(
                self, [("irc.wikimedia.org", 6667)], self.username, "Bot-Jagwar [IRCbot v2].")
            self.joined.append(language)
            self.channels_list.append(irc_bot)
        print ("Vita ny fampitohizana")

    def on_welcome(self, serv, ev):
        for language in self.joined:
            # print ("Mangataka tonga soa avy amin'i #"+language+".wiktionary")
            serv.join("#" + language + ".wiktionary")

    def on_kick(self, serv, ev):
        for language in self.joined:
            print ("Voadaka. Mangataka tonga soa avy amin'i #" + language + ".wiktionary")
            serv.join("#" + language + ".wiktionary")

    def on_pubmsg(self, serv, ev):
        try:
            ct_time = time.time()
            msg = _prepare_message(ev)
            wikilanguage = _get_origin_wiki(msg)
            pagename = _get_pagename(msg)
            url = self.service_address + "/wiktionary_page/%s" % (wikilanguage)
            data = {u'title': pagename}
            requests.post(url, json=data)
            self.edits += 1
            if not self.edits % 5:
                throughput = 60. * 5. / (float(ct_time) - self.chronometer)
                self.chronometer = ct_time
                print ("Fiovana faha-%d (fiovana %.2f / min)" % (self.edits, throughput))

        except Exception as e:
            print (traceback.format_exc())
            self.stats['errors'] += 1
            errstr = u"\n%s" % e.message
            self.errfile.write(errstr.encode('utf8'))

def _get_origin_wiki(message):
    try:
        lang = re.search('//([a-z|\-]+).wiktionary', message).groups()[0]
        return lang
    except AttributeError:
        lang = 'fr'  # fiteny defaulta
        return lang

def _get_pagename(message):
    message = message[:message.find('http')]
    item = re.search("\[\[(.*)\]\]", message).groups()[0]
    item = unicode(item[3:-3])
    if len(item) > 200:
        print (item)
        raise IrcBotException("Title is too long")
    return item

def _get_message_type(message):
    if "Log/delete" in message:
        return "delete"
    else:
        return "edit"

def _prepare_message(ev):
    try:
        message = ev.arguments()[0].decode('utf8')
    except UnicodeDecodeError:
        message = ev.arguments()[0].decode('latin1')
    return message

if __name__ == '__main__':
    try:
        irc_retrieve()
    finally:
        print "bye"<|endoftext|>


<commit_before>'''
Common utilities
'''
import re
import os
import sys
import struct
import logging
import platform
import threading
import traceback

from socket import AF_INET
from socket import inet_pton
from socket import inet_ntop
from socket import inet_aton

try:
    basestring = basestring
except NameError:
    basestring = (str, bytes)

AF_PIPE = 255  # Right now AF_MAX == 40
DEFAULT_RCVBUF = 16384
ANCIENT = (platform.dist()[0] in ('redhat', 'centos') and
           platform.dist()[1].startswith('6.') or
           os.environ.get('PYROUTE2_ANCIENT', False))

size_suffixes = {'b': 1,
                 'k': 1024,
                 'kb': 1024,
                 'm': 1024 * 1024,
                 'mb': 1024 * 1024,
                 'g': 1024 * 1024 * 1024,
                 'gb': 1024 * 1024 * 1024,
                 'kbit': 1024 / 8,
                 'mbit': 1024 * 1024 / 8,
                 'gbit': 1024 * 1024 * 1024 / 8}

time_suffixes = {'s': 1,
                 'sec': 1,
                 'secs': 1,
                 'ms': 1000,
                 'msec': 1000,
                 'msecs': 1000,
                 'us': 1000000,
                 'usec': 1000000,
                 'usecs': 1000000}

rate_suffixes = {'bit': 1,
                 'Kibit': 1024,
                 'kbit': 1000,
                 'mibit': 1024 * 1024,
                 'mbit': 1000000,
                 'gibit': 1024 * 1024 * 1024,
                 'gbit': 1000000000,
                 'tibit': 1024 * 1024 * 1024 * 1024,
                 'tbit': 1000000000000,
                 'Bps': 8,
                 'KiBps': 8 * 1024,
                 'KBps': 8000,
                 'MiBps': 8 * 1024 * 1024,
                 'MBps': 8000000,
                 'GiBps': 8 * 1024 * 1024 * 1024,
                 'GBps': 8000000000,
                 'TiBps': 8 * 1024 * 1024 * 1024 * 1024,
                 'TBps': 8000000000000}

##
# Debug
#
_log_configured = False

def debug(f):
    '''
    Debug decorator, that logs all the function calls
    '''
    if os.environ.get('DEBUG', None) is None:
        return f

    global _log_configured
    # some nosetests bug prevents the rest of this function
    # being marked as "covered", despite it really IS; so
    # FIXME: trace and file nosetests bug
    # possibly related to the log capture
    if not _log_configured:
        _log_configured = True
        filename = os.environ.get('DEBUG_FILE', None)
        logging.basicConfig(filename=filename,
                            level=logging.DEBUG)

    def wrap(*argv, **kwarg):
        tid = id(threading.current_thread())
        bt = ''.join(traceback.format_stack()[:-1])
        logging.debug("%s %s: bt: %s" % (tid, f.__name__, bt))
        logging.debug("%s %s: call: %s; %s" % (tid,
                                               f.__name__,
                                               argv,
                                               kwarg))
        ret = f(*argv, **kwarg)
        logging.debug("%s %s: ret: %s" % (tid, f.__name__, ret))
        return ret

    return wrap

##
# General purpose
#
class Dotkeys(dict):
    '''
    This is a sick-minded hack of dict, intended to be an eye-candy.
    It allows to get dict's items byt dot reference:

    ipdb["lo"] == ipdb.lo
    ipdb["eth0"] == ipdb.eth0

    Obviously, it will not work for some cases, like unicode names
    of interfaces and so on. Beside of that, it introduces some
    complexity.

    But it simplifies live for old-school admins, who works with good
    old "lo", "eth0", and like that naming schemes.
    '''
    var_name = re.compile('^[a-zA-Z_]+[a-zA-Z_0-9]*$')

    def __dir__(self):
        return [i for i in self if
                type(i) == str and self.var_name.match(i)]

    def __getattribute__(self, key, *argv):
        try:
            return dict.__getattribute__(self, key)
        except AttributeError as e:
            if key == '__deepcopy__':
                raise e
            return self[key]

    def __setattr__(self, key, value):
        if key in self:
            self[key] = value
        else:
            dict.__setattr__(self, key, value)

    def __delattr__(self, key):
        if key in self:
            del self[key]
        else:
            dict.__delattr__(self, key)

def map_namespace(prefix, ns):
    '''
    Take the namespace prefix, list all constants and build two
    dictionaries -- straight and reverse mappings. E.g.:

    ## neighbor attributes
    NDA_UNSPEC = 0
    NDA_DST = 1
    NDA_LLADDR = 2
    NDA_CACHEINFO = 3
    NDA_PROBES = 4
    (NDA_NAMES, NDA_VALUES) = map_namespace('NDA', globals())

    Will lead to:

    NDA_NAMES = {'NDA_UNSPEC': 0,
                 ...
                 'NDA_PROBES': 4}
    NDA_VALUES = {0: 'NDA_UNSPEC',
                  ...
                  4: 'NDA_PROBES'}

    '''
    by_name = dict([(i, ns[i]) for i in ns.keys() if i.startswith(prefix)])
    by_value = dict([(ns[i], i) for i in ns.keys() if i.startswith(prefix)])
    return (by_name, by_value)

def list_subnet(dqn, mask, family=AF_INET):
    '''
    List all IPs in the network
    '''
    if family != AF_INET:
        raise NotImplementedError('please report the issue')

    ret = []
    net = struct.unpack('>I', inet_pton(family, dqn))[0]
    shift = 32 - mask
    for host in range(1, 2 ** shift):
        ret.append(inet_ntop(family, struct.pack('>I', net | host)))
    return ret

def dqn2int(mask):
    '''
    IPv4 dotted quad notation to int mask conversion
    '''
    return bin(struct.unpack('>L', inet_aton(mask))[0]).count('1')

def hexdump(payload, length=0):
    '''
    Represent byte string as hex -- for debug purposes
    '''
    if sys.version[0] == '3':
        return ':'.join('{0:02x}'.format(c)
                        for c in payload[:length] or payload)
    else:
        return ':'.join('{0:02x}'.format(ord(c))
                        for c in payload[:length] or payload)

class AddrPool(object):
    '''
    Address pool
    '''
    cell = 0xffffffffffffffff

    def __init__(self, minaddr=0xf, maxaddr=0xffffff, reverse=False):
        self.cell_size = 0  # in bits
        mx = self.cell
        self.reverse = reverse
        self.ban = []
        while mx:
            mx >>= 8
            self.cell_size += 1
        self.cell_size *= 8
        # calculate, how many ints we need to bitmap all addresses
        self.cells = int((maxaddr - minaddr) / self.cell_size + 1)
        # initial array
        self.addr_map = [self.cell]
        self.minaddr = minaddr
        self.maxaddr = maxaddr
        self.lock = threading.RLock()

    def alloc(self):
        with self.lock:
            # gc self.ban:
            for item in tuple(self.ban):
                if item['counter'] == 0:
                    self.free(item['addr'])
                    self.ban.remove(item)
                else:
                    item['counter'] -= 1

            # iterate through addr_map
            base = 0
            for cell in self.addr_map:
                if cell:
                    # not allocated addr
                    bit = 0
                    while True:
                        if (1 << bit) & self.addr_map[base]:
                            self.addr_map[base] ^= 1 << bit
                            break
                        bit += 1
                    ret = (base * self.cell_size + bit)

                    if self.reverse:
                        ret = self.maxaddr - ret
                    else:
                        ret = ret + self.minaddr

                    if self.minaddr <= ret <= self.maxaddr:
                        return ret
                    else:
                        self.free(ret)
                        raise KeyError('no free address available')

                base += 1
            # no free address available
            if len(self.addr_map) < self.cells:
                # create new cell to allocate address from
                self.addr_map.append(self.cell)
                return self.alloc()
            else:
                raise KeyError('no free address available')

    def free(self, addr, ban=0):
        with self.lock:
            if ban != 0:
                self.ban.append({'addr': addr,
                                 'counter': ban})
            else:
                if self.reverse:
                    addr = self.maxaddr - addr
                else:
                    addr -= self.minaddr
                base = addr // self.cell_size
                bit = addr % self.cell_size
                if len(self.addr_map) <= base:
                    raise KeyError('address is not allocated')
                if self.addr_map[base] & (1 << bit):
                    raise KeyError('address is not allocated')
                self.addr_map[base] ^= 1 << bit
<commit_msg>common: remove unused code<commit_after>'''
Common utilities
'''
import re
import os
import sys
import struct
import platform
import threading

from socket import inet_aton

try:
    basestring = basestring
except NameError:
    basestring = (str, bytes)

AF_PIPE = 255  # Right now AF_MAX == 40
DEFAULT_RCVBUF = 16384
ANCIENT = (platform.dist()[0] in ('redhat', 'centos') and
           platform.dist()[1].startswith('6.') or
           os.environ.get('PYROUTE2_ANCIENT', False))

size_suffixes = {'b': 1,
                 'k': 1024,
                 'kb': 1024,
                 'm': 1024 * 1024,
                 'mb': 1024 * 1024,
                 'g': 1024 * 1024 * 1024,
                 'gb': 1024 * 1024 * 1024,
                 'kbit': 1024 / 8,
                 'mbit': 1024 * 1024 / 8,
                 'gbit': 1024 * 1024 * 1024 / 8}

time_suffixes = {'s': 1,
                 'sec': 1,
                 'secs': 1,
                 'ms': 1000,
                 'msec': 1000,
                 'msecs': 1000,
                 'us': 1000000,
                 'usec': 1000000,
                 'usecs': 1000000}

rate_suffixes = {'bit': 1,
                 'Kibit': 1024,
                 'kbit': 1000,
                 'mibit': 1024 * 1024,
                 'mbit': 1000000,
                 'gibit': 1024 * 1024 * 1024,
                 'gbit': 1000000000,
                 'tibit': 1024 * 1024 * 1024 * 1024,
                 'tbit': 1000000000000,
                 'Bps': 8,
                 'KiBps': 8 * 1024,
                 'KBps': 8000,
                 'MiBps': 8 * 1024 * 1024,
                 'MBps': 8000000,
                 'GiBps': 8 * 1024 * 1024 * 1024,
                 'GBps': 8000000000,
                 'TiBps': 8 * 1024 * 1024 * 1024 * 1024,
                 'TBps': 8000000000000}

##
# General purpose
#
class Dotkeys(dict):
    '''
    This is a sick-minded hack of dict, intended to be an eye-candy.
    It allows to get dict's items byt dot reference:

    ipdb["lo"] == ipdb.lo
    ipdb["eth0"] == ipdb.eth0

    Obviously, it will not work for some cases, like unicode names
    of interfaces and so on. Beside of that, it introduces some
    complexity.

    But it simplifies live for old-school admins, who works with good
    old "lo", "eth0", and like that naming schemes.
    '''
    var_name = re.compile('^[a-zA-Z_]+[a-zA-Z_0-9]*$')

    def __dir__(self):
        return [i for i in self if
                type(i) == str and self.var_name.match(i)]

    def __getattribute__(self, key, *argv):
        try:
            return dict.__getattribute__(self, key)
        except AttributeError as e:
            if key == '__deepcopy__':
                raise e
            return self[key]

    def __setattr__(self, key, value):
        if key in self:
            self[key] = value
        else:
            dict.__setattr__(self, key, value)

    def __delattr__(self, key):
        if key in self:
            del self[key]
        else:
            dict.__delattr__(self, key)

def map_namespace(prefix, ns):
    '''
    Take the namespace prefix, list all constants and build two
    dictionaries -- straight and reverse mappings. E.g.:

    ## neighbor attributes
    NDA_UNSPEC = 0
    NDA_DST = 1
    NDA_LLADDR = 2
    NDA_CACHEINFO = 3
    NDA_PROBES = 4
    (NDA_NAMES, NDA_VALUES) = map_namespace('NDA', globals())

    Will lead to:

    NDA_NAMES = {'NDA_UNSPEC': 0,
                 ...
                 'NDA_PROBES': 4}
    NDA_VALUES = {0: 'NDA_UNSPEC',
                  ...
                  4: 'NDA_PROBES'}

    '''
    by_name = dict([(i, ns[i]) for i in ns.keys() if i.startswith(prefix)])
    by_value = dict([(ns[i], i) for i in ns.keys() if i.startswith(prefix)])
    return (by_name, by_value)

def dqn2int(mask):
    '''
    IPv4 dotted quad notation to int mask conversion
    '''
    return bin(struct.unpack('>L', inet_aton(mask))[0]).count('1')

def hexdump(payload, length=0):
    '''
    Represent byte string as hex -- for debug purposes
    '''
    if sys.version[0] == '3':
        return ':'.join('{0:02x}'.format(c)
                        for c in payload[:length] or payload)
    else:
        return ':'.join('{0:02x}'.format(ord(c))
                        for c in payload[:length] or payload)

class AddrPool(object):
    '''
    Address pool
    '''
    cell = 0xffffffffffffffff

    def __init__(self, minaddr=0xf, maxaddr=0xffffff, reverse=False):
        self.cell_size = 0  # in bits
        mx = self.cell
        self.reverse = reverse
        self.ban = []
        while mx:
            mx >>= 8
            self.cell_size += 1
        self.cell_size *= 8
        # calculate, how many ints we need to bitmap all addresses
        self.cells = int((maxaddr - minaddr) / self.cell_size + 1)
        # initial array
        self.addr_map = [self.cell]
        self.minaddr = minaddr
        self.maxaddr = maxaddr
        self.lock = threading.RLock()

    def alloc(self):
        with self.lock:
            # gc self.ban:
            for item in tuple(self.ban):
                if item['counter'] == 0:
                    self.free(item['addr'])
                    self.ban.remove(item)
                else:
                    item['counter'] -= 1

            # iterate through addr_map
            base = 0
            for cell in self.addr_map:
                if cell:
                    # not allocated addr
                    bit = 0
                    while True:
                        if (1 << bit) & self.addr_map[base]:
                            self.addr_map[base] ^= 1 << bit
                            break
                        bit += 1
                    ret = (base * self.cell_size + bit)

                    if self.reverse:
                        ret = self.maxaddr - ret
                    else:
                        ret = ret + self.minaddr

                    if self.minaddr <= ret <= self.maxaddr:
                        return ret
                    else:
                        self.free(ret)
                        raise KeyError('no free address available')

                base += 1
            # no free address available
            if len(self.addr_map) < self.cells:
                # create new cell to allocate address from
                self.addr_map.append(self.cell)
                return self.alloc()
            else:
                raise KeyError('no free address available')

    def free(self, addr, ban=0):
        with self.lock:
            if ban != 0:
                self.ban.append({'addr': addr,
                                 'counter': ban})
            else:
                if self.reverse:
                    addr = self.maxaddr - addr
                else:
                    addr -= self.minaddr
                base = addr // self.cell_size
                bit = addr % self.cell_size
                if len(self.addr_map) <= base:
                    raise KeyError('address is not allocated')
                if self.addr_map[base] & (1 << bit):
                    raise KeyError('address is not allocated')
                self.addr_map[base] ^= 1 << bit
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import
from __future__ import unicode_literals

from .table import Table
from . import loaders, parsers, errors, helpers

DEFAULT_SCHEME = 'file'

LOADERS = {
    'file': loaders.File,
    'text': loaders.Text,
    'ftp': loaders.Web,
    'ftps': loaders.Web,
    'http': loaders.Web,
    'https': loaders.Web,
}

PARSERS = {
    'csv': parsers.CSV,
    'xls': parsers.Excel,
    'xlsx': parsers.Excel,
    'json': parsers.JSON,
}

def topen(source,
          scheme=None, format=None, encoding=None,
          loader_options=None, parser_options=None):
    """Open table from source with scheme, encoding and format.

    Function `topen` is a wrapper around `Table` interface.

    Parameters
    ----------
    source: str
        Path of contents.
    scheme: str
        Scheme of source:
            - file (default)
            - text
            - http
            - https
            - ftp
            - ftps
    format: str
        Format of source:
            - None (detect)
            - csv
            - json
            - xls
            - xlsx
    encoding: str
        Encoding of source:
            - None (detect)
            - utf-8
            - <any>
    loader_options: dict
        Loader options.
    parser_options: dict
        Parser options.

    Returns
    -------
    table: `Table`
        Opened Table instance.

    """
    # Initiate if None
    if loader_options is None:
        loader_options = {}
    if parser_options is None:
        parser_options = {}

    # Get scheme, format
    if scheme is None:
        scheme = helpers.detect_scheme(source) or DEFAULT_SCHEME
    if format is None:
        format = helpers.detect_format(source)

    # Lookup and initiate loader
    if scheme not in LOADERS:
        message = 'Scheme "%s" is not supported' % scheme
        raise errors.Error(message)
    loader = LOADERS[scheme](source, encoding, **loader_options)

    # Lookup and initiate parser
    if format not in PARSERS:
        message = 'Format "%s" is not supported' % format
        raise errors.Error(message)
    parser = PARSERS[format]()

    # Initiate, open table
    table = Table(loader=loader, parser=parser, **parser_options)
    table.open()

    return table
<commit_msg>[#16] loader_class, parser_class params to topen<commit_after># -*- coding: utf-8 -*-
from __future__ import division
from __future__ import print_function
from __future__ import absolute_import
from __future__ import unicode_literals

from .table import Table
from . import loaders, parsers, errors, helpers

DEFAULT_SCHEME = 'file'

LOADERS = {
    'file': loaders.File,
    'text': loaders.Text,
    'ftp': loaders.Web,
    'ftps': loaders.Web,
    'http': loaders.Web,
    'https': loaders.Web,
}

PARSERS = {
    'csv': parsers.CSV,
    'xls': parsers.Excel,
    'xlsx': parsers.Excel,
    'json': parsers.JSON,
}

def topen(source,
          scheme=None, format=None, encoding=None,
          loader_options=None, parser_options=None,
          loader_class=None, parser_class=None):
    """Open table from source with scheme, encoding and format.

    Function `topen` is a wrapper around `Table` interface.

    Parameters
    ----------
    source: str
        Path of contents.
    scheme: str
        Scheme of source:
            - file (default)
            - text
            - http
            - https
            - ftp
            - ftps
    format: str
        Format of source:
            - None (detect)
            - csv
            - json
            - xls
            - xlsx
    encoding: str
        Encoding of source:
            - None (detect)
            - utf-8
            - <any>
    loader_options: dict
        Loader options.
    parser_options: dict
        Parser options.
    loader_class: type
        Loader class.
    parser_class: type
        Parser class.

    Returns
    -------
    table: `Table`
        Opened Table instance.

    """
    # Initiate if None
    if loader_options is None:
        loader_options = {}
    if parser_options is None:
        parser_options = {}

    # Get loader
    if loader_class is None:
        if scheme is None:
            scheme = helpers.detect_scheme(source) or DEFAULT_SCHEME
        if scheme not in LOADERS:
            message = 'Scheme "%s" is not supported' % scheme
            raise errors.Error(message)
        loader_class = LOADERS[scheme]
    loader = loader_class(source, encoding, **loader_options)

    # Get parser
    if parser_class is None:
        if format is None:
            format = helpers.detect_format(source)
        if format not in PARSERS:
            message = 'Format "%s" is not supported' % format
            raise errors.Error(message)
        parser_class = PARSERS[format]
    parser = parser_class(**parser_options)

    # Initiate and open table
    table = Table(loader=loader, parser=parser)
    table.open()

    return table
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Kernel of Pyslvs.

This kernel can work without GUI.

Modules:
+ Solver:
    + parser
    + tinycadlib
        + Sketch Solve solver
    + triangulation
+ Number synthesis:
    + number
+ Structure Synthesis:
    + atlas
+ Dimensional synthesis:
    + planarlinkage
    + rga
    + firefly
    + de

Dependents:
+ lark-parser
+ pygments (optional: provide highlighting)
"""

__author__ = "Yuan Chang"
__copyright__ = "Copyright (C) 2016-2019"
__license__ = "AGPL"
__email__ = "pyslvs@gmail.com"
__version__ = "19.09.0"

from .expression import (
    get_vlinks,
    VJoint,
    VPoint,
    VLink,
    Coordinate,
)
from .bfgs import SolverSystem
from .triangulation import vpoints_configure, ExpressionStack
from .tinycadlib import (
    plap,
    pllp,
    plpp,
    pxy,
    vpoint_dof,
    expr_parser,
    expr_solving,
    data_collecting,
)
from .Adesign import (
    Verification,
    Genetic,
    Firefly,
    Differential,
)
from .planar_linkage import Planar
from .number import link_synthesis, contracted_link_synthesis
from .graph import (
    Graph,
    link_assortment,
    contracted_link_assortment,
    labeled_enumerate,
)
from .planar_check import is_planar
from .graph_layout import external_loop_layout
from .atlas import conventional_graph, contracted_graph
from .expression_parser import (
    color_names,
    color_rgb,
    parse_params,
    parse_pos,
    parse_vpoints,
    parse_vlinks,
    edges_view,
    graph2vpoints,
    HAS_PYGMENTS,
)
from .example import example_list
from .collection import collection_list
if HAS_PYGMENTS:
    from .expression_parser import PMKSLexer

__all__ = [
    '__version__',
    'Genetic',
    'Firefly',
    'Differential',
    'Coordinate',
    'plap',
    'pllp',
    'plpp',
    'pxy',
    'expr_parser',
    'expr_solving',
    'data_collecting',
    'get_vlinks',
    'VJoint',
    'VPoint',
    'VLink',
    'SolverSystem',
    'Planar',
    'link_synthesis',
    'contracted_link_synthesis',
    'is_planar',
    'external_loop_layout',
    'conventional_graph',
    'contracted_graph',
    'Graph',
    'link_assortment',
    'contracted_link_assortment',
    'labeled_enumerate',
    'vpoints_configure',
    'ExpressionStack',
    'vpoint_dof',
    'color_names',
    'color_rgb',
    'parse_params',
    'parse_pos',
    'parse_vpoints',
    'parse_vlinks',
    'edges_view',
    'graph2vpoints',
    'PMKSLexer',
    'example_list',
    'collection_list',
    'Verification',
]
<commit_msg>Update version.<commit_after># -*- coding: utf-8 -*-

"""Kernel of Pyslvs.

This kernel can work without GUI.

Modules:
+ Solver:
    + parser
    + tinycadlib
        + Sketch Solve solver
    + triangulation
+ Number synthesis:
    + number
+ Structure Synthesis:
    + atlas
+ Dimensional synthesis:
    + planarlinkage
    + rga
    + firefly
    + de

Dependents:
+ lark-parser
+ pygments (optional: provide highlighting)
"""

__author__ = "Yuan Chang"
__copyright__ = "Copyright (C) 2016-2019"
__license__ = "AGPL"
__email__ = "pyslvs@gmail.com"
__version__ = "19.09.0.post0"

from .expression import (
    get_vlinks,
    VJoint,
    VPoint,
    VLink,
    Coordinate,
)
from .bfgs import SolverSystem
from .triangulation import vpoints_configure, ExpressionStack
from .tinycadlib import (
    plap,
    pllp,
    plpp,
    pxy,
    vpoint_dof,
    expr_parser,
    expr_solving,
    data_collecting,
)
from .Adesign import (
    Verification,
    Genetic,
    Firefly,
    Differential,
)
from .planar_linkage import Planar
from .number import link_synthesis, contracted_link_synthesis
from .graph import (
    Graph,
    link_assortment,
    contracted_link_assortment,
    labeled_enumerate,
)
from .planar_check import is_planar
from .graph_layout import external_loop_layout
from .atlas import conventional_graph, contracted_graph
from .expression_parser import (
    color_names,
    color_rgb,
    parse_params,
    parse_pos,
    parse_vpoints,
    parse_vlinks,
    edges_view,
    graph2vpoints,
    HAS_PYGMENTS,
)
from .example import example_list
from .collection import collection_list
if HAS_PYGMENTS:
    from .expression_parser import PMKSLexer

__all__ = [
    '__version__',
    'Genetic',
    'Firefly',
    'Differential',
    'Coordinate',
    'plap',
    'pllp',
    'plpp',
    'pxy',
    'expr_parser',
    'expr_solving',
    'data_collecting',
    'get_vlinks',
    'VJoint',
    'VPoint',
    'VLink',
    'SolverSystem',
    'Planar',
    'link_synthesis',
    'contracted_link_synthesis',
    'is_planar',
    'external_loop_layout',
    'conventional_graph',
    'contracted_graph',
    'Graph',
    'link_assortment',
    'contracted_link_assortment',
    'labeled_enumerate',
    'vpoints_configure',
    'ExpressionStack',
    'vpoint_dof',
    'color_names',
    'color_rgb',
    'parse_params',
    'parse_pos',
    'parse_vpoints',
    'parse_vlinks',
    'edges_view',
    'graph2vpoints',
    'PMKSLexer',
    'example_list',
    'collection_list',
    'Verification',
]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Public API
from .client import Client
from .oauth_client import OAuthClient
from .query_builder import QueryBuilder
from .resource import Resource

__author__ = "Robert Wikman <rbw@vault13.org>"
__version__ = "0.6.7"
<commit_msg>Bumped, implemented logger<commit_after># -*- coding: utf-8 -*-

# Public API
from .client import Client
from .oauth_client import OAuthClient
from .query_builder import QueryBuilder
from .resource import Resource

__author__ = "Robert Wikman <rbw@vault13.org>"
__version__ = "0.6.8"

# Set default logging handler to avoid "No handler found" warnings.
import logging
try:  # Python 2.7+
    from logging import NullHandler
except ImportError:
    class NullHandler(logging.Handler):
        def emit(self, record):
            pass

logging.getLogger(__name__).addHandler(NullHandler())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright 2014-2015 Parsely, Inc.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

'''
This module exists solely for version information so we only have to change it
in one place. Based on the suggestion `here. <http://bit.ly/16LbuJF>`_

:organization: Parsely
'''

def _safe_int(string):
    """ Simple function to convert strings into ints without dying. """
    try:
        return int(string)
    except ValueError:
        return string

__version__ = '3.1.3'
VERSION = tuple(_safe_int(x) for x in __version__.split('.'))
<commit_msg>Bump version to 3.1.4<commit_after># -*- coding: utf-8 -*-

# Copyright 2014-2015 Parsely, Inc.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

'''
This module exists solely for version information so we only have to change it
in one place. Based on the suggestion `here. <http://bit.ly/16LbuJF>`_

:organization: Parsely
'''

def _safe_int(string):
    """ Simple function to convert strings into ints without dying. """
    try:
        return int(string)
    except ValueError:
        return string

__version__ = '3.1.4'
VERSION = tuple(_safe_int(x) for x in __version__.split('.'))
<|endoftext|>


<commit_before>
import base64
import json
import numpy
import urllib.request
import uuid

from . import png

__all__ = ['URL', 'image', 'images', 'plot']

URL = 'http://localhost:8000/events'

def uid():
  return 'pane_%s' % uuid.uuid4()

def send(**command):
  command = json.dumps(command)
  req = urllib.request.Request(URL, method='POST')
  req.add_header('Content-Type', 'application/text')
  req.data = command.encode('ascii')
  try:
    resp = urllib.request.urlopen(req)
    return resp != None
  except:
    raise
    return False

def normalize(img, opts):
  minval = opts.get('min')
  if minval is None:
    minval = numpy.amin(img)
  maxval = opts.get('max')
  if maxval is None:
    maxval = numpy.amax(img)

  return numpy.uint8((img - minval) * (255/(maxval - minval)))

def to_rgb(img):
  nchannels = img.shape[2] if img.ndim == 3 else 1
  if nchannels == 3:
    return img
  if nchannels == 1:
    return img[:, :, numpy.newaxis].repeat(3, axis=2)
  raise ValueError('Image must be RGB or gray-scale')

def image(img, **opts):
  assert img.ndim == 2 or img.ndim == 3
  win = opts.get('win') or uid()

  if isinstance(img, list):
    return images(img, opts)
  # TODO: if img is a 3d tensor, then unstack it into a list of images

  img = to_rgb(normalize(img, opts))
  pngbytes = png.encode(img.tostring(), img.shape[1], img.shape[0])
  imgdata = 'data:image/png;base64,' + base64.b64encode(pngbytes).decode('ascii')

  send(command='image', id=win, src=imgdata,
    labels=opts.get('labels'),
    width=opts.get('width'),
    title=opts.get('title'))
  return win

def images(images, **opts):
  # TODO: need to merge images into a single canvas
  raise Exception('Not implemented')

def plot(data, **opts):
  """ Plot data as line chart.
  Params:
    data: either a 2-d numpy array or a list of lists.
    win: pane id
    labels: list of series names, first series is always the X-axis
    see http://dygraphs.com/options.html for other supported options
  """
  win = opts.get('win') or uid()

  dataset = {}
  if type(data).__module__ == numpy.__name__:
    dataset = data.tolist()
  else:
    dataset = data

  # clone opts into options
  options = dict(opts)
  options['file'] = dataset
  if options.get('labels'):
    options['xlabel'] = options['labels'][0]

  # Don't pass our options to dygraphs.
  options.pop('win', None)

  send(command='plot', id=win, title=opts.get('title'), options=options)
  return win

<commit_msg>update python module to match new command format<commit_after>
import base64
import json
import numpy
import urllib.request
import uuid

from . import png

__all__ = ['URL', 'image', 'images', 'plot']

URL = 'http://localhost:8000/events'

def uid():
  return 'pane_%s' % uuid.uuid4()

def send(**command):
  command = json.dumps(command)
  req = urllib.request.Request(URL, method='POST')
  req.add_header('Content-Type', 'application/text')
  req.data = command.encode('ascii')
  try:
    resp = urllib.request.urlopen(req)
    return resp != None
  except:
    raise
    return False

def pane(panetype, win, title, content):
  win = win or uid()
  send(command='pane', type=panetype, id=win, title=title, content=content)
  return win

def normalize(img, opts):
  minval = opts.get('min')
  if minval is None:
    minval = numpy.amin(img)
  maxval = opts.get('max')
  if maxval is None:
    maxval = numpy.amax(img)

  return numpy.uint8((img - minval) * (255/(maxval - minval)))

def to_rgb(img):
  nchannels = img.shape[2] if img.ndim == 3 else 1
  if nchannels == 3:
    return img
  if nchannels == 1:
    return img[:, :, numpy.newaxis].repeat(3, axis=2)
  raise ValueError('Image must be RGB or gray-scale')

def image(img, **opts):
  assert img.ndim == 2 or img.ndim == 3

  if isinstance(img, list):
    return images(img, opts)
  # TODO: if img is a 3d tensor, then unstack it into a list of images

  img = to_rgb(normalize(img, opts))
  pngbytes = png.encode(img.tostring(), img.shape[1], img.shape[0])
  imgdata = 'data:image/png;base64,' + base64.b64encode(pngbytes).decode('ascii')

  return pane('image', opts.get('win'), opts.get('title'), content={
      'src': imgdata,
      'labels': opts.get('labels'),
      'width': opts.get('width'),
  })
  return win

def images(images, **opts):
  # TODO: need to merge images into a single canvas
  raise Exception('Not implemented')

def plot(data, **opts):
  """ Plot data as line chart.
  Params:
    data: either a 2-d numpy array or a list of lists.
    win: pane id
    labels: list of series names, first series is always the X-axis
    see http://dygraphs.com/options.html for other supported options
  """
  dataset = {}
  if type(data).__module__ == numpy.__name__:
    dataset = data.tolist()
  else:
    dataset = data

  # clone opts into options
  options = dict(opts)
  options['file'] = dataset
  if options.get('labels'):
    options['xlabel'] = options['labels'][0]

  # Don't pass our options to dygraphs.
  options.pop('win', None)

  return pane('plot', opts.get('win'), opts.get('title'), content=options)

<|endoftext|>


<commit_before><commit_msg>txt_utils: added a new function: find_all(item, items, regex=False, regex_flags=None).<commit_after><|endoftext|>


<commit_before># coding=utf-8
from ia.models import Problem
from ia.methods import breadth_first_tree_search

class MissionersProblem(Problem):
    '''Missioners and cannibals problem.'''

    def __init__(self):
        super(MissionersProblem, self).__init__(initial_state=(3, 3, 0))
        # each action has a printable text, and the number of missioners
        # and cannibals to move on that action
        self._actions = [('1c', (0,1)),
                         ('1m', (1, 0)),
                         ('2c', (0, 2)),
                         ('2m', (2, 0)),
                         ('1m1c', (1, 1))]

    def actions(self, s):
        '''Possible actions from a state.'''
        # we try to generate every possible state and then filter those
        # states that are valid
        return [a for a in self._actions if self._is_valid(self.result(s, a))]

    def _is_valid(self, s):
        '''Check if a state is valid.'''
        # valid states: no more cannibals than missioners on each side,
        # and numbers between 0 and 3
        return ((s[0] >= s[1] or s[0] == 0)) and \
                ((3 - s[0]) >= (3 - s[1]) or s[0] == 3) and \
                (0 <= s[0] <= 3) and \
                (0 <= s[1] <= 3)

    def result(self, s, a):
        '''Result of applying an action to a state.'''
        # result: boat on opposite side, and numbers of missioners and
        # cannibals updated according to the move
        if s[2] == 0:
            return (s[0] - a[1][0], s[1] - a[1][1], 1)
        else:
            return (s[0] + a[1][0], s[1] + a[1][1], 0)

    def is_goal(self, state):
        return state == (0, 0, 1)

problem = MissionersProblem()
result = breadth_first_tree_search(problem)
print result.path()
<commit_msg>Renamed ia to ai on imports<commit_after># coding=utf-8
from ai.models import Problem
from ai.methods import breadth_first_tree_search

class MissionersProblem(Problem):
    '''Missioners and cannibals problem.'''

    def __init__(self):
        super(MissionersProblem, self).__init__(initial_state=(3, 3, 0))
        # each action has a printable text, and the number of missioners
        # and cannibals to move on that action
        self._actions = [('1c', (0,1)),
                         ('1m', (1, 0)),
                         ('2c', (0, 2)),
                         ('2m', (2, 0)),
                         ('1m1c', (1, 1))]

    def actions(self, s):
        '''Possible actions from a state.'''
        # we try to generate every possible state and then filter those
        # states that are valid
        return [a for a in self._actions if self._is_valid(self.result(s, a))]

    def _is_valid(self, s):
        '''Check if a state is valid.'''
        # valid states: no more cannibals than missioners on each side,
        # and numbers between 0 and 3
        return ((s[0] >= s[1] or s[0] == 0)) and \
                ((3 - s[0]) >= (3 - s[1]) or s[0] == 3) and \
                (0 <= s[0] <= 3) and \
                (0 <= s[1] <= 3)

    def result(self, s, a):
        '''Result of applying an action to a state.'''
        # result: boat on opposite side, and numbers of missioners and
        # cannibals updated according to the move
        if s[2] == 0:
            return (s[0] - a[1][0], s[1] - a[1][1], 1)
        else:
            return (s[0] + a[1][0], s[1] + a[1][1], 0)

    def is_goal(self, state):
        return state == (0, 0, 1)

problem = MissionersProblem()
result = breadth_first_tree_search(problem)
print result.path()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys
from distutils import spawn

class CompilerException(Exception):
    pass

def generate_proto(source, output_dir,
                   with_plugin='python', suffix='_pb2.py', plugin_binary=None):
    """Invokes the Protocol Compiler to generate a _pb2.py from the given
    .proto file.  Does nothing if the output already exists and is newer than
    the input."""
    protoc = spawn.find_executable("protoc")
    if protoc is None:
        raise CompilerException(
            "protoc not found. Make sure that it is in the path.")

    output = os.path.join(
            output_dir,
            os.path.basename(source.replace(".proto", suffix)))

    if not os.path.exists(source):
        raise CompilerException("Can't find required file: " + source)

    if (os.path.exists(output) and
        os.path.getmtime(source) <= os.path.getmtime(output)):
        print "Generated proto %s is up-to-date." % output
        return

    print "Generating %s" % output

    protoc_command = protoc + ' -I "%s" --%s_out="%s" "%s"' % (
            os.path.dirname(source), with_plugin, output_dir, source)
    if plugin_binary:
        if os.path.exists(plugin_binary):
            protoc_command += ' --plugin=protoc-gen-%s=%s' % (with_plugin,
                                                              plugin_binary)
        else:
            print ("Plugin not found at '%s'. We are going to run protoc "
                   "anyway, and perhaps it will be able to find it in its "
                   "search path." % plugin_binary)
    if os.system(protoc_command) != 0:
        raise CompilerException(
            "Error occurred while running protoc.")
    else:
        print "Generated source successfully."
<commit_msg>update print statement to python3<commit_after>#!/usr/bin/env python

from __future__ import print_function

import os
import sys
from distutils import spawn

class CompilerException(Exception):
    pass

def generate_proto(source, output_dir,
                   with_plugin='python', suffix='_pb2.py', plugin_binary=None):
    """Invokes the Protocol Compiler to generate a _pb2.py from the given
    .proto file.  Does nothing if the output already exists and is newer than
    the input."""
    protoc = spawn.find_executable("protoc")
    if protoc is None:
        raise CompilerException(
            "protoc not found. Make sure that it is in the path.")

    output = os.path.join(
            output_dir,
            os.path.basename(source.replace(".proto", suffix)))

    if not os.path.exists(source):
        raise CompilerException("Can't find required file: " + source)

    if (os.path.exists(output) and
        os.path.getmtime(source) <= os.path.getmtime(output)):
        print("Generated proto {0} is up-to-date.".format(output))
        return

    print("Generating {0}".format(output))

    protoc_command = protoc + ' -I "%s" --%s_out="%s" "%s"' % (
            os.path.dirname(source), with_plugin, output_dir, source)
    if plugin_binary:
        if os.path.exists(plugin_binary):
            protoc_command += ' --plugin=protoc-gen-%s=%s' % (with_plugin,
                                                              plugin_binary)
        else:
            print("Plugin not found at '{0}'. We are going to run protoc "
                  "anyway, and perhaps it will be able to find it in its "
                  "search path.".format(plugin_binary))
    if os.system(protoc_command) != 0:
        raise CompilerException(
            "Error occurred while running protoc.")
    else:
        print("Generated source successfully.")
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import argparse
import logging
import xml.etree.cElementTree as ET
import sys
import read_records as rr
import extract as x
from db_info import *
log_levels = { "DEBUG"   : logging.DEBUG,
               "INFO"    : logging.INFO,
               "WARNING" : logging.WARNING,
               "ERROR"   : logging.ERROR,
               "CRITICAL": logging.CRITICAL
}

def main (sourcefile, year, datadir, data_format):

    count  = 0
    logging.debug("Starting processing {0}".format(sourcefile))

    Pubs_list = []
    Lang_list = []
    Head_list = []
    Subh_list = []
    Subj_list = []
    Publ_list = []
    Auth_list = []
    Inst_list = []
    NaIn_list = []
    Edit_list = []
    Refs_list = []
    Ftxt_list = []
    Fund_list = []
    Keyw_list = []
    Keyp_list = []
    Conf_list = []
    CoSp_list = []

    with open(args.sourcefile, 'r') as data:

        while True:

            count  += 1
            record  = rr.get_record(data)

            if not record:
                logging.debug("Completed processing {0}".format(sourcefile))
                print "Processed {0} records".format(count-1)
                break

            try :
                #context = extract.load_data(record)
                REC    = ET.fromstring(record)
                wos_id = list(REC.iterfind('UID'))[0].text

                Pub, Languages, Headings, Subheadings, Subjects    = x.extract_pub_info(wos_id, REC)
                Pubs_list.extend(Pub)
                Lang_list.extend(Languages)
                Head_list.extend(Headings)
                Subh_list.extend(Subheadings)
                Subj_list.extend(Subjects)

                #print pub
                Publishers = x.extract_publisher(wos_id, REC)
                Publ_list.extend(Publishers)

                #print publisher
                Authors   = x.extract_authors(wos_id, REC)
                Auth_list.extend(Authors)

                ##print authors
                Institutions, Name_inst_relation = x.extract_addresses(wos_id, REC)
                Inst_list.extend(Institutions)
                NaIn_list.extend(Name_inst_relation)
                #print len(Inst_list), Institutions
                #print len(NaIn_list), Name_inst_relation

                Editions = x.extract_editions(wos_id, REC)
                Edit_list.extend(Editions)

                References = x.extract_references(wos_id, REC)
                Refs_list.extend(References)

                Ftext, Funding = x.extract_funding(wos_id, REC)
                Ftxt_list.extend(Ftext)
                Fund_list.extend(Funding)

                Conf, Sponsor  = x.extract_conferences(wos_id, REC)
                Conf_list.extend(Conf)
                CoSp_list.extend(Sponsor)
                #print len(Conf_list), Conf
                #print len(CoSp_list), Sponsor

                Keywords, Keywords_plus = x.extract_keywords(wos_id, REC)
                Keyw_list.extend(Keywords)
                Keyp_list.extend(Keywords_plus)
                #print Keywords

            except Exception as e:
                print "[ERROR:{0}] Caught an exception : {1}".format(wos_id, e)
                pass

    try :
        x.dump(Pubs_list,    h_source,       t_source,         year+'source',       '{0}/source.{1}'.format(datadir, data_format),     data_format=data_format)
        """
        x.dump(Edit_list,    h_editions,       t_editions,         year+'editions',       '{0}/editions.{1}'.format(datadir, data_format),     data_format=data_format)
        x.dump(Ftxt_list,    h_fundingtexts,   t_fundingtexts,     year+'fundingtext',    '{0}/fundingtext.{1}'.format(datadir, data_format),  data_format=data_format)
        x.dump(Fund_list,    h_funding,        t_funding,          year+'funding',        '{0}/funding.{1}'.format(datadir, data_format),      data_format=data_format)
        x.dump(Keyw_list,    h_keywords,       t_keywords,         year+'keywords',       '{0}/keywords.{1}'.format(datadir, data_format),     data_format=data_format)
        x.dump(Keyp_list,    h_keywords_plus,  t_keywords_plus,    year+'keywords_plus',  '{0}/keywords_plus.{1}'.format(datadir, data_format),data_format=data_format)
        x.dump(Conf_list,    h_conferences,    t_conferences,      year+'conferences',    '{0}/conferences.{1}'.format(datadir, data_format),  data_format=data_format)
        x.dump(CoSp_list,    h_conf_sponsors,  t_conf_sponsors,    year+'confSponsors',   '{0}/confSponsors.{1}'.format(datadir, data_format), data_format=data_format)
        x.dump(Refs_list,    h_references,     t_references,       year+'refs',           '{0}/references.{1}'.format(datadir, data_format),   data_format=data_format)
        x.dump(Pubs_list,    h_publications,   t_publications,     year+'publications',   '{0}/publications.{1}'.format(datadir, data_format), data_format=data_format)
        x.dump(Lang_list,    h_languages,      t_languages,        year+'languages',      '{0}/langauges.{1}'.format(datadir, data_format),    data_format=data_format)
        x.dump(Head_list,    h_headings,       t_headings,         year+'headings',       '{0}/headings.{1}'.format(datadir, data_format),     data_format=data_format)
        x.dump(Subh_list,    h_subheadings,    t_subheadings,      year+'subheadings',    '{0}/subheadings.{1}'.format(datadir, data_format),  data_format=data_format)
        x.dump(Subj_list,    h_subjects,       t_subjects,         year+'subjects',       '{0}/subjects.{1}'.format(datadir, data_format),     data_format=data_format)
        x.dump(Publ_list,    h_publishers,     t_publishers,       year+'publishers',     '{0}/publishers.{1}'.format(datadir, data_format),   data_format=data_format)
        x.dump(Auth_list,    h_contributors,   t_contributors,     year+'contributors',   '{0}/contributors.{1}'.format(datadir, data_format), data_format=data_format)
        x.dump(Inst_list,    h_institutions,   t_institutions,     year+'institutions',   '{0}/institutions.{1}'.format(datadir, data_format), data_format=data_format)
        x.dump(NaIn_list,    h_name_inst,      t_name_inst,        year+'affiliations',   '{0}/affiliations.{1}'.format(datadir, data_format), data_format=data_format)
        """
    except Exception as e:
        print "[ERROR] Dumping failed for {0}".format(sourcefile)
        logging.error("[ERROR] Dumping failed for {0}".format(sourcefile))
        exit(-1)

    return

if __name__ == "__main__" :

    parser   = argparse.ArgumentParser()
    parser.add_argument("-s", "--sourcefile", default="sample.xml", help="Path to data file")
    parser.add_argument("-v", "--verbosity", default="DEBUG", help="set level of verbosity, DEBUG, INFO, WARN")
    parser.add_argument("-l", "--logfile", default="./extract.log", help="Logfile path. Defaults to ./tabulator.log")
    parser.add_argument("-d", "--dir", default=".", help="Folder to write data to, Default is current folder")
    parser.add_argument("-f", "--format", default="sql", help="Output format to dump into")
    args   = parser.parse_args()

    print "Processing : {0}".format(args.sourcefile)

    logging.basicConfig(filename=args.logfile, level=log_levels[args.verbosity],
                        format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                        datefmt='%m-%d %H:%M')

    logging.debug("Document processing starts")

    if not os.path.exists(args.dir):
        os.makedirs(args.dir)
    logging.debug("Data output folder confirmed  :  {0}".format(args.dir))

    year = ''
    if os.path.basename(args.sourcefile).startswith('WR'):
        s = args.sourcefile.split('_')
        year = s[1] + "_"
        print args.sourcefile.replace('.xml', '')

    print "[DEBUG] Processing year : {0}".format(year)

    main(args.sourcefile, year, args.dir, args.format)

<commit_msg>Updated code<commit_after>#!/usr/bin/env python
import os
import argparse
import logging
import xml.etree.cElementTree as ET
import sys
import read_records as rr
import extract as x
from db_info import *
log_levels = { "DEBUG"   : logging.DEBUG,
               "INFO"    : logging.INFO,
               "WARNING" : logging.WARNING,
               "ERROR"   : logging.ERROR,
               "CRITICAL": logging.CRITICAL
}

def main (sourcefile, year, datadir, data_format):

    count  = 0
    logging.debug("Starting processing {0}".format(sourcefile))

    Pubs_list = []
    Lang_list = []
    Head_list = []
    Subh_list = []
    Subj_list = []
    Publ_list = []
    Auth_list = []
    Inst_list = []
    NaIn_list = []
    Edit_list = []
    Refs_list = []
    Ftxt_list = []
    Fund_list = []
    Keyw_list = []
    Keyp_list = []
    Conf_list = []
    CoSp_list = []

    with open(args.sourcefile, 'r') as data:

        while True:

            count  += 1
            record  = rr.get_record(data)

            if not record:
                logging.debug("Completed processing {0}".format(sourcefile))
                print "Processed {0} records".format(count-1)
                break

            try :
                #context = extract.load_data(record)
                REC    = ET.fromstring(record)
                wos_id = list(REC.iterfind('UID'))[0].text

                Pub, Languages, Headings, Subheadings, Subjects    = x.extract_pub_info(wos_id, REC)
                Pubs_list.extend(Pub)
                Lang_list.extend(Languages)
                Head_list.extend(Headings)
                Subh_list.extend(Subheadings)
                Subj_list.extend(Subjects)

                """
                #print pub
                Publishers = x.extract_publisher(wos_id, REC)
                Publ_list.extend(Publishers)

                #print publisher
                Authors   = x.extract_authors(wos_id, REC)
                Auth_list.extend(Authors)

                ##print authors
                Institutions, Name_inst_relation = x.extract_addresses(wos_id, REC)
                Inst_list.extend(Institutions)
                NaIn_list.extend(Name_inst_relation)
                #print len(Inst_list), Institutions
                #print len(NaIn_list), Name_inst_relation

                Editions = x.extract_editions(wos_id, REC)
                Edit_list.extend(Editions)

                References = x.extract_references(wos_id, REC)
                Refs_list.extend(References)

                Ftext, Funding = x.extract_funding(wos_id, REC)
                Ftxt_list.extend(Ftext)
                Fund_list.extend(Funding)

                Conf, Sponsor  = x.extract_conferences(wos_id, REC)
                Conf_list.extend(Conf)
                CoSp_list.extend(Sponsor)
                #print len(Conf_list), Conf
                #print len(CoSp_list), Sponsor

                Keywords, Keywords_plus = x.extract_keywords(wos_id, REC)
                Keyw_list.extend(Keywords)
                Keyp_list.extend(Keywords_plus)
                #print Keywords
                """
            except Exception as e:
                print "[ERROR:{0}] Caught an exception : {1}".format(wos_id, e)
                pass

    try :
        x.dump(Pubs_list,    h_source,       t_source,         year+'source',       '{0}/source.{1}'.format(datadir, data_format),     data_format=data_format)
        """
        x.dump(Edit_list,    h_editions,       t_editions,         year+'editions',       '{0}/editions.{1}'.format(datadir, data_format),     data_format=data_format)
        x.dump(Ftxt_list,    h_fundingtexts,   t_fundingtexts,     year+'fundingtext',    '{0}/fundingtext.{1}'.format(datadir, data_format),  data_format=data_format)
        x.dump(Fund_list,    h_funding,        t_funding,          year+'funding',        '{0}/funding.{1}'.format(datadir, data_format),      data_format=data_format)
        x.dump(Keyw_list,    h_keywords,       t_keywords,         year+'keywords',       '{0}/keywords.{1}'.format(datadir, data_format),     data_format=data_format)
        x.dump(Keyp_list,    h_keywords_plus,  t_keywords_plus,    year+'keywords_plus',  '{0}/keywords_plus.{1}'.format(datadir, data_format),data_format=data_format)
        x.dump(Conf_list,    h_conferences,    t_conferences,      year+'conferences',    '{0}/conferences.{1}'.format(datadir, data_format),  data_format=data_format)
        x.dump(CoSp_list,    h_conf_sponsors,  t_conf_sponsors,    year+'confSponsors',   '{0}/confSponsors.{1}'.format(datadir, data_format), data_format=data_format)
        x.dump(Refs_list,    h_references,     t_references,       year+'refs',           '{0}/references.{1}'.format(datadir, data_format),   data_format=data_format)
        x.dump(Pubs_list,    h_publications,   t_publications,     year+'publications',   '{0}/publications.{1}'.format(datadir, data_format), data_format=data_format)
        x.dump(Lang_list,    h_languages,      t_languages,        year+'languages',      '{0}/langauges.{1}'.format(datadir, data_format),    data_format=data_format)
        x.dump(Head_list,    h_headings,       t_headings,         year+'headings',       '{0}/headings.{1}'.format(datadir, data_format),     data_format=data_format)
        x.dump(Subh_list,    h_subheadings,    t_subheadings,      year+'subheadings',    '{0}/subheadings.{1}'.format(datadir, data_format),  data_format=data_format)
        x.dump(Subj_list,    h_subjects,       t_subjects,         year+'subjects',       '{0}/subjects.{1}'.format(datadir, data_format),     data_format=data_format)
        x.dump(Publ_list,    h_publishers,     t_publishers,       year+'publishers',     '{0}/publishers.{1}'.format(datadir, data_format),   data_format=data_format)
        x.dump(Auth_list,    h_contributors,   t_contributors,     year+'contributors',   '{0}/contributors.{1}'.format(datadir, data_format), data_format=data_format)
        x.dump(Inst_list,    h_institutions,   t_institutions,     year+'institutions',   '{0}/institutions.{1}'.format(datadir, data_format), data_format=data_format)
        x.dump(NaIn_list,    h_name_inst,      t_name_inst,        year+'affiliations',   '{0}/affiliations.{1}'.format(datadir, data_format), data_format=data_format)
        """
    except Exception as e:
        print "[ERROR] Dumping failed for {0}".format(sourcefile)
        logging.error("[ERROR] Dumping failed for {0}".format(sourcefile))
        exit(-1)

    return

if __name__ == "__main__" :

    parser   = argparse.ArgumentParser()
    parser.add_argument("-s", "--sourcefile", default="sample.xml", help="Path to data file")
    parser.add_argument("-v", "--verbosity", default="DEBUG", help="set level of verbosity, DEBUG, INFO, WARN")
    parser.add_argument("-l", "--logfile", default="./extract.log", help="Logfile path. Defaults to ./tabulator.log")
    parser.add_argument("-d", "--dir", default=".", help="Folder to write data to, Default is current folder")
    parser.add_argument("-f", "--format", default="sql", help="Output format to dump into")
    args   = parser.parse_args()

    print "Processing : {0}".format(args.sourcefile)

    logging.basicConfig(filename=args.logfile, level=log_levels[args.verbosity],
                        format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                        datefmt='%m-%d %H:%M')

    logging.debug("Document processing starts")

    if not os.path.exists(args.dir):
        os.makedirs(args.dir)
    logging.debug("Data output folder confirmed  :  {0}".format(args.dir))

    year = ''
    if os.path.basename(args.sourcefile).startswith('WR'):
        s = args.sourcefile.split('_')
        year = s[1] + "_"
        print args.sourcefile.replace('.xml', '')

    print "[DEBUG] Processing year : {0}".format(year)

    main(args.sourcefile, year, args.dir, args.format)

<|endoftext|>