<commit_before># -*- coding: utf-8 -*-

########################################################################
#
# License: BSD
# Created: 2007-02-23
# Author: Ivan Vilata i Balaguer - ivan at selidor dot net
#
# $Id$
#
########################################################################

"""Functionality related with filters in a PyTables file."""
from __future__ import absolute_import

# Imports
# =======
import warnings
import numpy

from . import utilsextension, blosc_compressor_list, blosc_compcode_to_compname
from .exceptions import FiltersWarning
import six
from distutils.version import LooseVersion

import tables
from tables.req_versions import min_blosc_bitshuffle_version

blosc_version = LooseVersion(tables.which_lib_version("blosc")[1])

# Public variables
# ================
__docformat__ = 'reStructuredText'
"""The format of documentation strings in this module."""

all_complibs = ['zlib', 'lzo', 'bzip2', 'blosc']
all_complibs += ['blosc:%s' % cname for cname in blosc_compressor_list()]

"""List of all compression libraries."""

foreign_complibs = ['szip']
"""List of known but unsupported compression libraries."""

default_complib = 'zlib'
"""The default compression library."""

# Private variables
# =================
_shuffle_flag = 0x1
_fletcher32_flag = 0x2
_rounding_flag = 0x4
_bitshuffle_flag = 0x8

# Classes
# =======
class Filters(object):
    """Container for filter properties.

    This class is meant to serve as a container that keeps information about
    the filter properties associated with the chunked leaves, that is Table,
    CArray, EArray and VLArray.

    Instances of this class can be directly compared for equality.

    Parameters
    ----------
    complevel : int
        Specifies a compression level for data. The allowed
        range is 0-9. A value of 0 (the default) disables
        compression.
    complib : str
        Specifies the compression library to be used. Right now, 'zlib' (the
        default), 'lzo', 'bzip2' and 'blosc' are supported.  Additional
        compressors for Blosc like 'blosc:blosclz' ('blosclz' is the default
        in case the additional compressor is not specified), 'blosc:lz4',
        'blosc:lz4hc', 'blosc:snappy' and 'blosc:zlib' are supported too.
        Specifying a compression library which is not available in the
        system issues a FiltersWarning and sets the library to the default
        one.
    shuffle : bool
        Whether or not to use the *Shuffle* filter in the HDF5
        library. This is normally used to improve the compression
        ratio. A false value disables shuffling and a true one enables
        it. The default value depends on whether compression is
        enabled or not; if compression is enabled, shuffling defaults
        to be enabled, else shuffling is disabled. Shuffling can only
        be used when compression is enabled.
    bitshuffle : bool
        Whether or not to use the *BitShuffle* filter in the Blosc
        library. This is normally used to improve the compression
        ratio. A false value disables bitshuffling and a true one
        enables it. The default value is disabled.
    fletcher32 : bool
        Whether or not to use the
        *Fletcher32* filter in the HDF5 library.
        This is used to add a checksum on each data chunk. A false
        value (the default) disables the checksum.
    least_significant_digit : int
        If specified, data will be truncated (quantized). In conjunction
        with enabling compression, this produces 'lossy', but
        significantly more efficient compression. For example, if
        *least_significant_digit=1*, data will be quantized using
        ``around(scale*data)/scale``, where ``scale = 2**bits``, and
        bits is determined so that a precision of 0.1 is retained (in
        this case bits=4). Default is *None*, or no quantization.

        .. note::

            quantization is only applied if some form of compression is
            enabled

    Examples
    --------

    This is a small example on using the Filters class::

        import numpy
        import tables

        fileh = tables.open_file('test5.h5', mode='w')
        atom = Float32Atom()
        filters = Filters(complevel=1, complib='blosc', fletcher32=True)
        arr = fileh.create_earray(fileh.root, 'earray', atom, (0,2),
                                 "A growable array", filters=filters)

        # Append several rows in only one call
        arr.append(numpy.array([[1., 2.],
                                [2., 3.],
                                [3., 4.]], dtype=numpy.float32))

        # Print information on that enlargeable array
        print("Result Array:")
        print(repr(arr))
        fileh.close()

    This enforces the use of the Blosc library, a compression level of 1 and a
    Fletcher32 checksum filter as well. See the output of this example::

        Result Array:
        /earray (EArray(3, 2), fletcher32, shuffle, blosc(1)) 'A growable array'
        type = float32
        shape = (3, 2)
        itemsize = 4
        nrows = 3
        extdim = 0
        flavor = 'numpy'
        byteorder = 'little'

    .. rubric:: Filters attributes

    .. attribute:: fletcher32

        Whether the *Fletcher32* filter is active or not.

    .. attribute:: complevel

        The compression level (0 disables compression).

    .. attribute:: complib

        The compression filter used (irrelevant when compression is not
        enabled).

    .. attribute:: shuffle

        Whether the *Shuffle* filter is active or not.

    .. attribute:: bitshuffle

        Whether the *BitShuffle* filter is active or not (Blosc only).

    """

    @property
    def shuffle_bitshuffle(self):
        """Encode NoShuffle (0), Shuffle (1) and BitShuffle (2) filters."""
        if (self.shuffle and self.bitshuffle):
            raise ValueError(
                "Shuffle and BitShuffle cannot be active at the same time")
        if not (self.shuffle or self.bitshuffle):
            return 0
        if self.shuffle:
            return 1
        if self.bitshuffle:
            return 2

    @classmethod
    def _from_leaf(class_, leaf):
        # Get a dictionary with all the filters
        parent = leaf._v_parent
        filters_dict = utilsextension.get_filters(parent._v_objectid,
                                                  leaf._v_name)
        if filters_dict is None:
            filters_dict = {}  # not chunked

        # Keyword arguments are all off
        kwargs = dict(complevel=0, shuffle=False, bitshuffle=False,
                      fletcher32=False, least_significant_digit=None,
                      _new=False)
        for (name, values) in six.iteritems(filters_dict):
            if name == 'deflate':
                name = 'zlib'
            if name in all_complibs:
                kwargs['complib'] = name
                if name == "blosc":
                    kwargs['complevel'] = values[4]
                    if values[5] == 1:
                        # Shuffle filter is internal to blosc
                        kwargs['shuffle'] = True
                    elif values[5] == 2:
                        # Shuffle filter is internal to blosc
                        kwargs['bitshuffle'] = True
                    # From Blosc 1.3 on, parameter 6 is used for the compressor
                    if len(values) > 6:
                        cname = blosc_compcode_to_compname(values[6])
                        kwargs['complib'] = "blosc:%s" % cname
                else:
                    kwargs['complevel'] = values[0]
            elif name in foreign_complibs:
                kwargs['complib'] = name
                kwargs['complevel'] = 1  # any nonzero value will do
            elif name in ['shuffle', 'fletcher32']:
                kwargs[name] = True
        return class_(**kwargs)

    @classmethod
    def _unpack(class_, packed):
        """Create a new `Filters` object from a packed version.

        >>> Filters._unpack(0)
        Filters(complevel=0, shuffle=False, bitshuffle=False, fletcher32=False, least_significant_digit=None)
        >>> Filters._unpack(0x101)
        Filters(complevel=1, complib='zlib', shuffle=False, bitshuffle=False, fletcher32=False, least_significant_digit=None)
        >>> Filters._unpack(0x30109)
        Filters(complevel=9, complib='zlib', shuffle=True, bitshuffle=False, fletcher32=True, least_significant_digit=None)
        >>> Filters._unpack(0x3010A)
        Traceback (most recent call last):
          ...
        ValueError: compression level must be between 0 and 9
        >>> Filters._unpack(0x1)
        Traceback (most recent call last):
          ...
        ValueError: invalid compression library id: 0

        """

        kwargs = {'_new': False}

        # Byte 0: compression level.
        kwargs['complevel'] = complevel = packed & 0xff
        packed >>= 8

        # Byte 1: compression library id (0 for none).
        if complevel > 0:
            complib_id = int(packed & 0xff)
            if not (0 < complib_id <= len(all_complibs)):
                raise ValueError("invalid compression library id: %d"
                                 % complib_id)
            kwargs['complib'] = all_complibs[complib_id - 1]
        packed >>= 8

        # Byte 2: parameterless filters.
        kwargs['shuffle'] = packed & _shuffle_flag
        kwargs['bitshuffle'] = packed & _bitshuffle_flag
        kwargs['fletcher32'] = packed & _fletcher32_flag
        has_rounding = packed & _rounding_flag
        packed >>= 8

        # Byte 3: least significant digit.
        if has_rounding:
            kwargs['least_significant_digit'] = numpy.int8(packed & 0xff)
        else:
            kwargs['least_significant_digit'] = None

        return class_(**kwargs)

    def _pack(self):
        """Pack the `Filters` object into a 64-bit NumPy integer."""

        packed = numpy.int64(0)

        # Byte 3: least significant digit.
        if self.least_significant_digit is not None:
            #assert isinstance(self.least_significant_digit, numpy.int8)
            packed |= self.least_significant_digit
        packed <<= 8

        # Byte 2: parameterless filters.
        if self.shuffle:
            packed |= _shuffle_flag
        if self.bitshuffle:
            packed |= _bitshuffle_flag
        if self.fletcher32:
            packed |= _fletcher32_flag
        if self.least_significant_digit:
            packed |= _rounding_flag
        packed <<= 8

        # Byte 1: compression library id (0 for none).
        if self.complevel > 0:
            packed |= all_complibs.index(self.complib) + 1
        packed <<= 8

        # Byte 0: compression level.
        packed |= self.complevel

        return packed

    def __init__(self, complevel=0, complib=default_complib,
                 shuffle=True, bitshuffle=False, fletcher32=False,
                 least_significant_digit=None, _new=True):

        if not (0 <= complevel <= 9):
            raise ValueError("compression level must be between 0 and 9")

        if _new and complevel > 0:
            # These checks are not performed when loading filters from disk.
            if complib not in all_complibs:
                raise ValueError(
                    "compression library ``%s`` is not supported; "
                    "it must be one of: %s"
                    % (complib, ", ".join(all_complibs)))
            if utilsextension.which_lib_version(complib) is None:
                warnings.warn("compression library ``%s`` is not available; "
                              "using ``%s`` instead"
                              % (complib, default_complib), FiltersWarning)
                complib = default_complib  # always available

        complevel = int(complevel)
        complib = str(complib)
        shuffle = bool(shuffle)
        bitshuffle = bool(bitshuffle)
        fletcher32 = bool(fletcher32)
        if least_significant_digit is not None:
            least_significant_digit = numpy.int8(least_significant_digit)

        if complevel == 0:
            # Override some inputs when compression is not enabled.
            complib = None  # make it clear there is no compression
            shuffle = False  # shuffling and not compressing makes no sense
            least_significant_digit = None
        elif complib not in all_complibs:
            # Do not try to use a meaningful level for unsupported libs.
            complevel = -1

        self.complevel = complevel
        """The compression level (0 disables compression)."""

        self.complib = complib
        """The compression filter used (irrelevant when compression is
        not enabled).
        """

        self.shuffle = shuffle
        """Whether the *Shuffle* filter is active or not."""

        self.bitshuffle = bitshuffle
        """Whether the *BitShuffle* filter is active or not."""

        if self.bitshuffle and not self.complib.startswith('blosc'):
            raise ValueError(
                "BitShuffle can only be used inside Blosc")

        if self.shuffle and self.bitshuffle:
            raise ValueError(
                "Shuffle and BitShuffle cannot be active at the same time")

        if self.bitshuffle and blosc_version < min_blosc_bitshuffle_version:
            raise ValueError(
                "This Blosc library does not have support for the bitshuffle "
                "filter.  Please update to Blosc >= %s" % \
                min_blosc_bitshuffle_version)

        self.fletcher32 = fletcher32
        """Whether the *Fletcher32* filter is active or not."""

        self.least_significant_digit = least_significant_digit
        """The least significant digit to which data shall be truncated."""

    def __repr__(self):
        args, complevel = [], self.complevel
        if complevel >= 0:  # meaningful compression level
            args.append('complevel=%d' % complevel)
        if complevel != 0:  # compression enabled (-1 or > 0)
            args.append('complib=%r' % self.complib)
        args.append('shuffle=%s' % self.shuffle)
        args.append('bitshuffle=%s' % self.bitshuffle)
        args.append('fletcher32=%s' % self.fletcher32)
        args.append(
            'least_significant_digit=%s' % self.least_significant_digit)
        return '%s(%s)' % (self.__class__.__name__, ', '.join(args))

    def __str__(self):
        return repr(self)

    def __eq__(self, other):
        if not isinstance(other, self.__class__):
            return False
        for attr in self.__dict__:
            if getattr(self, attr) != getattr(other, attr):
                return False
        return True

    # XXX: API incompatible change for PyTables 3 line
    # Overriding __eq__ blocks inheritance of __hash__ in 3.x
    # def __hash__(self):
    #    return hash((self.__class__, self.complevel, self.complib,
    #                 self.shuffle, self.bitshuffle, self.fletcher32))

    def copy(self, **override):
        """Get a copy of the filters, possibly overriding some arguments.

        Constructor arguments to be overridden must be passed as keyword
        arguments.

        Using this method is recommended over replacing the attributes of an
        instance, since instances of this class may become immutable in the
        future::

            >>> filters1 = Filters()
            >>> filters2 = filters1.copy()
            >>> filters1 == filters2
            True
            >>> filters1 is filters2
            False
            >>> filters3 = filters1.copy(complevel=1) #doctest: +ELLIPSIS
            Traceback (most recent call last):
            ...
            ValueError: compression library ``None`` is not supported...
            >>> filters3 = filters1.copy(complevel=1, complib='zlib')
            >>> print(filters1)
            Filters(complevel=0, shuffle=False, bitshuffle=False, fletcher32=False, least_significant_digit=None)
            >>> print(filters3)
            Filters(complevel=1, complib='zlib', shuffle=False, bitshuffle=False, fletcher32=False, least_significant_digit=None)
            >>> filters1.copy(foobar=42)
            Traceback (most recent call last):
            ...
            TypeError: __init__() got an unexpected keyword argument 'foobar'

        """

        newargs = self.__dict__.copy()
        newargs.update(override)
        return self.__class__(**newargs)

# Main part
# =========
def _test():
    """Run ``doctest`` on this module."""

    import doctest
    doctest.testmod()

if __name__ == '__main__':
    _test()
<commit_msg>Protection against undefined complib<commit_after># -*- coding: utf-8 -*-

########################################################################
#
# License: BSD
# Created: 2007-02-23
# Author: Ivan Vilata i Balaguer - ivan at selidor dot net
#
# $Id$
#
########################################################################

"""Functionality related with filters in a PyTables file."""
from __future__ import absolute_import

# Imports
# =======
import warnings
import numpy

from . import utilsextension, blosc_compressor_list, blosc_compcode_to_compname
from .exceptions import FiltersWarning
import six
from distutils.version import LooseVersion

import tables
from tables.req_versions import min_blosc_bitshuffle_version

blosc_version = LooseVersion(tables.which_lib_version("blosc")[1])

# Public variables
# ================
__docformat__ = 'reStructuredText'
"""The format of documentation strings in this module."""

all_complibs = ['zlib', 'lzo', 'bzip2', 'blosc']
all_complibs += ['blosc:%s' % cname for cname in blosc_compressor_list()]

"""List of all compression libraries."""

foreign_complibs = ['szip']
"""List of known but unsupported compression libraries."""

default_complib = 'zlib'
"""The default compression library."""

# Private variables
# =================
_shuffle_flag = 0x1
_fletcher32_flag = 0x2
_rounding_flag = 0x4
_bitshuffle_flag = 0x8

# Classes
# =======
class Filters(object):
    """Container for filter properties.

    This class is meant to serve as a container that keeps information about
    the filter properties associated with the chunked leaves, that is Table,
    CArray, EArray and VLArray.

    Instances of this class can be directly compared for equality.

    Parameters
    ----------
    complevel : int
        Specifies a compression level for data. The allowed
        range is 0-9. A value of 0 (the default) disables
        compression.
    complib : str
        Specifies the compression library to be used. Right now, 'zlib' (the
        default), 'lzo', 'bzip2' and 'blosc' are supported.  Additional
        compressors for Blosc like 'blosc:blosclz' ('blosclz' is the default
        in case the additional compressor is not specified), 'blosc:lz4',
        'blosc:lz4hc', 'blosc:snappy' and 'blosc:zlib' are supported too.
        Specifying a compression library which is not available in the
        system issues a FiltersWarning and sets the library to the default
        one.
    shuffle : bool
        Whether or not to use the *Shuffle* filter in the HDF5
        library. This is normally used to improve the compression
        ratio. A false value disables shuffling and a true one enables
        it. The default value depends on whether compression is
        enabled or not; if compression is enabled, shuffling defaults
        to be enabled, else shuffling is disabled. Shuffling can only
        be used when compression is enabled.
    bitshuffle : bool
        Whether or not to use the *BitShuffle* filter in the Blosc
        library. This is normally used to improve the compression
        ratio. A false value disables bitshuffling and a true one
        enables it. The default value is disabled.
    fletcher32 : bool
        Whether or not to use the
        *Fletcher32* filter in the HDF5 library.
        This is used to add a checksum on each data chunk. A false
        value (the default) disables the checksum.
    least_significant_digit : int
        If specified, data will be truncated (quantized). In conjunction
        with enabling compression, this produces 'lossy', but
        significantly more efficient compression. For example, if
        *least_significant_digit=1*, data will be quantized using
        ``around(scale*data)/scale``, where ``scale = 2**bits``, and
        bits is determined so that a precision of 0.1 is retained (in
        this case bits=4). Default is *None*, or no quantization.

        .. note::

            quantization is only applied if some form of compression is
            enabled

    Examples
    --------

    This is a small example on using the Filters class::

        import numpy
        import tables

        fileh = tables.open_file('test5.h5', mode='w')
        atom = Float32Atom()
        filters = Filters(complevel=1, complib='blosc', fletcher32=True)
        arr = fileh.create_earray(fileh.root, 'earray', atom, (0,2),
                                 "A growable array", filters=filters)

        # Append several rows in only one call
        arr.append(numpy.array([[1., 2.],
                                [2., 3.],
                                [3., 4.]], dtype=numpy.float32))

        # Print information on that enlargeable array
        print("Result Array:")
        print(repr(arr))
        fileh.close()

    This enforces the use of the Blosc library, a compression level of 1 and a
    Fletcher32 checksum filter as well. See the output of this example::

        Result Array:
        /earray (EArray(3, 2), fletcher32, shuffle, blosc(1)) 'A growable array'
        type = float32
        shape = (3, 2)
        itemsize = 4
        nrows = 3
        extdim = 0
        flavor = 'numpy'
        byteorder = 'little'

    .. rubric:: Filters attributes

    .. attribute:: fletcher32

        Whether the *Fletcher32* filter is active or not.

    .. attribute:: complevel

        The compression level (0 disables compression).

    .. attribute:: complib

        The compression filter used (irrelevant when compression is not
        enabled).

    .. attribute:: shuffle

        Whether the *Shuffle* filter is active or not.

    .. attribute:: bitshuffle

        Whether the *BitShuffle* filter is active or not (Blosc only).

    """

    @property
    def shuffle_bitshuffle(self):
        """Encode NoShuffle (0), Shuffle (1) and BitShuffle (2) filters."""
        if (self.shuffle and self.bitshuffle):
            raise ValueError(
                "Shuffle and BitShuffle cannot be active at the same time")
        if not (self.shuffle or self.bitshuffle):
            return 0
        if self.shuffle:
            return 1
        if self.bitshuffle:
            return 2

    @classmethod
    def _from_leaf(class_, leaf):
        # Get a dictionary with all the filters
        parent = leaf._v_parent
        filters_dict = utilsextension.get_filters(parent._v_objectid,
                                                  leaf._v_name)
        if filters_dict is None:
            filters_dict = {}  # not chunked

        # Keyword arguments are all off
        kwargs = dict(complevel=0, shuffle=False, bitshuffle=False,
                      fletcher32=False, least_significant_digit=None,
                      _new=False)
        for (name, values) in six.iteritems(filters_dict):
            if name == 'deflate':
                name = 'zlib'
            if name in all_complibs:
                kwargs['complib'] = name
                if name == "blosc":
                    kwargs['complevel'] = values[4]
                    if values[5] == 1:
                        # Shuffle filter is internal to blosc
                        kwargs['shuffle'] = True
                    elif values[5] == 2:
                        # Shuffle filter is internal to blosc
                        kwargs['bitshuffle'] = True
                    # From Blosc 1.3 on, parameter 6 is used for the compressor
                    if len(values) > 6:
                        cname = blosc_compcode_to_compname(values[6])
                        kwargs['complib'] = "blosc:%s" % cname
                else:
                    kwargs['complevel'] = values[0]
            elif name in foreign_complibs:
                kwargs['complib'] = name
                kwargs['complevel'] = 1  # any nonzero value will do
            elif name in ['shuffle', 'fletcher32']:
                kwargs[name] = True
        return class_(**kwargs)

    @classmethod
    def _unpack(class_, packed):
        """Create a new `Filters` object from a packed version.

        >>> Filters._unpack(0)
        Filters(complevel=0, shuffle=False, bitshuffle=False, fletcher32=False, least_significant_digit=None)
        >>> Filters._unpack(0x101)
        Filters(complevel=1, complib='zlib', shuffle=False, bitshuffle=False, fletcher32=False, least_significant_digit=None)
        >>> Filters._unpack(0x30109)
        Filters(complevel=9, complib='zlib', shuffle=True, bitshuffle=False, fletcher32=True, least_significant_digit=None)
        >>> Filters._unpack(0x3010A)
        Traceback (most recent call last):
          ...
        ValueError: compression level must be between 0 and 9
        >>> Filters._unpack(0x1)
        Traceback (most recent call last):
          ...
        ValueError: invalid compression library id: 0

        """

        kwargs = {'_new': False}

        # Byte 0: compression level.
        kwargs['complevel'] = complevel = packed & 0xff
        packed >>= 8

        # Byte 1: compression library id (0 for none).
        if complevel > 0:
            complib_id = int(packed & 0xff)
            if not (0 < complib_id <= len(all_complibs)):
                raise ValueError("invalid compression library id: %d"
                                 % complib_id)
            kwargs['complib'] = all_complibs[complib_id - 1]
        packed >>= 8

        # Byte 2: parameterless filters.
        kwargs['shuffle'] = packed & _shuffle_flag
        kwargs['bitshuffle'] = packed & _bitshuffle_flag
        kwargs['fletcher32'] = packed & _fletcher32_flag
        has_rounding = packed & _rounding_flag
        packed >>= 8

        # Byte 3: least significant digit.
        if has_rounding:
            kwargs['least_significant_digit'] = numpy.int8(packed & 0xff)
        else:
            kwargs['least_significant_digit'] = None

        return class_(**kwargs)

    def _pack(self):
        """Pack the `Filters` object into a 64-bit NumPy integer."""

        packed = numpy.int64(0)

        # Byte 3: least significant digit.
        if self.least_significant_digit is not None:
            #assert isinstance(self.least_significant_digit, numpy.int8)
            packed |= self.least_significant_digit
        packed <<= 8

        # Byte 2: parameterless filters.
        if self.shuffle:
            packed |= _shuffle_flag
        if self.bitshuffle:
            packed |= _bitshuffle_flag
        if self.fletcher32:
            packed |= _fletcher32_flag
        if self.least_significant_digit:
            packed |= _rounding_flag
        packed <<= 8

        # Byte 1: compression library id (0 for none).
        if self.complevel > 0:
            packed |= all_complibs.index(self.complib) + 1
        packed <<= 8

        # Byte 0: compression level.
        packed |= self.complevel

        return packed

    def __init__(self, complevel=0, complib=default_complib,
                 shuffle=True, bitshuffle=False, fletcher32=False,
                 least_significant_digit=None, _new=True):

        if not (0 <= complevel <= 9):
            raise ValueError("compression level must be between 0 and 9")

        if _new and complevel > 0:
            # These checks are not performed when loading filters from disk.
            if complib not in all_complibs:
                raise ValueError(
                    "compression library ``%s`` is not supported; "
                    "it must be one of: %s"
                    % (complib, ", ".join(all_complibs)))
            if utilsextension.which_lib_version(complib) is None:
                warnings.warn("compression library ``%s`` is not available; "
                              "using ``%s`` instead"
                              % (complib, default_complib), FiltersWarning)
                complib = default_complib  # always available

        complevel = int(complevel)
        complib = str(complib)
        shuffle = bool(shuffle)
        bitshuffle = bool(bitshuffle)
        fletcher32 = bool(fletcher32)
        if least_significant_digit is not None:
            least_significant_digit = numpy.int8(least_significant_digit)

        if complevel == 0:
            # Override some inputs when compression is not enabled.
            complib = None  # make it clear there is no compression
            shuffle = False  # shuffling and not compressing makes no sense
            least_significant_digit = None
        elif complib not in all_complibs:
            # Do not try to use a meaningful level for unsupported libs.
            complevel = -1

        self.complevel = complevel
        """The compression level (0 disables compression)."""

        self.complib = complib
        """The compression filter used (irrelevant when compression is
        not enabled).
        """

        self.shuffle = shuffle
        """Whether the *Shuffle* filter is active or not."""

        self.bitshuffle = bitshuffle
        """Whether the *BitShuffle* filter is active or not."""

        if (self.complib and self.bitshuffle and
            not self.complib.startswith('blosc'):
            raise ValueError(
                "BitShuffle can only be used inside Blosc")

        if self.shuffle and self.bitshuffle:
            raise ValueError(
                "Shuffle and BitShuffle cannot be active at the same time")

        if self.bitshuffle and blosc_version < min_blosc_bitshuffle_version:
            raise ValueError(
                "This Blosc library does not have support for the bitshuffle "
                "filter.  Please update to Blosc >= %s" % \
                min_blosc_bitshuffle_version)

        self.fletcher32 = fletcher32
        """Whether the *Fletcher32* filter is active or not."""

        self.least_significant_digit = least_significant_digit
        """The least significant digit to which data shall be truncated."""

    def __repr__(self):
        args, complevel = [], self.complevel
        if complevel >= 0:  # meaningful compression level
            args.append('complevel=%d' % complevel)
        if complevel != 0:  # compression enabled (-1 or > 0)
            args.append('complib=%r' % self.complib)
        args.append('shuffle=%s' % self.shuffle)
        args.append('bitshuffle=%s' % self.bitshuffle)
        args.append('fletcher32=%s' % self.fletcher32)
        args.append(
            'least_significant_digit=%s' % self.least_significant_digit)
        return '%s(%s)' % (self.__class__.__name__, ', '.join(args))

    def __str__(self):
        return repr(self)

    def __eq__(self, other):
        if not isinstance(other, self.__class__):
            return False
        for attr in self.__dict__:
            if getattr(self, attr) != getattr(other, attr):
                return False
        return True

    # XXX: API incompatible change for PyTables 3 line
    # Overriding __eq__ blocks inheritance of __hash__ in 3.x
    # def __hash__(self):
    #    return hash((self.__class__, self.complevel, self.complib,
    #                 self.shuffle, self.bitshuffle, self.fletcher32))

    def copy(self, **override):
        """Get a copy of the filters, possibly overriding some arguments.

        Constructor arguments to be overridden must be passed as keyword
        arguments.

        Using this method is recommended over replacing the attributes of an
        instance, since instances of this class may become immutable in the
        future::

            >>> filters1 = Filters()
            >>> filters2 = filters1.copy()
            >>> filters1 == filters2
            True
            >>> filters1 is filters2
            False
            >>> filters3 = filters1.copy(complevel=1) #doctest: +ELLIPSIS
            Traceback (most recent call last):
            ...
            ValueError: compression library ``None`` is not supported...
            >>> filters3 = filters1.copy(complevel=1, complib='zlib')
            >>> print(filters1)
            Filters(complevel=0, shuffle=False, bitshuffle=False, fletcher32=False, least_significant_digit=None)
            >>> print(filters3)
            Filters(complevel=1, complib='zlib', shuffle=False, bitshuffle=False, fletcher32=False, least_significant_digit=None)
            >>> filters1.copy(foobar=42)
            Traceback (most recent call last):
            ...
            TypeError: __init__() got an unexpected keyword argument 'foobar'

        """

        newargs = self.__dict__.copy()
        newargs.update(override)
        return self.__class__(**newargs)

# Main part
# =========
def _test():
    """Run ``doctest`` on this module."""

    import doctest
    doctest.testmod()

if __name__ == '__main__':
    _test()
<|endoftext|>


<commit_before>"""
Copyright (C) 2019 J. Paul Getty Trust

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <https://www.gnu.org/licenses/>.
"""

import re
import json
from arches.app.models.system_settings import settings
from arches.app.datatypes.base import BaseDataType
from arches.app.models.models import Widget
from arches.app.search.elasticsearch_dsl_builder import Match, Exists, Term

from rdflib import ConjunctiveGraph as Graph
from rdflib import URIRef, Literal, Namespace
from rdflib.namespace import RDF, RDFS, XSD, DC, DCTERMS

archesproject = Namespace(settings.ARCHES_NAMESPACE_FOR_DATA_EXPORT)
cidoc_nm = Namespace("http://www.cidoc-crm.org/cidoc-crm/")

import logging

logger = logging.getLogger(__name__)

default_widget_name = "urldatatype"
default_url_widget = None
try:
    default_url_widget = Widget.objects.get(name=default_widget_name)
except Widget.DoesNotExist as e:
    logger.warn("Setting 'url' datatype's default widget to None ({0} widget not found).".format(default_widget_name))

details = {
    "datatype": "url",
    "iconclass": "fa fa-location-arrow",
    "modulename": "url.py",
    "classname": "URLDataType",
    "defaultwidget": default_url_widget,
    "defaultconfig": None,
    "configcomponent": "views/components/datatypes/url",
    "configname": "url-datatype-config",
    "isgeometric": False,
    "issearchable": True,
}

class FailRegexURLMatch(Exception):
    pass

class URLDataType(BaseDataType):
    """
    URL Datatype to store an optionally labelled hyperlink to a (typically) external resource
    """

    URL_REGEX = re.compile(r"https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)")

    def validate(self, value, row_number=None, source=None, node=None, nodeid=None):
        errors = []
        try:
            if value.get("url") is not None:
                # check URL conforms to URL structure
                url_test = self.URL_REGEX.match(value["url"])
                if url_test is None:
                    raise FailRegexURLMatch
        except FailRegexURLMatch:
            errors.append(
                {
                    "type": "ERROR",
                    "message": "datatype: {0} value: {1} {2} {3} - {4}. {5}".format(
                        self.datatype_model.datatype,
                        value,
                        source,
                        row_number,
                        "this is not a valid HTTP/HTTPS URL",
                        "This data was not imported.",
                    ),
                }
            )
        return errors

    def get_display_value(self, tile, node):
        data = self.get_tile_data(tile)
        if data:
            display_value = data.get(str(node.nodeid))

            if display_value:
                return json.dumps(display_value)

    def append_to_document(self, document, nodevalue, nodeid, tile, provisional=False):
        if nodevalue.get("url") is not None:
            if nodevalue.get("url_label") is not None:
                val = {
                    "string": nodevalue["url_label"],
                    "nodegroup_id": tile.nodegroup_id,
                    "provisional": provisional,
                }
                document["strings"].append(val)

            # FIXME: URLs searchable?
            val = {
                "string": nodevalue["url"],
                "nodegroup_id": tile.nodegroup_id,
                "provisional": provisional,
            }
            document["strings"].append(val)

    def get_search_terms(self, nodevalue, nodeid=None):
        terms = []
        if nodevalue.get("url") is not None:
            if nodevalue.get("url_label") is not None:
                if settings.WORDS_PER_SEARCH_TERM is None or (len(nodevalue["url_label"].split(" ")) < settings.WORDS_PER_SEARCH_TERM):
                    terms.append(nodevalue["url_label"])
            # terms.append(nodevalue['url'])       FIXME: URLs searchable?
        return terms

    def append_search_filters(self, value, node, query, request):
        # Match the label in the same manner as a String datatype

        try:
            if value["val"] != "":
                match_type = "phrase_prefix" if "~" in value["op"] else "phrase"
                if "~" in value["op"]:
                    match_query = Match(
                        field="tiles.data.%s.url" % (str(node.pk)),
                        query=value["val"],
                        type=match_type,
                    )
                if "=" in value["op"]:
                    match_query = Term(field="tiles.data.%s.url.keyword" % (str(node.pk)), term=value["val"])
                if "!" in value["op"]:
                    query.must_not(match_query)
                    query.filter(Exists(field="tiles.data.%s" % (str(node.pk))))
                else:
                    query.must(match_query)
        except KeyError as e:
            pass

    def get_rdf_uri(self, node, data, which="r"):
        return URIRef(data["url"])

    def accepts_rdf_uri(self, uri):
        return self.URL_REGEX.match(uri) and not (
            uri.startswith("urn:uuid:") or uri.startswith(settings.ARCHES_NAMESPACE_FOR_DATA_EXPORT + "resources/")
        )

    def is_a_literal_in_rdf(self):
        # Should this be a terminating node? Should be True if it is...
        return False

    def ignore_keys(self):
        # We process label into the datatype, so downstream processing should ignore it.
        return ["http://www.w3.org/2000/01/rdf-schema#label http://www.w3.org/2000/01/rdf-schema#Literal"]

    def to_rdf(self, edge_info, edge):
        # returns an in-memory graph object, containing the domain resource, its
        # type and the string as a string literal
        g = Graph()
        if edge_info["range_tile_data"] is not None and edge_info["range_tile_data"].get("url") is not None:
            g.add((edge_info["d_uri"], RDF.type, URIRef(edge.domainnode.ontologyclass)))
            g.add(
                (
                    edge_info["d_uri"],
                    URIRef(edge.ontologyproperty),
                    URIRef(edge_info["range_tile_data"]["url"]),
                )
            )
            if edge_info["range_tile_data"].get("url_label") is not None:
                g.add(
                    (
                        URIRef(edge_info["range_tile_data"]["url"]),
                        RDFS.label,
                        Literal(edge_info["range_tile_data"]["url_label"]),
                    )
                )
        return g

    def from_rdf(self, json_ld_node):
        """
        The json-ld representation of this datatype should look like the following (once expanded)
          {
            "http://www.w3.org/2000/01/rdf-schema#label": [
              {
                "@value": "Link to spectro report"
              }
            ],
            "@id": "https://host/url/to/link"
          }
        """

        value = {}
        if type(json_ld_node) == list:
            url_node = json_ld_node[0]
        else:
            url_node = json_ld_node
        try:
            value["url"] = url_node["@id"]
            value["url_label"] = None
            if "http://www.w3.org/2000/01/rdf-schema#label" in url_node:
                value["url_label"] = url_node["http://www.w3.org/2000/01/rdf-schema#label"][0]["@value"]
        except (IndexError, AttributeError, KeyError) as e:
            print(f"Broke trying to import url datatype: {json_ld_node}")
            return None
        return value

    def default_es_mapping(self):
        """
        Default mapping if not specified is a text field
        """

        return {
            "properties": {
                "url": {
                    "type": "text",
                    "fields": {"keyword": {"ignore_above": 256, "type": "keyword"}},
                },
                "url_label": {
                    "type": "text",
                    "fields": {"keyword": {"ignore_above": 256, "type": "keyword"}},
                },
            }
        }
<commit_msg>fix bug where urls weren't getting imported during csv import, re #7243<commit_after>"""
Copyright (C) 2019 J. Paul Getty Trust

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <https://www.gnu.org/licenses/>.
"""

import re
import json
from arches.app.models.system_settings import settings
from arches.app.datatypes.base import BaseDataType
from arches.app.models.models import Widget
from arches.app.search.elasticsearch_dsl_builder import Match, Exists, Term

from rdflib import ConjunctiveGraph as Graph
from rdflib import URIRef, Literal, Namespace
from rdflib.namespace import RDF, RDFS, XSD, DC, DCTERMS

archesproject = Namespace(settings.ARCHES_NAMESPACE_FOR_DATA_EXPORT)
cidoc_nm = Namespace("http://www.cidoc-crm.org/cidoc-crm/")

import logging

logger = logging.getLogger(__name__)

default_widget_name = "urldatatype"
default_url_widget = None
try:
    default_url_widget = Widget.objects.get(name=default_widget_name)
except Widget.DoesNotExist as e:
    logger.warn("Setting 'url' datatype's default widget to None ({0} widget not found).".format(default_widget_name))

details = {
    "datatype": "url",
    "iconclass": "fa fa-location-arrow",
    "modulename": "url.py",
    "classname": "URLDataType",
    "defaultwidget": default_url_widget,
    "defaultconfig": None,
    "configcomponent": "views/components/datatypes/url",
    "configname": "url-datatype-config",
    "isgeometric": False,
    "issearchable": True,
}

class FailRegexURLMatch(Exception):
    pass

class URLDataType(BaseDataType):
    """
    URL Datatype to store an optionally labelled hyperlink to a (typically) external resource
    """

    URL_REGEX = re.compile(r"https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)")

    def validate(self, value, row_number=None, source=None, node=None, nodeid=None):
        errors = []
        try:
            if value.get("url") is not None:
                # check URL conforms to URL structure
                url_test = self.URL_REGEX.match(value["url"])
                if url_test is None:
                    raise FailRegexURLMatch
        except FailRegexURLMatch:
            errors.append(
                {
                    "type": "ERROR",
                    "message": "datatype: {0} value: {1} {2} {3} - {4}. {5}".format(
                        self.datatype_model.datatype,
                        value,
                        source,
                        row_number,
                        "this is not a valid HTTP/HTTPS URL",
                        "This data was not imported.",
                    ),
                }
            )
        return errors

    def transform_value_for_tile(self, value):
        """
        Used, for example, during import for transforming incomming data to

        Arguments
        value -- can either be a url string like "http://archesproject.org" or
        a json string like '{"url": "", "url_label": ""}'
        """

        ret = {"url": "", "url_label": ""}

        try:
            ret = json.loads(value)
        except BaseException:
            if isinstance(value, dict):
                ret = value
            else:
                ret["url"] = value

        return ret

    def get_display_value(self, tile, node):
        data = self.get_tile_data(tile)
        if data:
            display_value = data.get(str(node.nodeid))

            if display_value:
                return json.dumps(display_value)

    def append_to_document(self, document, nodevalue, nodeid, tile, provisional=False):
        if nodevalue.get("url") is not None:
            if nodevalue.get("url_label") is not None:
                val = {
                    "string": nodevalue["url_label"],
                    "nodegroup_id": tile.nodegroup_id,
                    "provisional": provisional,
                }
                document["strings"].append(val)

            # FIXME: URLs searchable?
            val = {
                "string": nodevalue["url"],
                "nodegroup_id": tile.nodegroup_id,
                "provisional": provisional,
            }
            document["strings"].append(val)

    def get_search_terms(self, nodevalue, nodeid=None):
        terms = []
        if nodevalue.get("url") is not None:
            if nodevalue.get("url_label") is not None:
                if settings.WORDS_PER_SEARCH_TERM is None or (len(nodevalue["url_label"].split(" ")) < settings.WORDS_PER_SEARCH_TERM):
                    terms.append(nodevalue["url_label"])
            # terms.append(nodevalue['url'])       FIXME: URLs searchable?
        return terms

    def append_search_filters(self, value, node, query, request):
        # Match the label in the same manner as a String datatype

        try:
            if value["val"] != "":
                match_type = "phrase_prefix" if "~" in value["op"] else "phrase"
                if "~" in value["op"]:
                    match_query = Match(
                        field="tiles.data.%s.url" % (str(node.pk)),
                        query=value["val"],
                        type=match_type,
                    )
                if "=" in value["op"]:
                    match_query = Term(field="tiles.data.%s.url.keyword" % (str(node.pk)), term=value["val"])
                if "!" in value["op"]:
                    query.must_not(match_query)
                    query.filter(Exists(field="tiles.data.%s" % (str(node.pk))))
                else:
                    query.must(match_query)
        except KeyError as e:
            pass

    def get_rdf_uri(self, node, data, which="r"):
        return URIRef(data["url"])

    def accepts_rdf_uri(self, uri):
        return self.URL_REGEX.match(uri) and not (
            uri.startswith("urn:uuid:") or uri.startswith(settings.ARCHES_NAMESPACE_FOR_DATA_EXPORT + "resources/")
        )

    def is_a_literal_in_rdf(self):
        # Should this be a terminating node? Should be True if it is...
        return False

    def ignore_keys(self):
        # We process label into the datatype, so downstream processing should ignore it.
        return ["http://www.w3.org/2000/01/rdf-schema#label http://www.w3.org/2000/01/rdf-schema#Literal"]

    def to_rdf(self, edge_info, edge):
        # returns an in-memory graph object, containing the domain resource, its
        # type and the string as a string literal
        g = Graph()
        if edge_info["range_tile_data"] is not None and edge_info["range_tile_data"].get("url") is not None:
            g.add((edge_info["d_uri"], RDF.type, URIRef(edge.domainnode.ontologyclass)))
            g.add(
                (
                    edge_info["d_uri"],
                    URIRef(edge.ontologyproperty),
                    URIRef(edge_info["range_tile_data"]["url"]),
                )
            )
            if edge_info["range_tile_data"].get("url_label") is not None:
                g.add(
                    (
                        URIRef(edge_info["range_tile_data"]["url"]),
                        RDFS.label,
                        Literal(edge_info["range_tile_data"]["url_label"]),
                    )
                )
        return g

    def from_rdf(self, json_ld_node):
        """
        The json-ld representation of this datatype should look like the following (once expanded)
          {
            "http://www.w3.org/2000/01/rdf-schema#label": [
              {
                "@value": "Link to spectro report"
              }
            ],
            "@id": "https://host/url/to/link"
          }
        """

        value = {}
        if type(json_ld_node) == list:
            url_node = json_ld_node[0]
        else:
            url_node = json_ld_node
        try:
            value["url"] = url_node["@id"]
            value["url_label"] = None
            if "http://www.w3.org/2000/01/rdf-schema#label" in url_node:
                value["url_label"] = url_node["http://www.w3.org/2000/01/rdf-schema#label"][0]["@value"]
        except (IndexError, AttributeError, KeyError) as e:
            print(f"Broke trying to import url datatype: {json_ld_node}")
            return None
        return value

    def default_es_mapping(self):
        """
        Default mapping if not specified is a text field
        """

        return {
            "properties": {
                "url": {
                    "type": "text",
                    "fields": {"keyword": {"ignore_above": 256, "type": "keyword"}},
                },
                "url_label": {
                    "type": "text",
                    "fields": {"keyword": {"ignore_above": 256, "type": "keyword"}},
                },
            }
        }
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Display WiFi bit rate, quality, signal and SSID using iw.

Configuration parameters:
    - cache_timeout : Update interval in seconds (default: 5)
    - device : Wireless device name (default: "wlan0")
    - down_color : Output color when disconnected, possible values:
      "good", "degraded", "bad" (default: "bad")
      percent (default: false)
    - signal_bad : Bad signal strength in percent (default: 29)
    - signal_degraded : Degraded signal strength in percent
      (default: 49)
    - bitrate_bad : Bad bit rate in Mbit/s (default: 26)
    - bitrate_degraded : Degraded bit rate in Mbit/s (default: 53)
    - round_bitrate : If true, bitrate is rounded to the nearest whole number
      (default: true)
    - format_up : See placeholders below (default:
      "W: {bitrate} {signal_percent} {ssid}").
    - format_down : Output when disconnected (default: "down")

Format of status string placeholders:
    {bitrate} - Display bit rate
    {signal_percent} - Display signal in percent
    {signal_dbm} - Display signal in dBm
    {ssid} - Display SSID

Requires:
    - iw

@author Markus Weimar <mail@markusweimar.de>
@license BSD
"""

import re
import subprocess
from time import time

class Py3status:
    """
    """
    # available configuration parameters
    cache_timeout = 5
    device = 'wlan0'
    down_color = 'bad'
    signal_percent_bad = 29
    signal_percent_degraded = 49
    bitrate_bad = 26
    bitrate_degraded = 53
    round_bitrate = True
    format_up = 'W: {bitrate} {signal_percent} {ssid}'
    format_down = 'W: down'

    def get_wifi(self, i3s_output_list, i3s_config):
        """
        Get WiFi status using iw.
        """
        self.signal_dbm_bad = self._percent_to_dbm(self.signal_percent_bad)
        self.signal_dbm_degraded = \
            self._percent_to_dbm(self.signal_percent_degraded)

        iw = subprocess.check_output(['iw', self.device, 'link']).decode(
            'utf-8')

        bitrate_out = re.search('tx bitrate: ([^\s]+) ([^\s]+)', iw)
        if bitrate_out:
            bitrate = float(bitrate_out.group(1))
            if self.round_bitrate:
                bitrate = round(bitrate)
            bitrate_unit = bitrate_out.group(2)
            if bitrate_unit == 'Gbit/s':
                bitrate *= 1000
        else:
            bitrate = None
            bitrate_unit = None
        signal_out = re.search('signal: ([\-0-9]+)', iw)
        if signal_out:
            signal_dbm = int(signal_out.group(1))
            signal_percent = min(self._dbm_to_percent(signal_dbm), 100)
        else:
            signal_dbm = None
            signal_percent = None
        ssid_out = re.search('SSID: (.+)', iw)
        if ssid_out:
            ssid = ssid_out.group(1)
        else:
            ssid = None

        if ssid is None:
            full_text = self.format_down
            color = i3s_config['color_{}'.format(self.down_color)]
        else:
            if bitrate <= self.bitrate_bad or \
                    signal_dbm <= self.signal_dbm_bad:
                color = i3s_config['color_bad']
            elif bitrate <= self.bitrate_degraded or \
                    signal_dbm <= self.signal_dbm_degraded:
                color = i3s_config['color_degraded']
            else:
                color = i3s_config['color_good']

            bitrate = '{} {}'.format(bitrate, bitrate_unit)
            signal_percent = '{}%'.format(signal_percent)
            signal_dbm = '{} dBm'.format(signal_dbm)

            full_text = self.format_up.format(bitrate=bitrate,
                                              signal_percent=signal_percent,
                                              signal_dbm=signal_dbm,
                                              ssid=ssid)

        response = {
            'cached_until': time() + self.cache_timeout,
            'full_text': full_text,
            'color': color
        }
        return response

    def _percent_to_dbm(self, percent):
        return (percent / 2) - 100

    def _dbm_to_percent(self, dbm):
        return 2 * (dbm + 100)

if __name__ == "__main__":
    """
    Test this module by calling it directly.
    This SHOULD work before contributing your module please.
    """
    from time import sleep
    x = Py3status()
    config = {
        'color_bad': '#FF0000',
        'color_degraded': '#FFFF00',
        'color_good': '#00FF00'
    }
    while True:
        print(x.get_wifi([], config))
        sleep(1)
<commit_msg>Fixed naming of configuration parameters.<commit_after># -*- coding: utf-8 -*-
"""
Display WiFi bit rate, quality, signal and SSID using iw.

Configuration parameters:
    - cache_timeout : Update interval in seconds (default: 5)
    - device : Wireless device name (default: "wlan0")
    - down_color : Output color when disconnected, possible values:
      "good", "degraded", "bad" (default: "bad")
      percent (default: false)
    - signal_bad : Bad signal strength in percent (default: 29)
    - signal_degraded : Degraded signal strength in percent
      (default: 49)
    - bitrate_bad : Bad bit rate in Mbit/s (default: 26)
    - bitrate_degraded : Degraded bit rate in Mbit/s (default: 53)
    - round_bitrate : If true, bitrate is rounded to the nearest whole number
      (default: true)
    - format_up : See placeholders below (default:
      "W: {bitrate} {signal_percent} {ssid}").
    - format_down : Output when disconnected (default: "down")

Format of status string placeholders:
    {bitrate} - Display bit rate
    {signal_percent} - Display signal in percent
    {signal_dbm} - Display signal in dBm
    {ssid} - Display SSID

Requires:
    - iw

@author Markus Weimar <mail@markusweimar.de>
@license BSD
"""

import re
import subprocess
from time import time

class Py3status:
    """
    """
    # available configuration parameters
    cache_timeout = 5
    device = 'wlan0'
    down_color = 'bad'
    signal_bad = 29
    signal_degraded = 49
    bitrate_bad = 26
    bitrate_degraded = 53
    round_bitrate = True
    format_up = 'W: {bitrate} {signal_percent} {ssid}'
    format_down = 'W: down'

    def get_wifi(self, i3s_output_list, i3s_config):
        """
        Get WiFi status using iw.
        """
        self.signal_dbm_bad = self._percent_to_dbm(self.signal_bad)
        self.signal_dbm_degraded = self._percent_to_dbm(self.signal_degraded)

        iw = subprocess.check_output(['iw', self.device, 'link']).decode(
            'utf-8')

        bitrate_out = re.search('tx bitrate: ([^\s]+) ([^\s]+)', iw)
        if bitrate_out:
            bitrate = float(bitrate_out.group(1))
            if self.round_bitrate:
                bitrate = round(bitrate)
            bitrate_unit = bitrate_out.group(2)
            if bitrate_unit == 'Gbit/s':
                bitrate *= 1000
        else:
            bitrate = None
            bitrate_unit = None
        signal_out = re.search('signal: ([\-0-9]+)', iw)
        if signal_out:
            signal_dbm = int(signal_out.group(1))
            signal_percent = min(self._dbm_to_percent(signal_dbm), 100)
        else:
            signal_dbm = None
            signal_percent = None
        ssid_out = re.search('SSID: (.+)', iw)
        if ssid_out:
            ssid = ssid_out.group(1)
        else:
            ssid = None

        if ssid is None:
            full_text = self.format_down
            color = i3s_config['color_{}'.format(self.down_color)]
        else:
            if bitrate <= self.bitrate_bad or \
                    signal_dbm <= self.signal_dbm_bad:
                color = i3s_config['color_bad']
            elif bitrate <= self.bitrate_degraded or \
                    signal_dbm <= self.signal_dbm_degraded:
                color = i3s_config['color_degraded']
            else:
                color = i3s_config['color_good']

            bitrate = '{} {}'.format(bitrate, bitrate_unit)
            signal_percent = '{}%'.format(signal_percent)
            signal_dbm = '{} dBm'.format(signal_dbm)

            full_text = self.format_up.format(bitrate=bitrate,
                                              signal_percent=signal_percent,
                                              signal_dbm=signal_dbm,
                                              ssid=ssid)

        response = {
            'cached_until': time() + self.cache_timeout,
            'full_text': full_text,
            'color': color
        }
        return response

    def _percent_to_dbm(self, percent):
        return (percent / 2) - 100

    def _dbm_to_percent(self, dbm):
        return 2 * (dbm + 100)

if __name__ == "__main__":
    """
    Test this module by calling it directly.
    This SHOULD work before contributing your module please.
    """
    from time import sleep
    x = Py3status()
    config = {
        'color_bad': '#FF0000',
        'color_degraded': '#FFFF00',
        'color_good': '#00FF00'
    }
    while True:
        print(x.get_wifi([], config))
        sleep(1)
<|endoftext|>


<commit_before># Add the App Engine SDK's bundled libraries (django, mox, webob, yaml, etc.) to
# sys.path so we can use them instead of adding them all to tests_require in
# setup.py.
# https://cloud.google.com/appengine/docs/python/tools/localunittesting#Python_Setting_up_a_testing_framework
import dev_appserver
dev_appserver.fix_sys_path()

# Suppress logging by default.
import logging, sys
if '--debug' in sys.argv:
  sys.argv.remove('--debug')
  logging.getLogger().setLevel(logging.DEBUG)
else:
  logging.disable(logging.CRITICAL + 1)

# Monkey patch to fix template loader issue:
#
# File "/usr/local/google_appengine/lib/django-1.4/django/template/loader.py", line 101, in find_template_loader:
# ImproperlyConfigured: Error importing template source loader django.template.loaders.filesystem.load_template_source: "'module' object has no attribute 'load_template_source'"
from django.template.loaders import filesystem
filesystem.load_template_source = filesystem._loader.load_template_source
<commit_msg>use app engine's mox; automatically show logging when running single test(s)<commit_after># Add the App Engine SDK's bundled libraries (django, webob, yaml, etc.) to
# sys.path so we can use them instead of adding them all to tests_require in
# setup.py.
# https://cloud.google.com/appengine/docs/python/tools/localunittesting#Python_Setting_up_a_testing_framework
import dev_appserver
dev_appserver.fix_sys_path()

# Also use the App Engine SDK's mox because it has bug fixes that aren't in pypi
# 0.5.3. (Annoyingly, they both say they're version 0.5.3.)
import os, sys
sys.path.append(os.path.join(dev_appserver._DIR_PATH, 'lib', 'mox'))

# Show logging when running individual test modules, methods, etc; suppress it
# when running all tests.
import logging
if sys.argv[-1] != 'discover':
  logging.getLogger().setLevel(logging.DEBUG)
else:
  logging.disable(logging.CRITICAL + 1)

# Monkey patch to fix template loader issue:
#
# File "/usr/local/google_appengine/lib/django-1.4/django/template/loader.py", line 101, in find_template_loader:
# ImproperlyConfigured: Error importing template source loader django.template.loaders.filesystem.load_template_source: "'module' object has no attribute 'load_template_source'"
from django.template.loaders import filesystem
filesystem.load_template_source = filesystem._loader.load_template_source
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on 2017-9-22

@author: cheng.li
"""

from typing import Tuple
import pandas as pd
from alphamind.execution.baseexecutor import ExecutorBase

class ThresholdExecutor(ExecutorBase):

    def __init__(self, turn_over_threshold: float):
        super().__init__()
        self.threshold = turn_over_threshold
        self.current_pos = pd.DataFrame()

    def execute(self, target_pos: pd.DataFrame) -> Tuple[float, pd.DataFrame]:

        if self.current_pos.empty:
            self.current_pos = target_pos.copy()
            return target_pos.weight.sum(), target_pos.copy()
        else:
            turn_over = self.calc_turn_over(target_pos, self.current_pos)

            if turn_over >= self.threshold:
                self.current_pos = target_pos.copy()
                return turn_over, self.current_pos.copy()
            else:
                return 0., self.current_pos.copy()
<commit_msg>turn over threshold to be relative<commit_after># -*- coding: utf-8 -*-
"""
Created on 2017-9-22

@author: cheng.li
"""

from typing import Tuple
import pandas as pd
from alphamind.execution.baseexecutor import ExecutorBase

class ThresholdExecutor(ExecutorBase):

    def __init__(self, turn_over_threshold: float):
        super().__init__()
        self.threshold = turn_over_threshold
        self.current_pos = pd.DataFrame()

    def execute(self, target_pos: pd.DataFrame) -> Tuple[float, pd.DataFrame]:

        if self.current_pos.empty:
            self.current_pos = target_pos.copy()
            return target_pos.weight.sum(), target_pos.copy()
        else:
            turn_over = self.calc_turn_over(target_pos, self.current_pos)

            if turn_over >= self.threshold * self.current_pos.weight.sum():
                self.current_pos = target_pos.copy()
                return turn_over, self.current_pos.copy()
            else:
                return 0., self.current_pos.copy()
<|endoftext|>


<commit_before>import re
import logging
#from src.tree_node import TreeNode

class BooleanQueryParser(object):

    AND = 'AND'
    OR = 'OR'
    NOT = 'NOT'
    LBRACKET = '('
    RBRACKET = ')'

    def parse_dnf_query(self, query):
        logging.info('parsing query: ' + query)

        if self.AND in query and self.OR not in query:
            query = self.LBRACKET + query + self.RBRACKET
            logging.debug('query has 1+ ANDs, 0 ORs -> added brackets: ' + str(query))

        query_toks = re.split('(\(|\)| )', query)
        query_toks = list(filter(str.strip, query_toks))
        logging.debug('parsed query tokens: ' + str(query_toks))

        outer_list = []
        inner_list = None
        current_list = outer_list
        for i in range(0, len(query_toks)):
            tok = query_toks[i]
            if tok == self.LBRACKET:
                inner_list = []
                current_list = inner_list
            elif tok == self.RBRACKET:
                outer_list.append(inner_list)
                current_list = outer_list
            elif self._is_word(tok):
                is_pos_literal = (i == 0 or query_toks[i-1] != self.NOT)
                current_list.append((tok, is_pos_literal))

        return outer_list

    def _is_word(self, token):
        return token not in (self.AND, self.OR, self.NOT, self.LBRACKET, self.RBRACKET)

    #===========================================================================
    # def parse(self, query):
    #     logging.debug('parsing query ' + query)
    #     tok_query_whitespace = re.split("(\(|\)| )", query)
    #     tok_query = list(filter(str.strip, tok_query_whitespace))
    #     logging.debug('tokenized query without whitespace ' + str(tok_query))
    #
    #     i = 0
    #     curr_node = TreeNode()
    #     while i < len(tok_query):
    #         tok = tok_query[i]
    #         if tok == '(':
    #             new_node = TreeNode(curr_node)
    #             curr_node.childs.append(new_node)
    #             curr_node = new_node
    #         elif tok == ')':
    #             curr_node = curr_node.parent
    #         elif tok in ('AND', 'OR'):
    #             curr_node.key = tok
    #         elif tok == 'NOT':
    #             # nimm nachfolgendes Token und fge NOT-Operation mit akt. Token
    #             new_node = TreeNode(curr_node, tok)
    #             new_node.childs.append(TreeNode(new_node, tok_query[i + 1]))
    #             curr_node.childs.append(new_node)
    #             # berspringe nachfolgendes Token
    #             i += 1
    #         else:
    #             curr_node.childs.append(TreeNode(curr_node, tok))
    #         i += 1
    #     logging.debug('tokenized query without whitespace ' + str(tok_query))
    #
    #     return curr_node
    #
    # def parse2(self, query):
    #     logging.debug('parsing query ' + query)
    #
    #     query_toks = re.split("(\(|\)| )", query)
    #     query_toks = list(filter(str.strip, query_toks))
    #     logging.debug('parsed query tokens ' + str(query_toks))
    #
    #     tuple_toks = self._replace_words_by_tuples(query_toks)
    #     logging.debug('replaced words by tuples ' + str(tuple_toks))
    #
    #     nested_toks = self._replace_quotes_by_nested_lists(tuple_toks)
    #     logging.debug('replaced quotes by nested lists ' + str(nested_toks))
    #
    #     return nested_toks
    #
    #
    # def _replace_words_by_tuples(self, query_toks):
    #     tuple_toks = []
    #     for i in range(0, len(query_toks)):
    #         tok = query_toks[i]
    #         if self._is_word(tok):
    #             is_pos_literal = i == 0 or query_toks[i-1] != 'NOT'
    #             tuple_toks.append((tok, is_pos_literal))
    #         elif tok not in ('AND', 'OR', 'NOT'):
    #             tuple_toks.append(tok)
    #     return tuple_toks
    #
    # def _replace_quotes_by_nested_lists(self, tuple_toks):
    #     nested_toks = []
    #     i = 0
    #     while i < len(tuple_toks):
    #         tok = tuple_toks[i]
    #         if tok == '(':
    #             inner_array = []
    #             j = i + 1
    #             while j < len(tuple_toks) and tuple_toks[j] != ')':
    #                 inner_array.append(tuple_toks[j])
    #                 j += 1
    #             nested_toks.append(inner_array)
    #             i = j
    #
    #         else:
    #             nested_toks.append(tok)
    #         i += 1
    #     return nested_toks
    #
    #===========================================================================

            <commit_msg>extracted method _generate_nested_tuple_list()<commit_after>import re
import logging
#from src.tree_node import TreeNode

class BooleanQueryParser(object):

    AND = 'AND'
    OR = 'OR'
    NOT = 'NOT'
    LBRACKET = '('
    RBRACKET = ')'

    def _generate_nested_tuple_list(self, query_toks):
        outer_list = []
        inner_list = None
        current_list = outer_list
        for i in range(0, len(query_toks)):
            tok = query_toks[i]
            if tok == self.LBRACKET:
                inner_list = []
                current_list = inner_list
            elif tok == self.RBRACKET:
                outer_list.append(inner_list)
                current_list = outer_list
            elif self._is_word(tok):
                is_pos_literal = i == 0 or query_toks[i - 1] != self.NOT
                current_list.append((tok, is_pos_literal))

        return outer_list

    def parse_dnf_query(self, query):
        logging.info('parsing query: ' + query)

        if self.AND in query and self.OR not in query:
            query = self.LBRACKET + query + self.RBRACKET
            logging.debug('query has 1+ ANDs, 0 ORs -> added brackets: ' + str(query))

        query_toks = re.split('(\(|\)| )', query)
        query_toks = list(filter(str.strip, query_toks))
        logging.debug('parsed query tokens: ' + str(query_toks))

        return self._generate_nested_tuple_list(query_toks)

    def _is_word(self, token):
        return token not in (self.AND, self.OR, self.NOT, self.LBRACKET, self.RBRACKET)

    #===========================================================================
    # def parse(self, query):
    #     logging.debug('parsing query ' + query)
    #     tok_query_whitespace = re.split("(\(|\)| )", query)
    #     tok_query = list(filter(str.strip, tok_query_whitespace))
    #     logging.debug('tokenized query without whitespace ' + str(tok_query))
    #
    #     i = 0
    #     curr_node = TreeNode()
    #     while i < len(tok_query):
    #         tok = tok_query[i]
    #         if tok == '(':
    #             new_node = TreeNode(curr_node)
    #             curr_node.childs.append(new_node)
    #             curr_node = new_node
    #         elif tok == ')':
    #             curr_node = curr_node.parent
    #         elif tok in ('AND', 'OR'):
    #             curr_node.key = tok
    #         elif tok == 'NOT':
    #             # nimm nachfolgendes Token und fge NOT-Operation mit akt. Token
    #             new_node = TreeNode(curr_node, tok)
    #             new_node.childs.append(TreeNode(new_node, tok_query[i + 1]))
    #             curr_node.childs.append(new_node)
    #             # berspringe nachfolgendes Token
    #             i += 1
    #         else:
    #             curr_node.childs.append(TreeNode(curr_node, tok))
    #         i += 1
    #     logging.debug('tokenized query without whitespace ' + str(tok_query))
    #
    #     return curr_node
    #
    # def parse2(self, query):
    #     logging.debug('parsing query ' + query)
    #
    #     query_toks = re.split("(\(|\)| )", query)
    #     query_toks = list(filter(str.strip, query_toks))
    #     logging.debug('parsed query tokens ' + str(query_toks))
    #
    #     tuple_toks = self._replace_words_by_tuples(query_toks)
    #     logging.debug('replaced words by tuples ' + str(tuple_toks))
    #
    #     nested_toks = self._replace_quotes_by_nested_lists(tuple_toks)
    #     logging.debug('replaced quotes by nested lists ' + str(nested_toks))
    #
    #     return nested_toks
    #
    #
    # def _replace_words_by_tuples(self, query_toks):
    #     tuple_toks = []
    #     for i in range(0, len(query_toks)):
    #         tok = query_toks[i]
    #         if self._is_word(tok):
    #             is_pos_literal = i == 0 or query_toks[i-1] != 'NOT'
    #             tuple_toks.append((tok, is_pos_literal))
    #         elif tok not in ('AND', 'OR', 'NOT'):
    #             tuple_toks.append(tok)
    #     return tuple_toks
    #
    # def _replace_quotes_by_nested_lists(self, tuple_toks):
    #     nested_toks = []
    #     i = 0
    #     while i < len(tuple_toks):
    #         tok = tuple_toks[i]
    #         if tok == '(':
    #             inner_array = []
    #             j = i + 1
    #             while j < len(tuple_toks) and tuple_toks[j] != ')':
    #                 inner_array.append(tuple_toks[j])
    #                 j += 1
    #             nested_toks.append(inner_array)
    #             i = j
    #
    #         else:
    #             nested_toks.append(tok)
    #         i += 1
    #     return nested_toks
    #
    #===========================================================================

            <|endoftext|>