<commit_before># coding: utf-8
"""
This modules defines an SQLAlchemy ORM model for a flat.
"""
# pylint: disable=locally-disabled,invalid-name,too-few-public-methods
from __future__ import absolute_import, print_function, unicode_literals

import logging
import enum

import arrow

from sqlalchemy import (
    Column, DateTime, Enum, Float, SmallInteger, String, Text, inspect
)
from sqlalchemy.orm import validates

from flatisfy.database.base import BASE
from flatisfy.database.types import MagicJSON

LOGGER = logging.getLogger(__name__)

class FlatUtilities(enum.Enum):
    """
    An enum of the possible utilities status for a flat entry.
    """
    included = 10
    unknown = 0
    excluded = -10

class FlatStatus(enum.Enum):
    """
    An enum of the possible status for a flat entry.
    """
    user_deleted = -100
    duplicate = -20
    ignored = -10
    new = 0
    followed = 10
    contacted = 20
    answer_no = 30
    answer_yes = 31

# List of statuses that are automatically handled, and which the user cannot
# manually set through the UI.
AUTOMATED_STATUSES = [
    FlatStatus.new,
    FlatStatus.duplicate,
    FlatStatus.ignored
]

class Flat(BASE):
    """
    SQLAlchemy ORM model to store a flat.
    """
    __tablename__ = "flats"
    __searchable__ = [
        "title", "text", "station", "location", "details", "notes"
    ]

    # Weboob data
    id = Column(String, primary_key=True)
    area = Column(Float)
    bedrooms = Column(Float)
    cost = Column(Float)
    currency = Column(String)
    utilities = Column(Enum(FlatUtilities), default=FlatUtilities.unknown)
    date = Column(DateTime)
    details = Column(MagicJSON)
    location = Column(String)
    phone = Column(String)
    photos = Column(MagicJSON)
    rooms = Column(Float)
    station = Column(String)
    text = Column(Text)
    title = Column(String)
    urls = Column(MagicJSON)
    merged_ids = Column(MagicJSON)
    notes = Column(Text)
    notation = Column(SmallInteger, default=0)

    # Flatisfy data
    # TODO: Should be in another table with relationships
    flatisfy_stations = Column(MagicJSON)
    flatisfy_postal_code = Column(String)
    flatisfy_time_to = Column(MagicJSON)
    flatisfy_constraint = Column(String)

    # Status
    status = Column(Enum(FlatStatus), default=FlatStatus.new)

    # Date for visit
    visit_date = Column(DateTime)

    @validates('utilities')
    def validate_utilities(self, _, utilities):
        """
        Utilities validation method
        """
        if isinstance(utilities, FlatUtilities):
            return utilities

        if utilities == "C.C.":
            return FlatUtilities.included
        elif utilities == "H.C.":
            return FlatUtilities.excluded
        return FlatUtilities.unknown

    @validates("status")
    def validate_status(self, _, status):
        """
        Status validation method
        """
        if isinstance(status, FlatStatus):
            return status

        try:
            return getattr(FlatStatus, status)
        except (AttributeError, TypeError):
            LOGGER.warn("Unkown flat status %s, ignoring it.",
                        status)
            return self.status.default.arg

    @validates("notation")
    def validate_notation(self, _, notation):
        """
        Notation validation method
        """
        try:
            notation = int(notation)
            assert notation >= 0 and notation <= 5
        except (ValueError, AssertionError):
            raise ValueError('notation should be an integer between 0 and 5')
        return notation

    @validates("date")
    def validate_date(self, _, date):
        """
        Date validation method
        """
        return arrow.get(date).naive

    @validates("visit_date")
    def validate_visit_date(self, _, visit_date):
        """
        Visit date validation method
        """
        return arrow.get(visit_date).naive

    @validates("photos")
    def validate_photos(self, _, photos):
        """
        Photos validation method
        """
        for photo in photos:
            try:
                # Remove computed hash to avoid storing it in db
                del photo["hash"]
            except KeyError:
                pass
        return photos

    @staticmethod
    def from_dict(flat_dict):
        """
        Create a Flat object from a flat dict as manipulated by the filtering
        pass.
        """
        # Handle flatisfy metadata
        flat_dict = flat_dict.copy()
        if "flatisfy" in flat_dict:
            flat_dict["flatisfy_stations"] = (
                flat_dict["flatisfy"].get("matched_stations", [])
            )
            flat_dict["flatisfy_postal_code"] = (
                flat_dict["flatisfy"].get("postal_code", None)
            )
            flat_dict["flatisfy_time_to"] = (
                flat_dict["flatisfy"].get("time_to", {})
            )
            flat_dict["flatisfy_constraint"] = (
                flat_dict["flatisfy"].get("constraint", "default")
            )
            del flat_dict["flatisfy"]

        flat_dict = {k: v for k, v in flat_dict.items()
                     if k in inspect(Flat).columns.keys()}
        return Flat(**flat_dict)

    def __repr__(self):
        return "<Flat(id=%s, urls=%s)>" % (self.id, self.urls)

    def json_api_repr(self):
        """
        Return a dict representation of this flat object that is JSON
        serializable.
        """
        flat_repr = {
            k: v
            for k, v in self.__dict__.items()
            if not k.startswith("_")
        }
        if isinstance(flat_repr["status"], FlatStatus):
            flat_repr["status"] = flat_repr["status"].name
        if isinstance(flat_repr["utilities"], FlatUtilities):
            flat_repr["utilities"] = flat_repr["utilities"].name

        return flat_repr
<commit_msg>Avoid a crash when no photos are found for a flat<commit_after># coding: utf-8
"""
This modules defines an SQLAlchemy ORM model for a flat.
"""
# pylint: disable=locally-disabled,invalid-name,too-few-public-methods
from __future__ import absolute_import, print_function, unicode_literals

import logging
import enum

import arrow

from sqlalchemy import (
    Column, DateTime, Enum, Float, SmallInteger, String, Text, inspect
)
from sqlalchemy.orm import validates

from flatisfy.database.base import BASE
from flatisfy.database.types import MagicJSON

LOGGER = logging.getLogger(__name__)

class FlatUtilities(enum.Enum):
    """
    An enum of the possible utilities status for a flat entry.
    """
    included = 10
    unknown = 0
    excluded = -10

class FlatStatus(enum.Enum):
    """
    An enum of the possible status for a flat entry.
    """
    user_deleted = -100
    duplicate = -20
    ignored = -10
    new = 0
    followed = 10
    contacted = 20
    answer_no = 30
    answer_yes = 31

# List of statuses that are automatically handled, and which the user cannot
# manually set through the UI.
AUTOMATED_STATUSES = [
    FlatStatus.new,
    FlatStatus.duplicate,
    FlatStatus.ignored
]

class Flat(BASE):
    """
    SQLAlchemy ORM model to store a flat.
    """
    __tablename__ = "flats"
    __searchable__ = [
        "title", "text", "station", "location", "details", "notes"
    ]

    # Weboob data
    id = Column(String, primary_key=True)
    area = Column(Float)
    bedrooms = Column(Float)
    cost = Column(Float)
    currency = Column(String)
    utilities = Column(Enum(FlatUtilities), default=FlatUtilities.unknown)
    date = Column(DateTime)
    details = Column(MagicJSON)
    location = Column(String)
    phone = Column(String)
    photos = Column(MagicJSON)
    rooms = Column(Float)
    station = Column(String)
    text = Column(Text)
    title = Column(String)
    urls = Column(MagicJSON)
    merged_ids = Column(MagicJSON)
    notes = Column(Text)
    notation = Column(SmallInteger, default=0)

    # Flatisfy data
    # TODO: Should be in another table with relationships
    flatisfy_stations = Column(MagicJSON)
    flatisfy_postal_code = Column(String)
    flatisfy_time_to = Column(MagicJSON)
    flatisfy_constraint = Column(String)

    # Status
    status = Column(Enum(FlatStatus), default=FlatStatus.new)

    # Date for visit
    visit_date = Column(DateTime)

    @validates('utilities')
    def validate_utilities(self, _, utilities):
        """
        Utilities validation method
        """
        if isinstance(utilities, FlatUtilities):
            return utilities

        if utilities == "C.C.":
            return FlatUtilities.included
        elif utilities == "H.C.":
            return FlatUtilities.excluded
        return FlatUtilities.unknown

    @validates("status")
    def validate_status(self, _, status):
        """
        Status validation method
        """
        if isinstance(status, FlatStatus):
            return status

        try:
            return getattr(FlatStatus, status)
        except (AttributeError, TypeError):
            LOGGER.warn("Unkown flat status %s, ignoring it.",
                        status)
            return self.status.default.arg

    @validates("notation")
    def validate_notation(self, _, notation):
        """
        Notation validation method
        """
        try:
            notation = int(notation)
            assert notation >= 0 and notation <= 5
        except (ValueError, AssertionError):
            raise ValueError('notation should be an integer between 0 and 5')
        return notation

    @validates("date")
    def validate_date(self, _, date):
        """
        Date validation method
        """
        return arrow.get(date).naive

    @validates("visit_date")
    def validate_visit_date(self, _, visit_date):
        """
        Visit date validation method
        """
        return arrow.get(visit_date).naive

    @validates("photos")
    def validate_photos(self, _, photos):
        """
        Photos validation method
        """
        if not photos:
            photos = []
        for photo in photos:
            try:
                # Remove computed hash to avoid storing it in db
                del photo["hash"]
            except KeyError:
                pass
        return photos

    @staticmethod
    def from_dict(flat_dict):
        """
        Create a Flat object from a flat dict as manipulated by the filtering
        pass.
        """
        # Handle flatisfy metadata
        flat_dict = flat_dict.copy()
        if "flatisfy" in flat_dict:
            flat_dict["flatisfy_stations"] = (
                flat_dict["flatisfy"].get("matched_stations", [])
            )
            flat_dict["flatisfy_postal_code"] = (
                flat_dict["flatisfy"].get("postal_code", None)
            )
            flat_dict["flatisfy_time_to"] = (
                flat_dict["flatisfy"].get("time_to", {})
            )
            flat_dict["flatisfy_constraint"] = (
                flat_dict["flatisfy"].get("constraint", "default")
            )
            del flat_dict["flatisfy"]

        flat_dict = {k: v for k, v in flat_dict.items()
                     if k in inspect(Flat).columns.keys()}
        return Flat(**flat_dict)

    def __repr__(self):
        return "<Flat(id=%s, urls=%s)>" % (self.id, self.urls)

    def json_api_repr(self):
        """
        Return a dict representation of this flat object that is JSON
        serializable.
        """
        flat_repr = {
            k: v
            for k, v in self.__dict__.items()
            if not k.startswith("_")
        }
        if isinstance(flat_repr["status"], FlatStatus):
            flat_repr["status"] = flat_repr["status"].name
        if isinstance(flat_repr["utilities"], FlatUtilities):
            flat_repr["utilities"] = flat_repr["utilities"].name

        return flat_repr
<|endoftext|>


<commit_before>#!/usr/bin/python

import unittest
import sys
import getopt

all_modules=["vcard", "jid", "disco", "cache","stream_reader",
                    "message", "presence", "dataforms", "interface"]

def suite(modules=None):
     if not modules:
        modules=all_modules
     suite = unittest.TestSuite()
     for mname in modules:
        mod=__import__(mname)
        suite.addTest(mod.suite())
     return suite

def usage():
    print "Usage:"
    print "  %s [-v <verbosity>] " % (sys.argv[0],)
    print "  %s -h"

def main(args=None):
    verbosity=1
    if args:
        try:
            optlist, modules = getopt.getopt(args, 'v:h')
        except getopt.GetoptError:
            usage()
            sys.exit(2)
        for o,v in optlist:
            if o=='-v':
                verbosity=int(v)
            elif o=='-h':
                usage()
                sys.exit(0)
    else:
        modules=None

    unittest.TextTestRunner(verbosity=verbosity).run(suite())

if __name__ == '__main__':
    main(sys.argv[1:])
# vi: sts=4 et sw=4
<commit_msg>Run only tests that make sense for current pyxmpp2<commit_after>#!/usr/bin/python

import unittest
import sys
import getopt

all_modules=[ "stream_reader", "xmppserializer",
#                "vcard", "jid", "disco", "cache","stream_reader",
#                 "message", "presence", "dataforms", "interface"
                ]

def suite(modules=None):
     if not modules:
        modules=all_modules
     suite = unittest.TestSuite()
     for mname in modules:
        mod=__import__(mname)
        suite.addTest(mod.suite())
     return suite

def usage():
    print "Usage:"
    print "  %s [-v <verbosity>] " % (sys.argv[0],)
    print "  %s -h"

def main(args=None):
    verbosity=1
    if args:
        try:
            optlist, modules = getopt.getopt(args, 'v:h')
        except getopt.GetoptError:
            usage()
            sys.exit(2)
        for o,v in optlist:
            if o=='-v':
                verbosity=int(v)
            elif o=='-h':
                usage()
                sys.exit(0)
    else:
        modules=None

    unittest.TextTestRunner(verbosity=verbosity).run(suite())

if __name__ == '__main__':
    main(sys.argv[1:])
# vi: sts=4 et sw=4
<|endoftext|>


<commit_before># Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
APIs for parsing and validating configuration.
"""

from __future__ import unicode_literals, absolute_import

from ._model import Application, DockerImage, Node, Deployment

class ConfigurationError(Exception):
    """
    Some part of the supplied configuration was wrong.

    The exception message will include some details about what.
    """

class Configuration(object):
    """
    Validate and parse configurations.
    """
    def _applications_from_configuration(self, application_configuration):
        """
        Validate and parse a given application configuration.

        :param dict application_configuration: The intermediate configuration
            representation to load into ``Application`` instances.  See
            :ref:`Configuration` for details.

        :raises ConfigurationError: if there are validation errors.

        :returns: A ``dict`` mapping application names to ``Application``
            instances.
        """
        if 'applications' not in application_configuration:
            raise ConfigurationError('Missing applications key')

        applications = {}
        for application_name, config in (
                application_configuration['applications'].items()):
            try:
                image_name = config.pop('image')
            except KeyError as e:
                raise ConfigurationError(
                    ("Application '{application_name}' has a config error. "
                     "Missing value for '{message}'.").format(
                         application_name=application_name, message=e.message)
                )

            try:
                image = DockerImage.from_string(image_name)
            except ValueError as e:
                raise ConfigurationError(
                    ("Application '{application_name}' has a config error. "
                     "Invalid Docker image name. {message}").format(
                         application_name=application_name, message=e.message)
                )

            applications[application_name] = Application(name=application_name,
                                                         image=image)

            if config:
                raise ConfigurationError(
                    ("Application '{application_name}' has a config error. "
                     "Unrecognised keys: {keys}.").format(
                         application_name=application_name,
                         keys=', '.join(config.keys()))
                )
        return applications

    def _deployment_from_configuration(self, deployment_configuration,
                                       all_applications):
        """
        Validate and parse a given deployment configuration.

        :param dict deployment_configuration: The intermediate configuration
            representation to load into ``Node`` instances.  See
            :ref:`Configuration` for details.

        :raises ConfigurationError: if there are validation errors.

        :returns: A ``set`` of ``Node`` instances.
        """
        if 'nodes' not in deployment_configuration:
            raise ConfigurationError('Missing nodes key')

        nodes = []
        for hostname, application_names in (
                deployment_configuration['nodes'].items()):
            if not isinstance(application_names, list):
                raise ConfigurationError(
                    "Node {node_name} has a config error. "
                    "Wrong value type: {value_type}. "
                    "Should be list.".format(
                        node_name=hostname,
                        value_type=application_names.__class__.__name__)
                )
            node_applications = []
            for name in application_names:
                application = all_applications.get(name)
                if application is None:
                    raise ConfigurationError(
                        "Node {hostname} has a config error. "
                        "Unrecognised application name: "
                        "{application_name}.".format(
                            hostname=hostname, application_name=name)
                    )
                node_applications.append(application)
            node = Node(hostname=hostname,
                        applications=frozenset(node_applications))
            nodes.append(node)
        return set(nodes)

    def model_from_configuration(self, application_configuration,
                                 deployment_configuration):
        """
        Validate and coerce the supplied application configuration and
        deployment configuration dictionaries into a ``Deployment`` instance.

        :param dict application_configuration: Map of applications to Docker
            images.

        :param dict deployment_configuration: Map of node names to application
            names.

        :raises ValueError: if there are validation errors.

        :raises KeyError: if there are validation errors.

        :returns: A ``Deployment`` object.
        """
        applications = self._applications_from_configuration(
            application_configuration)
        nodes = self._deployment_from_configuration(
            deployment_configuration, applications)
        return Deployment(nodes=frozenset(nodes))

model_from_configuration = Configuration().model_from_configuration
<commit_msg>One string is unicode<commit_after># Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
APIs for parsing and validating configuration.
"""

from __future__ import unicode_literals, absolute_import

from ._model import Application, DockerImage, Node, Deployment

class ConfigurationError(Exception):
    """
    Some part of the supplied configuration was wrong.

    The exception message will include some details about what.
    """

class Configuration(object):
    """
    Validate and parse configurations.
    """
    def _applications_from_configuration(self, application_configuration):
        """
        Validate and parse a given application configuration.

        :param dict application_configuration: The intermediate configuration
            representation to load into ``Application`` instances.  See
            :ref:`Configuration` for details.

        :raises ConfigurationError: if there are validation errors.

        :returns: A ``dict`` mapping application names to ``Application``
            instances.
        """
        if u'applications' not in application_configuration:
            raise ConfigurationError('Missing applications key')

        applications = {}
        for application_name, config in (
                application_configuration['applications'].items()):
            try:
                image_name = config.pop('image')
            except KeyError as e:
                raise ConfigurationError(
                    ("Application '{application_name}' has a config error. "
                     "Missing value for '{message}'.").format(
                         application_name=application_name, message=e.message)
                )

            try:
                image = DockerImage.from_string(image_name)
            except ValueError as e:
                raise ConfigurationError(
                    ("Application '{application_name}' has a config error. "
                     "Invalid Docker image name. {message}").format(
                         application_name=application_name, message=e.message)
                )

            applications[application_name] = Application(name=application_name,
                                                         image=image)

            if config:
                raise ConfigurationError(
                    ("Application '{application_name}' has a config error. "
                     "Unrecognised keys: {keys}.").format(
                         application_name=application_name,
                         keys=', '.join(config.keys()))
                )
        return applications

    def _deployment_from_configuration(self, deployment_configuration,
                                       all_applications):
        """
        Validate and parse a given deployment configuration.

        :param dict deployment_configuration: The intermediate configuration
            representation to load into ``Node`` instances.  See
            :ref:`Configuration` for details.

        :raises ConfigurationError: if there are validation errors.

        :returns: A ``set`` of ``Node`` instances.
        """
        if 'nodes' not in deployment_configuration:
            raise ConfigurationError('Missing nodes key')

        nodes = []
        for hostname, application_names in (
                deployment_configuration['nodes'].items()):
            if not isinstance(application_names, list):
                raise ConfigurationError(
                    "Node {node_name} has a config error. "
                    "Wrong value type: {value_type}. "
                    "Should be list.".format(
                        node_name=hostname,
                        value_type=application_names.__class__.__name__)
                )
            node_applications = []
            for name in application_names:
                application = all_applications.get(name)
                if application is None:
                    raise ConfigurationError(
                        "Node {hostname} has a config error. "
                        "Unrecognised application name: "
                        "{application_name}.".format(
                            hostname=hostname, application_name=name)
                    )
                node_applications.append(application)
            node = Node(hostname=hostname,
                        applications=frozenset(node_applications))
            nodes.append(node)
        return set(nodes)

    def model_from_configuration(self, application_configuration,
                                 deployment_configuration):
        """
        Validate and coerce the supplied application configuration and
        deployment configuration dictionaries into a ``Deployment`` instance.

        :param dict application_configuration: Map of applications to Docker
            images.

        :param dict deployment_configuration: Map of node names to application
            names.

        :raises ValueError: if there are validation errors.

        :raises KeyError: if there are validation errors.

        :returns: A ``Deployment`` object.
        """
        applications = self._applications_from_configuration(
            application_configuration)
        nodes = self._deployment_from_configuration(
            deployment_configuration, applications)
        return Deployment(nodes=frozenset(nodes))

model_from_configuration = Configuration().model_from_configuration
<|endoftext|>


<commit_before># flake8: noqa
from django.forms import (BaseModelForm, model_to_dict, fields_for_model,
                          save_instance, ValidationError, Media,
                          MediaDefiningClass)

from .fields import *
from .forms import *
from .models import *
from .widgets import *

try:
    from . import gis
except Exception:
    import warnings
    warnings.warn(
        "Unable to import floppyforms.gis, geometry widgets not available")

__version__ = '1.5.1'
<commit_msg>Preparing 1.5.2 release<commit_after># flake8: noqa
from django.forms import (BaseModelForm, model_to_dict, fields_for_model,
                          save_instance, ValidationError, Media,
                          MediaDefiningClass)

from .fields import *
from .forms import *
from .models import *
from .widgets import *

try:
    from . import gis
except Exception:
    import warnings
    warnings.warn(
        "Unable to import floppyforms.gis, geometry widgets not available")

__version__ = '1.5.2'
<|endoftext|>


<commit_before>"""
Various transforms for FCS data
"""

from scipy.optimize import brentq
from scipy import interpolate
import numpy

import logicle as clogicle

def quantile(x, n):
    """return the lower nth quantile"""
    try:
        return sorted(x)[int(n * len(x))]
    except IndexError:
        return 0

def product_log(x):
    """
    Product logarithm or LambertW function computes principal solution
    for w in f(w) = w*exp(w).
    """
    #  fast estimate with closed-form approximation
    if x <= 500:
        lxl = numpy.log(x + 1.0)
        return 0.665 * (1 + 0.0195 * lxl) * lxl + 0.04
    else:
        return numpy.log(x - 4.0) - (1.0 - 1.0 / numpy.log(x)) * numpy.log(numpy.log(x))

def s(x, y, t, m, w):
    p = w / (2 * product_log(0.5 * numpy.exp(-w / 2) * w))
    sgn = numpy.sign(x - w)
    xw = sgn * (x - w)
    return sgn * t * numpy.exp(-(m - w)) * (numpy.exp(xw) - p ** 2 * numpy.exp(-xw / p) + p ** 2 - 1) - y

def _logicle(y, t=262144, m=4.5, r=None, w=0.5, a=0):
    y = numpy.array(y, dtype='double')
    if w is None:  # we need an r then...
        if r == 0:
            w = 1  # don't like this but it works... FIX!
        else:
            w = (m - numpy.log10(t / numpy.abs(r))) / 2.0

    clogicle.logicle_scale(t, w, m, a, y)
    return y

def logicle(
        data,
        channels,
        t,
        m=4.5,
        r=None,
        scale_max=1e5,
        w=0.5,
        a=0,
        r_quant=None):
    """
    return logicle transformed points for channels listed
    """
    data_copy = data.copy()
    for i in channels:
        if r_quant:
            w = None
            tmp = data_copy[:, i]
            r = quantile(tmp[tmp < 0], 0.05)
        if r is None and w is None:
            w = 0.5
        tmp = scale_max * _logicle(data_copy[:, i].T, t, m, r, w, a)
        data_copy.T[i] = tmp
    return data_copy

def eh(x, y, b, d, r):
    e = float(d) / r
    sgn = numpy.sign(x)
    return sgn * 10 ** (sgn * e * x) + b * e * x - sgn - y

def hyperlog0(y, b, d, r):
    return brentq(eh, -10 ** 6, 10 ** 6, (y, b, d, r))

hyperlog0 = numpy.vectorize(hyperlog0)

def _hyperlog(y, b, d, r, order=2, intervals=1000.0):
    ub = numpy.log(numpy.max(y) + 1 - numpy.min(y))
    xx = numpy.exp(numpy.arange(0, ub[0], ub / intervals)) - 1 + numpy.min(y)
    yy = hyperlog0(xx, b, d, r)
    t = interpolate.splrep(xx, yy, k=order)
    return interpolate.splev(y, t)

def hyperlog(npy, channels, b, d, r):
    npnts = npy.copy()
    for i in channels:
        npnts.T[i] = _hyperlog(
            npnts[:, i].T,
            b,
            d,
            r,
            order=2,
            intervals=1000.0)
    return npy

def log_transform(npy, channels):
    npnts = npy.copy()
    for i in channels:
        npnts[:, i] = _log_transform(npnts[:, i])
    return npnts

def _log_transform(npnts):
    return numpy.where(npnts <= 1, 0, numpy.log10(npnts))<commit_msg>don't scale logicle transform<commit_after>"""
Various transforms for FCS data
"""

from scipy.optimize import brentq
from scipy import interpolate
import numpy

import logicle as clogicle

def quantile(x, n):
    """return the lower nth quantile"""
    try:
        return sorted(x)[int(n * len(x))]
    except IndexError:
        return 0

def product_log(x):
    """
    Product logarithm or LambertW function computes principal solution
    for w in f(w) = w*exp(w).
    """
    #  fast estimate with closed-form approximation
    if x <= 500:
        lxl = numpy.log(x + 1.0)
        return 0.665 * (1 + 0.0195 * lxl) * lxl + 0.04
    else:
        return numpy.log(x - 4.0) - (1.0 - 1.0 / numpy.log(x)) * numpy.log(numpy.log(x))

def s(x, y, t, m, w):
    p = w / (2 * product_log(0.5 * numpy.exp(-w / 2) * w))
    sgn = numpy.sign(x - w)
    xw = sgn * (x - w)
    return sgn * t * numpy.exp(-(m - w)) * (numpy.exp(xw) - p ** 2 * numpy.exp(-xw / p) + p ** 2 - 1) - y

def _logicle(y, t=262144, m=4.5, r=None, w=0.5, a=0):
    y = numpy.array(y, dtype='double')
    if w is None:  # we need an r then...
        if r == 0:
            w = 1  # don't like this but it works... FIX!
        else:
            w = (m - numpy.log10(t / numpy.abs(r))) / 2.0

    clogicle.logicle_scale(t, w, m, a, y)
    return y

def logicle(
        data,
        channels,
        t,
        m=4.5,
        r=None,
        w=0.5,
        a=0,
        r_quant=None):
    """
    return logicle transformed points for channels listed
    """
    data_copy = data.copy()
    for i in channels:
        if r_quant:
            w = None
            tmp = data_copy[:, i]
            r = quantile(tmp[tmp < 0], 0.05)
        if r is None and w is None:
            w = 0.5
        tmp = _logicle(data_copy[:, i].T, t, m, r, w, a)
        data_copy.T[i] = tmp
    return data_copy

def eh(x, y, b, d, r):
    e = float(d) / r
    sgn = numpy.sign(x)
    return sgn * 10 ** (sgn * e * x) + b * e * x - sgn - y

def hyperlog0(y, b, d, r):
    return brentq(eh, -10 ** 6, 10 ** 6, (y, b, d, r))

hyperlog0 = numpy.vectorize(hyperlog0)

def _hyperlog(y, b, d, r, order=2, intervals=1000.0):
    ub = numpy.log(numpy.max(y) + 1 - numpy.min(y))
    xx = numpy.exp(numpy.arange(0, ub[0], ub / intervals)) - 1 + numpy.min(y)
    yy = hyperlog0(xx, b, d, r)
    t = interpolate.splrep(xx, yy, k=order)
    return interpolate.splev(y, t)

def hyperlog(npy, channels, b, d, r):
    npnts = npy.copy()
    for i in channels:
        npnts.T[i] = _hyperlog(
            npnts[:, i].T,
            b,
            d,
            r,
            order=2,
            intervals=1000.0)
    return npy

def log_transform(npy, channels):
    npnts = npy.copy()
    for i in channels:
        npnts[:, i] = _log_transform(npnts[:, i])
    return npnts

def _log_transform(npnts):
    return numpy.where(npnts <= 1, 0, numpy.log10(npnts))<|endoftext|>


<commit_before>#! python

"""Python script to list all installed software."""

from optparse import OptionParser
import pprint
import re
import subprocess

import win32con

from installed_apps import InstalledApps
from installed_apps import NAME_KEY
from installed_apps import UNINSTALL_KEY
from installed_apps import VERSION_KEY

class InstalledApp(object):
    """Models an installed application."""

    def __init__(self, name, version=None):
        """Initialize an instance with name and version."""
        self.name = name
        self.version = version

    def __lt__(self, other):
        """Am I less than other?"""
        if self.name < other.name:
            return True
        elif self.name > other.name:
            return False
        else:
            result = (self.version < other.version)

    def __str__(self):
        """Represent myself as a string."""
        return str((self.name, self.version))

def list_apps(ia, options):
    about_installed = ia.find(filter=options.text)
    installed = [InstalledApp(i['DisplayName'], i['DisplayVersion']) for
                 i in about_installed]
    installed.sort()
    format_str = '{0:50.50}{1:25.25}'
    print(format_str.format('Name', 'Version'))
    print(format_str.format('=' * 48, '=' * 25))
    for i in installed:
        print(format_str.format(i.name, i.version))

def uninstall_apps(ia, options):
    to_uninstall = ia.find(filter=options.text,
                           what_info=[VERSION_KEY, UNINSTALL_KEY])
    for uninstall_info in to_uninstall:
        name = uninstall_info[NAME_KEY]
        version = uninstall_info[VERSION_KEY] if \
                  VERSION_KEY in uninstall_info else "N/A"
        uninstall_cmd = uninstall_info[UNINSTALL_KEY]
        prompt = 'Uninstall {0} (version {1}) (y/n)? '.format(name, version)
        answer = raw_input(prompt)
        if answer[0] in 'yY':
            subprocess.check_call(uninstall_cmd)

actions = { 'list' : list_apps,
            'uninstall' : uninstall_apps, }

if __name__ == '__main__':
    usage = """%prog [options] [command...]

    Manage installed applications. If no command is supplied, the
    script lists the installed applications.

    Valid commands are:
    list\t- Print the installed applications.
    uninstall\t- Uninstall the specified application(s).
    """
    parser = OptionParser(usage=usage)

    parser.add_option('-t', '--text',
                      default='.*',
                      help='Select applications of interest (default=all).')
    (options, args) = parser.parse_args()

    if len(args) == 0:
        commands = ['list']
    else:
        commands = args

    ia = InstalledApps()
    for command in commands:
        actions[command](ia, options)
<commit_msg>- Repaired defect resulting from an installed application not having a   'DisplayedVersion' member.<commit_after>#! python

"""Python script to list all installed software."""

from optparse import OptionParser
import pprint
import re
import subprocess

import win32con

from installed_apps import InstalledApps
from installed_apps import NAME_KEY
from installed_apps import UNINSTALL_KEY
from installed_apps import VERSION_KEY

class InstalledApp(object):
    """Models an installed application."""

    def __init__(self, name, version=None):
        """Initialize an instance with name and version."""
        self.name = name
        self.version = version

    def __lt__(self, other):
        """Am I less than other?"""
        if self.name < other.name:
            return True
        elif self.name > other.name:
            return False
        else:
            result = (self.version < other.version)

    def __str__(self):
        """Represent myself as a string."""
        return str((self.name, self.version))

def list_apps(ia, options):
    about_installed = ia.find(filter=options.text)
    installed = [InstalledApp(i['DisplayName'],
                              i.get('DisplayVersion')) for
                 i in about_installed]
    installed.sort()
    format_str = '{0:50.50}{1:25.25}'
    print(format_str.format('Name', 'Version'))
    print(format_str.format('=' * 48, '=' * 25))
    for i in installed:
        print(format_str.format(i.name, i.version))

def uninstall_apps(ia, options):
    to_uninstall = ia.find(filter=options.text,
                           what_info=[VERSION_KEY, UNINSTALL_KEY])
    for uninstall_info in to_uninstall:
        name = uninstall_info[NAME_KEY]
        version = uninstall_info[VERSION_KEY] if \
                  VERSION_KEY in uninstall_info else "N/A"
        uninstall_cmd = uninstall_info[UNINSTALL_KEY]
        prompt = 'Uninstall {0} (version {1}) (y/n)? '.format(name, version)
        answer = raw_input(prompt)
        if answer[0] in 'yY':
            subprocess.check_call(uninstall_cmd)

actions = { 'list' : list_apps,
            'uninstall' : uninstall_apps, }

if __name__ == '__main__':
    usage = """%prog [options] [command...]

    Manage installed applications. If no command is supplied, the
    script lists the installed applications.

    Valid commands are:
    list\t- Print the installed applications.
    uninstall\t- Uninstall the specified application(s).
    """
    parser = OptionParser(usage=usage)

    parser.add_option('-t', '--text',
                      default='.*',
                      help='Select applications of interest (default=all).')
    (options, args) = parser.parse_args()

    if len(args) == 0:
        commands = ['list']
    else:
        commands = args

    ia = InstalledApps()
    for command in commands:
        actions[command](ia, options)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2009 Samson Yeung, Phil Sung
#
# This file is part of Zeya.
#
# Zeya is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Zeya is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Zeya. If not, see <http://www.gnu.org/licenses/>.

# Directory backend. (Experimental)
#
# Files in the specified directory are read for artist/title/album tag which is
# then saved in a database. The
#
# This is very much work in progress.
#
# Fundamental questions regarding database handling and how to store db info
# have yet to be handled. Also, how to handle new files if the database is
# being reloaded is completely unaddressed.

import os
import tagpy
import pickle

from backend import LibraryBackend

KEY = 'key'
TITLE = 'title'
ARTIST = 'artist'
ALBUM = 'album'

DB = 'db'
KEY_FILENAME = 'key_filename'
MTIMES = 'mtimes'

class DirectoryBackend(LibraryBackend):
    """
    Object that controls access to music in a given directory.
    """
    def __init__(self, media_path, save_db=True):
        self._media_path = media_path
        self._save_db = save_db
        self.db = []
        self.key_filename = {}
        self.mtimes = {}

        self.setup_db()

    def get_db_filename(self):
        return os.path.join(self._media_path, 'zeya.db')

    def setup_db(self):
        if os.path.exists(self.get_db_filename()):
            self.read_db_from_disk()
        self.fill_db()
        if self._save_db:
            self.save_db()

    def read_db_from_disk(self):
        self.info = pickle.load(open(self.get_db_filename(), 'r'))
        self.db = self.info[DB]
        self.key_filename = self.info[KEY_FILENAME]
        self.mtimes = self.info[MTIMES]

        #for path, dirs, files in os.walk(self._media_path):

    def save_db(self):
        self.info = {DB: self.db,
                     MTIMES: self.mtimes,
                     KEY_FILENAME: self.key_filename}
        pickle.dump(self.info, open(self.get_db_filename(), 'wb+'))

    def fill_db(self):
        # Walk
        i = len(self.key_filename) # Use simple int as key (sub-optimal, but
        # better than sending entire path across wire for small db's)
        for path, dirs, files in os.walk(self._media_path):
            for filename in [os.path.abspath(os.path.join(path, filename)) for
                         filename in files]:
                file_mtime = os.stat(filename).st_mtime
                rec_mtime = self.mtimes.get(filename, 0)

                if rec_mtime <= file_mtime:
                    try:
                        tag = tagpy.FileRef(filename).tag()
                    except:
                        # If there was any exception, then ignore the file and
                        # continue. Catching ValueError is sufficient to catch
                        # non-audio but we want to not abort from this.
                        continue

                    data = {KEY: i,
                            ARTIST: tag.artist,
                            TITLE: tag.title,
                            ALBUM: tag.album,
                           }
                    self.db.append(data)
                    self.key_filename[i] = filename
                    self.mtimes[filename] = file_mtime
                    i += 1

    def get_library_contents(self):
        return self.db

    def get_filename_from_key(self, key):
        return self.key_filename[int(key)]

<commit_msg>Refactor to remove loop-carried dependency in directory.py.<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2009 Samson Yeung, Phil Sung
#
# This file is part of Zeya.
#
# Zeya is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Zeya is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Zeya. If not, see <http://www.gnu.org/licenses/>.

# Directory backend. (Experimental)
#
# Files in the specified directory are read for artist/title/album tag which is
# then saved in a database. The
#
# This is very much work in progress.
#
# Fundamental questions regarding database handling and how to store db info
# have yet to be handled. Also, how to handle new files if the database is
# being reloaded is completely unaddressed.

import os
import tagpy
import pickle

from backend import LibraryBackend

KEY = 'key'
TITLE = 'title'
ARTIST = 'artist'
ALBUM = 'album'

DB = 'db'
KEY_FILENAME = 'key_filename'
MTIMES = 'mtimes'

class DirectoryBackend(LibraryBackend):
    """
    Object that controls access to music in a given directory.
    """
    def __init__(self, media_path, save_db=True):
        self._media_path = media_path
        self._save_db = save_db
        self.db = []
        self.key_filename = {}
        self.mtimes = {}

        self.setup_db()

    def get_db_filename(self):
        return os.path.join(self._media_path, 'zeya.db')

    def setup_db(self):
        if os.path.exists(self.get_db_filename()):
            self.read_db_from_disk()
        self.fill_db()
        if self._save_db:
            self.save_db()

    def read_db_from_disk(self):
        self.info = pickle.load(open(self.get_db_filename(), 'r'))
        self.db = self.info[DB]
        self.key_filename = self.info[KEY_FILENAME]
        self.mtimes = self.info[MTIMES]

        #for path, dirs, files in os.walk(self._media_path):

    def save_db(self):
        self.info = {DB: self.db,
                     MTIMES: self.mtimes,
                     KEY_FILENAME: self.key_filename}
        pickle.dump(self.info, open(self.get_db_filename(), 'wb+'))

    def fill_db(self):
        for path, dirs, files in os.walk(self._media_path):
            for filename in [os.path.abspath(os.path.join(path, filename)) for
                         filename in files]:
                file_mtime = os.stat(filename).st_mtime
                rec_mtime = self.mtimes.get(filename, 0)

                if rec_mtime <= file_mtime:
                    try:
                        tag = tagpy.FileRef(filename).tag()
                    except:
                        # If there was any exception, then ignore the file and
                        # continue. Catching ValueError is sufficient to catch
                        # non-audio but we want to not abort from this.
                        continue

                    # Use simple int as key (sub-optimal, but better than
                    # sending entire path across wire for small db's)
                    next_key = len(self.key_filename)
                    data = {KEY: next_key,
                            ARTIST: tag.artist,
                            TITLE: tag.title,
                            ALBUM: tag.album,
                           }
                    self.db.append(data)
                    self.key_filename[next_key] = filename
                    self.mtimes[filename] = file_mtime

    def get_library_contents(self):
        return self.db

    def get_filename_from_key(self, key):
        return self.key_filename[int(key)]

<|endoftext|>


<commit_before>import os
import pickle

import threading
from math import sqrt

from gi.repository import GLib, GObject, Gtk

import config
from display.configwindow import GtkFontLoadingWindow
import font

keys = list()
fonts = dict()

total_files = 0
loaded_files = 0
current_file_name = ""
total_cache = 0
loaded_cache = 0

def load_cache():
    # TODO if you have a lot of fonts this might kill memory, we should load and
    # del fonts as necessary.
    exceptions = set()
    try:
        exceptions = pickle.load(open("{}/exceptions.tcat".format(config.CACHE_LOCATION), "rb"))
        print("Loaded exceptions list!")
    except FileNotFoundError:
        print("Exception file not found, initializing blank one")
        exceptions = set()
    global total_cache, loaded_cache
    cachedfonts = os.listdir(config.CACHE_LOCATION)
    total_cache = len(cachedfonts)
    for f in cachedfonts:
        if f[-7:] != ".pickle" or f[:-7] in exceptions:
            print("File {} is in exception list".format(f))
            continue
        fontname = f[:-7]
        try:
            loadfont = pickle.load(open("{}/{}".format(
                config.CACHE_LOCATION, f), "rb"))
            fonts[fontname] = loadfont
            print("Loaded {} from cache".format(fonts[fontname].name))
        except font.RenderError:
            print("Skipping {}, unable to render correctly, adding to exceptions".format(fontname))
            exceptions.add(fontname)
        loaded_cache += 1
    global keys
    keys = list(fonts.keys())
    pickle.dump(exceptions, open("{}/exceptions.tcat".format(config.CACHE_LOCATION), "wb"))
    print("Dumping to path {}/exceptions.tcat".format(config.CACHE_LOCATION))

def load_files():
    def run():
        t = threading.currentThread()

        fontpaths = []
        global total_files, loaded_files, current_file_name, exceptions
        try:
            exceptions = pickle.load(open("{}/exceptions.tcat".format(config.CACHE_LOCATION), "rb"))
        except FileNotFoundError:
            print("Exception file not found, initializing blank one")
        for fontdir in config.FONT_DIRS:
            for dirpath, dirnames, filenames in os.walk(fontdir):
                for d in filenames:
                    idx = d.rfind(".")
                    if idx != -1 and d[idx:] in config.FONT_FILE_EXTENSIONS:
                        fontpaths.append(os.path.join(dirpath, d))
        total_files = len(fontpaths)

        for f in fontpaths:
            try:
                fontname = font.Font.extract_name(f)
                current_file_name = fontname
                if fontname in exceptions or f in exceptions:
                    print("Skipping font {}, in exception list".format(fontname))
                    continue
                g = font.Font(f)
                fonts[fontname] = g
                g.save()
                print("Loaded {} from file".format(
                    g.name))
            except Exception as e:
                print(("Failed to read font at path {}"
                       "with exception {}").format(
                    f, e))
                exceptions.add(f)
                pickle.dump(exceptions, open("{}/exceptions.tcat".format(config.CACHE_LOCATION), "wb"))

            loaded_files += 1
            GLib.idle_add(win.update_bar, [float(loaded_files / total_files), current_file_name])

            if getattr(t, "stop_flag", True):
                break
            if loaded_files == total_files:
                GLib.idle_add(win.cancel, None)

    thread = threading.Thread(target=run)
    setattr(thread, "stop_flag", False)
    thread.daemon = True
    win = GtkFontLoadingWindow(thread)
    win.connect("delete-event", win.exit_handler)
    win.show_all()
    GObject.threads_init()
    thread.start()
    Gtk.main()
    thread.join()

    global keys
    keys = list(fonts.keys())

def load_fonts():
    load_cache()
    load_files()<commit_msg>Fixed some first-time startup bugs<commit_after>import os
import pickle

import threading
from math import sqrt

from gi.repository import GLib, GObject, Gtk

import config
from display.configwindow import GtkFontLoadingWindow
import font

keys = list()
fonts = dict()

total_files = 0
loaded_files = 0
current_file_name = ""
total_cache = 0
loaded_cache = 0

def load_cache():
    # TODO if you have a lot of fonts this might kill memory, we should load and
    # del fonts as necessary.
    exceptions = set()
    try:
        exceptions = pickle.load(open("{}/exceptions.tcat".format(config.CACHE_LOCATION), "rb"))
        print("Loaded exceptions list!")
    except FileNotFoundError:
        print("Exception file not found, initializing blank one")
        exceptions = set()
    global total_cache, loaded_cache
    cachedfonts = os.listdir(config.CACHE_LOCATION)
    total_cache = len(cachedfonts)
    for f in cachedfonts:
        if f[-7:] != ".pickle" or f[:-7] in exceptions:
            print("File {} is in exception list".format(f))
            continue
        fontname = f[:-7]
        try:
            loadfont = pickle.load(open("{}/{}".format(
                config.CACHE_LOCATION, f), "rb"))
            fonts[fontname] = loadfont
            print("Loaded {} from cache".format(fonts[fontname].name))
        except font.RenderError:
            print("Skipping {}, unable to render correctly, adding to exceptions".format(fontname))
            exceptions.add(fontname)
        loaded_cache += 1
    global keys
    keys = list(fonts.keys())
    pickle.dump(exceptions, open("{}/exceptions.tcat".format(config.CACHE_LOCATION), "wb"))
    print("Dumping to path {}/exceptions.tcat".format(config.CACHE_LOCATION))

def load_files():
    def run():
        t = threading.currentThread()

        fontpaths = []
        global total_files, loaded_files, current_file_name, exceptions
        try:
            exceptions = pickle.load(open("{}/exceptions.tcat".format(config.CACHE_LOCATION), "rb"))
        except FileNotFoundError:
            exceptions = set()
            print("Exception file not found, initializing blank one")
        for fontdir in config.FONT_DIRS:
            for dirpath, dirnames, filenames in os.walk(fontdir):
                for d in filenames:
                    idx = d.rfind(".")
                    if idx != -1 and d[idx:] in config.FONT_FILE_EXTENSIONS:
                        fontpaths.append(os.path.join(dirpath, d))
        total_files = len(fontpaths)

        for f in fontpaths:
            try:
                fontname = font.Font.extract_name(f)
                current_file_name = fontname
                if fontname in exceptions or f in exceptions:
                    print("Skipping font {}, in exception list".format(fontname))
                    continue
                g = font.Font(f)
                fonts[fontname] = g
                g.save()
                print("Loaded {} from file".format(
                    g.name))
            except Exception as e:
                print(("Failed to read font at path {}"
                       "with exception {}").format(
                    f, e))
                exceptions.add(f)
                pickle.dump(exceptions, open("{}/exceptions.tcat".format(config.CACHE_LOCATION), "wb"))

            loaded_files += 1
            GLib.idle_add(win.update_bar, [float(loaded_files / total_files), current_file_name])

            if getattr(t, "stop_flag", True):
                break
            if loaded_files == total_files:
                GLib.idle_add(win.cancel, None)

    thread = threading.Thread(target=run)
    setattr(thread, "stop_flag", False)
    thread.daemon = True
    win = GtkFontLoadingWindow(thread)
    win.connect("delete-event", win.exit_handler)
    win.show_all()
    GObject.threads_init()
    thread.start()
    Gtk.main()
    thread.join()

    global keys
    keys = list(fonts.keys())

def load_fonts():
    load_cache()
    load_files()
<|endoftext|>


<commit_before>from couchbase_core._ixmgmt import N1QL_PRIMARY_INDEX, IxmgmtRequest, N1qlIndex
from couchbase_core.bucketmanager import BucketManager
from couchbase.options import OptionBlock, OptionBlockTimeOut, forward_args, timedelta
from typing import *
from couchbase.management.generic import GenericManager
import attr
from attr.validators import instance_of as io, deep_mapping as dm
from couchbase_core._pyport import Protocol
from couchbase.exceptions import HTTPException, ErrorMapper, AnyPattern, QueryIndexAlreadyExistsException, \
    QueryIndexNotFoundException, DocumentNotFoundException, DocumentExistsException

class QueryErrorMapper(ErrorMapper):
    @staticmethod
    def mapping():
        # type: (...) -> Dict[CBErrorType,Dict[Any, CBErrorType]]
        return {DocumentNotFoundException: {AnyPattern(): QueryIndexNotFoundException},
                DocumentExistsException: {AnyPattern(): QueryIndexAlreadyExistsException}}

@QueryErrorMapper.wrap
class QueryIndexManager(GenericManager):
    def __init__(self, parent_cluster):
        """
        Query Index Manager
        The Query Index Manager interface contains the means for managing indexes used for queries.
        :param parent_cluster: Parent cluster
        """
        super(QueryIndexManager,self).__init__(parent_cluster)

    def get_all_indexes(self,           # type: QueryIndexManager
                        bucket_name,    # type: str
                        *options,       # type: GetAllQueryIndexOptions
                        **kwargs
                        ):
        # type: (...) -> List[QueryIndex]
        """
        Fetches all indexes from the server.

        :param str bucket_name: the name of the bucket.
        :param GetAllQueryIndexOptions options: Options to use for getting all indexes.
        :param Any kwargs: Override corresponding value in options.
        :return: A list of QueryIndex objects.
        :raises: InvalidArgumentsException
        """
        # N1QL
        # SELECT idx.* FROM system:indexes AS idx
        # WHERE keyspace_id = "bucket_name"
        # ORDER BY is_primary DESC, name ASC
        info = N1qlIndex()
        info.keyspace = bucket_name
        response = IxmgmtRequest(self._admin_bucket, 'list', info, **forward_args(kwargs, *options)).execute()
        return list(map(QueryIndex.from_n1qlindex, response))

    def _mk_index_def(self, bucket_name, ix, primary=False):
        if isinstance(ix, N1qlIndex):
            return N1qlIndex(ix)

        info = N1qlIndex()
        info.keyspace = bucket_name
        info.primary = primary

        if ix:
            info.name = ix
        elif not primary:
            raise ValueError('Missing name for non-primary index')

        return info

    def _n1ql_index_create(self, bucket_name, ix, defer=False, ignore_exists=False, primary=False, fields=None, cond = None, timeout=None, **kwargs):
        """
        Create an index for use with N1QL.

        :param str ix: The name of the index to create
        :param bool defer: Whether the building of indexes should be
            deferred. If creating multiple indexes on an existing
            dataset, using the `defer` option in conjunction with
            :meth:`build_deferred_indexes` and :meth:`watch_indexes` may
            result in substantially reduced build times.
        :param bool ignore_exists: Do not throw an exception if the index
            already exists.
        :param Iterable[str] fields: A list of fields that should be supplied
            as keys for the index. For non-primary indexes, this must
            be specified and must contain at least one field name.
        :param bool primary: Whether this is a primary index. If creating
            a primary index, the name may be an empty string and `fields`
            must be empty.
        :param str condition: Specify a condition for indexing. Using
            a condition reduces an index size
        :raise: :exc:`~.DocumentExistsException` if the index already exists

        .. seealso:: :meth:`n1ql_index_create_primary`
        """
        fields = fields or []

        if kwargs:
            raise TypeError('Unknown keyword arguments', kwargs)

        info = self._mk_index_def(bucket_name, ix, primary)

        if primary and fields:
            raise TypeError('Cannot create primary index with explicit fields')
        elif not primary and not fields:
            raise ValueError('Fields required for non-primary index')

        if fields:
            info.fields = fields

        if primary and info.name is N1QL_PRIMARY_INDEX:
            del info.name

        if cond:
            if primary:
                raise ValueError('cannot specify condition for primary index')
            info.condition = cond

        options = {
            'ignore_exists': ignore_exists,
            'defer': defer
        }

        if timeout:
            options['timeout']=timeout
        # Now actually create the indexes
        return IxmgmtRequest(self._admin_bucket, 'create', info, **options).execute()

    def create_index(self,          # type: QueryIndexManager
                     bucket_name,   # type: str
                     index_name,    # type: str
                     fields,        # type: Iterable[str]
                     *options,      # type: CreateQueryIndexOptions
                     **kwargs
                     ):
        # type: (...) -> None
        """
        Creates a new index.

        :param str bucket_name: name of the bucket.
        :param str index_name: the name of the index.
        :param Iterable[str] fields: Fields over which to create the index.
        :param CreateQueryIndexOptions options: Options to use when creating index.
        :param Any kwargs: Override corresponding value in options.
        :raises: QueryIndexAlreadyExistsException
        :raises: InvalidArgumentsException
        """
        # CREATE INDEX index_name ON bucket_name WITH { "num_replica": 2 }
        #         https://docs.couchbase.com/server/current/n1ql/n1ql-language-reference/createindex.html
        #
        self._create_index(bucket_name, fields, index_name, *options, **kwargs)

    def _create_index(self, bucket_name, fields, index_name, *options, **kwargs):
        final_args = {
            k.replace('deferred', 'defer').replace('condition', 'cond').replace('ignore_if_exists', 'ignore_exists'): v
            for k, v in forward_args(kwargs, *options).items()}
        try:
            self._n1ql_index_create(bucket_name, index_name, fields=fields, **final_args)
        except QueryIndexAlreadyExistsException:
            if not final_args.get('ignore_exists', False):
                raise

    def create_primary_index(self,  # type: QueryIndexManager
                             bucket_name,  # type: str
                             *options,  # type: CreatePrimaryQueryIndexOptions
                             **kwargs
                             ):
        """
        Creates a new primary index.

        :param str bucket_name:  name of the bucket.
        :param str index_name: name of the index.
        :param CreatePrimaryQueryIndexOptions options: Options to use when creating primary index
        :param Any kwargs: Override corresponding values in options.
        :raises: QueryIndexAlreadyExistsException
        :raises: InvalidArgumentsException
        """
        # CREATE INDEX index_name ON bucket_name WITH { "num_replica": 2 }
        #         https://docs.couchbase.com/server/current/n1ql/n1ql-language-reference/createindex.html
        #
        kwargs['primary'] = True
        index_name = ""
        if options and options[0] :
            index_name = options[0].pop("index_name", "")
        fields = []
        self._create_index(bucket_name, fields, index_name, *options, **kwargs)

    def _drop_index(self, bucket_name, index_name, *options, **kwargs):
        info = BucketManager._mk_index_def(bucket_name, index_name, primary=kwargs.pop('primary',False))
        final_args = {k.replace('ignore_if_not_exists','ignore_missing'):v for k,v in  forward_args(kwargs, *options).items()}
        try:
            IxmgmtRequest(self._admin_bucket, 'drop', info, **final_args).execute()
        except QueryIndexNotFoundException:
            if not final_args.get("ignore_missing", False):
                raise

    def drop_index(self,            # type: QueryIndexManager
                   bucket_name,     # type: str
                   index_name,      # type: str
                   *options,        # type: DropQueryIndexOptions
                   **kwargs):
        """
        Drops an index.

        :param str bucket_name: name of the bucket.
        :param str index_name: name of the index.
        :param DropQueryIndexOptions options: Options for dropping index.
        :param Any kwargs: Override corresponding value in options.
        :raises: QueryIndexNotFoundException
        :raises: InvalidArgumentsException
        """
        final_args = forward_args(kwargs, *options)
        self._drop_index(bucket_name, index_name, **final_args)

    def drop_primary_index(self,            # type: QueryIndexManager
                           bucket_name,     # type: str
                           *options,        # type: DropPrimaryQueryIndexOptions
                           **kwargs):
        """
        Drops a primary index.

        :param bucket_name: name of the bucket.
        :param index_name:  name of the index.
        :param ignore_if_not_exists: Don't error/throw if the index does not exist.
        :param timeout:  the time allowed for the operation to be terminated. This is controlled by the client.

        :raises: QueryIndexNotFoundException
        :raises: InvalidArgumentsException
        """
        final_args=forward_args(kwargs, *options)
        final_args['primary'] = True
        index_name = final_args.pop("index_name", "")
        self._drop_index(bucket_name, index_name, **final_args)

    def watch_indexes(self,         # type: QueryIndexManager
                      bucket_name,  # type: str
                      index_names,  # type: Iterable[str]
                      *options,     # type: WatchQueryIndexOptions
                      **kwargs):
        """
        Watch polls indexes until they are online.

        :param str bucket_name: name of the bucket.
        :param Iterable[str] index_names: name(s) of the index(es).
        :param WatchQueryIndexOptions options: Options for request to watch indexes.
        :param Any kwargs: Override corresponding valud in options.
        :raises: QueryIndexNotFoundException
        :raises: InvalidArgumentsException
        """
        final_args=forward_args(kwargs, *options)
        BucketManager(self._admin_bucket).n1ql_index_watch(index_names, **final_args)

    def build_deferred_indexes(self,            # type: QueryIndexManager
                               bucket_name,     # type: str
                               *options,        # type: BuildDeferredQueryIndexOptions
                               **kwargs
                               ):
        """
        Build Deferred builds all indexes which are currently in deferred state.

        :param str bucket_name: name of the bucket.
        :param BuildDeferredQueryIndexOptions options: Options for building deferred indexes.
        :param Any kwargs: Override corresponding value in options.
        :raise: InvalidArgumentsException

        """
        final_args=forward_args(kwargs, *options)
        return BucketManager._n1ql_index_build_deferred(bucket_name, self._admin_bucket, **final_args)

class IndexType(object):
    pass

@attr.s
class QueryIndex(Protocol):
    """The QueryIndex protocol provides a means of mapping a query index into an object."""

    name = attr.ib(validator=io(str))  # type: str
    is_primary = attr.ib(validator=io(bool))  # type: bool
    type = attr.ib(validator=io(IndexType), type=IndexType)  # type: IndexType
    state = attr.ib(validator=io(str))  # type: str
    keyspace = attr.ib(validator=io(str))  # type: str
    index_key = attr.ib(validator=io(Iterable))  # type: Iterable[str]
    condition = attr.ib(validator=io(str))  # type: str
    @classmethod
    def from_n1qlindex(cls,
                       n1qlindex  # type: N1qlIndex
                       ):
        return cls(n1qlindex.name, bool(n1qlindex.primary), IndexType(), n1qlindex.state, n1qlindex.keyspace, [], n1qlindex.condition or "")

class GetAllQueryIndexOptions(OptionBlockTimeOut):
    pass

class CreateQueryIndexOptions(OptionBlockTimeOut):
    @overload
    def __init__(self,
                 timeout=None,           # type: timedelta
                 ignore_if_exists=None,  # type: bool
                 num_replicas=None,      # type: int
                 deferred=None,          # type: bool
                 ):
        pass

    def __init__(self, **kwargs):
        if 'ignore_if_exists' not in kwargs:
            kwargs['ignore_if_exists'] = False
        super(CreateQueryIndexOptions, self).__init__()

class CreatePrimaryQueryIndexOptions(CreateQueryIndexOptions):
    @overload
    def __init__(self,
                 index_name=None,        # type: str
                 timeout=None,           # type: timedelta
                 ignore_if_exists=None,  # type: bool
                 num_replicas=None,      # type: int
                 deferred=None,          # type: bool
                 ):
        pass

    def __init__(self, **kwargs):
        super(CreatePrimaryQueryIndexOptions, self).__init__()

class DropQueryIndexOptions(OptionBlockTimeOut):
    @overload
    def __init__(self,
                 ignore_if_not_exists=None,  # type: bool
                 timeout=None                # type: timedelta
                 ):
        pass

    def __init__(self, **kwargs):
        super(DropQueryIndexOptions, self).__init__(**kwargs)

class DropPrimaryQueryIndexOptions(OptionBlockTimeOut):
    @overload
    def __init__(self,
                 index_name=None,            # str
                 ignore_if_not_exists=None,  # type: bool
                 timeout=None                # type: timedelta
                 ):
        pass

    def __init__(self, **kwargs):
        super(DropPrimaryQueryIndexOptions, self).__init__(**kwargs)

class WatchQueryIndexOptions(OptionBlock):
    @overload
    def __init__(self,
                 watch_primary=None  # type: bool
                 ):
        pass

    def __init__(self, **kwargs):
        super(WatchQueryIndexOptions, self).__init__(**kwargs)

class BuildDeferredQueryIndexOptions(OptionBlockTimeOut):
    pass
<commit_msg>PYCBC-934: Some Query OptionBlocks not passing kwargs to constructor<commit_after>from couchbase_core._ixmgmt import N1QL_PRIMARY_INDEX, IxmgmtRequest, N1qlIndex
from couchbase_core.bucketmanager import BucketManager
from couchbase.options import OptionBlock, OptionBlockTimeOut, forward_args, timedelta
from typing import *
from couchbase.management.generic import GenericManager
import attr
from attr.validators import instance_of as io, deep_mapping as dm
from couchbase_core._pyport import Protocol
from couchbase.exceptions import HTTPException, ErrorMapper, AnyPattern, QueryIndexAlreadyExistsException, \
    QueryIndexNotFoundException, DocumentNotFoundException, DocumentExistsException

class QueryErrorMapper(ErrorMapper):
    @staticmethod
    def mapping():
        # type: (...) -> Dict[CBErrorType,Dict[Any, CBErrorType]]
        return {DocumentNotFoundException: {AnyPattern(): QueryIndexNotFoundException},
                DocumentExistsException: {AnyPattern(): QueryIndexAlreadyExistsException}}

@QueryErrorMapper.wrap
class QueryIndexManager(GenericManager):
    def __init__(self, parent_cluster):
        """
        Query Index Manager
        The Query Index Manager interface contains the means for managing indexes used for queries.
        :param parent_cluster: Parent cluster
        """
        super(QueryIndexManager,self).__init__(parent_cluster)

    def get_all_indexes(self,           # type: QueryIndexManager
                        bucket_name,    # type: str
                        *options,       # type: GetAllQueryIndexOptions
                        **kwargs
                        ):
        # type: (...) -> List[QueryIndex]
        """
        Fetches all indexes from the server.

        :param str bucket_name: the name of the bucket.
        :param GetAllQueryIndexOptions options: Options to use for getting all indexes.
        :param Any kwargs: Override corresponding value in options.
        :return: A list of QueryIndex objects.
        :raises: InvalidArgumentsException
        """
        # N1QL
        # SELECT idx.* FROM system:indexes AS idx
        # WHERE keyspace_id = "bucket_name"
        # ORDER BY is_primary DESC, name ASC
        info = N1qlIndex()
        info.keyspace = bucket_name
        response = IxmgmtRequest(self._admin_bucket, 'list', info, **forward_args(kwargs, *options)).execute()
        return list(map(QueryIndex.from_n1qlindex, response))

    def _mk_index_def(self, bucket_name, ix, primary=False):
        if isinstance(ix, N1qlIndex):
            return N1qlIndex(ix)

        info = N1qlIndex()
        info.keyspace = bucket_name
        info.primary = primary

        if ix:
            info.name = ix
        elif not primary:
            raise ValueError('Missing name for non-primary index')

        return info

    def _n1ql_index_create(self, bucket_name, ix, defer=False, ignore_exists=False, primary=False, fields=None, cond = None, timeout=None, **kwargs):
        """
        Create an index for use with N1QL.

        :param str ix: The name of the index to create
        :param bool defer: Whether the building of indexes should be
            deferred. If creating multiple indexes on an existing
            dataset, using the `defer` option in conjunction with
            :meth:`build_deferred_indexes` and :meth:`watch_indexes` may
            result in substantially reduced build times.
        :param bool ignore_exists: Do not throw an exception if the index
            already exists.
        :param Iterable[str] fields: A list of fields that should be supplied
            as keys for the index. For non-primary indexes, this must
            be specified and must contain at least one field name.
        :param bool primary: Whether this is a primary index. If creating
            a primary index, the name may be an empty string and `fields`
            must be empty.
        :param str condition: Specify a condition for indexing. Using
            a condition reduces an index size
        :raise: :exc:`~.DocumentExistsException` if the index already exists

        .. seealso:: :meth:`n1ql_index_create_primary`
        """
        fields = fields or []

        if kwargs:
            raise TypeError('Unknown keyword arguments', kwargs)

        info = self._mk_index_def(bucket_name, ix, primary)

        if primary and fields:
            raise TypeError('Cannot create primary index with explicit fields')
        elif not primary and not fields:
            raise ValueError('Fields required for non-primary index')

        if fields:
            info.fields = fields

        if primary and info.name is N1QL_PRIMARY_INDEX:
            del info.name

        if cond:
            if primary:
                raise ValueError('cannot specify condition for primary index')
            info.condition = cond

        options = {
            'ignore_exists': ignore_exists,
            'defer': defer
        }

        if timeout:
            options['timeout']=timeout
        # Now actually create the indexes
        return IxmgmtRequest(self._admin_bucket, 'create', info, **options).execute()

    def create_index(self,          # type: QueryIndexManager
                     bucket_name,   # type: str
                     index_name,    # type: str
                     fields,        # type: Iterable[str]
                     *options,      # type: CreateQueryIndexOptions
                     **kwargs
                     ):
        # type: (...) -> None
        """
        Creates a new index.

        :param str bucket_name: name of the bucket.
        :param str index_name: the name of the index.
        :param Iterable[str] fields: Fields over which to create the index.
        :param CreateQueryIndexOptions options: Options to use when creating index.
        :param Any kwargs: Override corresponding value in options.
        :raises: QueryIndexAlreadyExistsException
        :raises: InvalidArgumentsException
        """
        # CREATE INDEX index_name ON bucket_name WITH { "num_replica": 2 }
        #         https://docs.couchbase.com/server/current/n1ql/n1ql-language-reference/createindex.html
        #
        self._create_index(bucket_name, fields, index_name, *options, **kwargs)

    def _create_index(self, bucket_name, fields, index_name, *options, **kwargs):
        final_args = {
            k.replace('deferred', 'defer').replace('condition', 'cond').replace('ignore_if_exists', 'ignore_exists'): v
            for k, v in forward_args(kwargs, *options).items()}
        try:
            self._n1ql_index_create(bucket_name, index_name, fields=fields, **final_args)
        except QueryIndexAlreadyExistsException:
            if not final_args.get('ignore_exists', False):
                raise

    def create_primary_index(self,  # type: QueryIndexManager
                             bucket_name,  # type: str
                             *options,  # type: CreatePrimaryQueryIndexOptions
                             **kwargs
                             ):
        """
        Creates a new primary index.

        :param str bucket_name:  name of the bucket.
        :param str index_name: name of the index.
        :param CreatePrimaryQueryIndexOptions options: Options to use when creating primary index
        :param Any kwargs: Override corresponding values in options.
        :raises: QueryIndexAlreadyExistsException
        :raises: InvalidArgumentsException
        """
        # CREATE INDEX index_name ON bucket_name WITH { "num_replica": 2 }
        #         https://docs.couchbase.com/server/current/n1ql/n1ql-language-reference/createindex.html
        #
        kwargs['primary'] = True
        index_name = ""
        if options and options[0] :
            index_name = options[0].pop("index_name", "")
        fields = []
        self._create_index(bucket_name, fields, index_name, *options, **kwargs)

    def _drop_index(self, bucket_name, index_name, *options, **kwargs):
        info = BucketManager._mk_index_def(bucket_name, index_name, primary=kwargs.pop('primary',False))
        final_args = {k.replace('ignore_if_not_exists','ignore_missing'):v for k,v in  forward_args(kwargs, *options).items()}
        try:
            IxmgmtRequest(self._admin_bucket, 'drop', info, **final_args).execute()
        except QueryIndexNotFoundException:
            if not final_args.get("ignore_missing", False):
                raise

    def drop_index(self,            # type: QueryIndexManager
                   bucket_name,     # type: str
                   index_name,      # type: str
                   *options,        # type: DropQueryIndexOptions
                   **kwargs):
        """
        Drops an index.

        :param str bucket_name: name of the bucket.
        :param str index_name: name of the index.
        :param DropQueryIndexOptions options: Options for dropping index.
        :param Any kwargs: Override corresponding value in options.
        :raises: QueryIndexNotFoundException
        :raises: InvalidArgumentsException
        """
        final_args = forward_args(kwargs, *options)
        self._drop_index(bucket_name, index_name, **final_args)

    def drop_primary_index(self,            # type: QueryIndexManager
                           bucket_name,     # type: str
                           *options,        # type: DropPrimaryQueryIndexOptions
                           **kwargs):
        """
        Drops a primary index.

        :param bucket_name: name of the bucket.
        :param index_name:  name of the index.
        :param ignore_if_not_exists: Don't error/throw if the index does not exist.
        :param timeout:  the time allowed for the operation to be terminated. This is controlled by the client.

        :raises: QueryIndexNotFoundException
        :raises: InvalidArgumentsException
        """
        final_args=forward_args(kwargs, *options)
        final_args['primary'] = True
        index_name = final_args.pop("index_name", "")
        self._drop_index(bucket_name, index_name, **final_args)

    def watch_indexes(self,         # type: QueryIndexManager
                      bucket_name,  # type: str
                      index_names,  # type: Iterable[str]
                      *options,     # type: WatchQueryIndexOptions
                      **kwargs):
        """
        Watch polls indexes until they are online.

        :param str bucket_name: name of the bucket.
        :param Iterable[str] index_names: name(s) of the index(es).
        :param WatchQueryIndexOptions options: Options for request to watch indexes.
        :param Any kwargs: Override corresponding valud in options.
        :raises: QueryIndexNotFoundException
        :raises: InvalidArgumentsException
        """
        final_args=forward_args(kwargs, *options)
        BucketManager(self._admin_bucket).n1ql_index_watch(index_names, **final_args)

    def build_deferred_indexes(self,            # type: QueryIndexManager
                               bucket_name,     # type: str
                               *options,        # type: BuildDeferredQueryIndexOptions
                               **kwargs
                               ):
        """
        Build Deferred builds all indexes which are currently in deferred state.

        :param str bucket_name: name of the bucket.
        :param BuildDeferredQueryIndexOptions options: Options for building deferred indexes.
        :param Any kwargs: Override corresponding value in options.
        :raise: InvalidArgumentsException

        """
        final_args=forward_args(kwargs, *options)
        return BucketManager._n1ql_index_build_deferred(bucket_name, self._admin_bucket, **final_args)

class IndexType(object):
    pass

@attr.s
class QueryIndex(Protocol):
    """The QueryIndex protocol provides a means of mapping a query index into an object."""

    name = attr.ib(validator=io(str))  # type: str
    is_primary = attr.ib(validator=io(bool))  # type: bool
    type = attr.ib(validator=io(IndexType), type=IndexType)  # type: IndexType
    state = attr.ib(validator=io(str))  # type: str
    keyspace = attr.ib(validator=io(str))  # type: str
    index_key = attr.ib(validator=io(Iterable))  # type: Iterable[str]
    condition = attr.ib(validator=io(str))  # type: str
    @classmethod
    def from_n1qlindex(cls,
                       n1qlindex  # type: N1qlIndex
                       ):
        return cls(n1qlindex.name, bool(n1qlindex.primary), IndexType(), n1qlindex.state, n1qlindex.keyspace, [], n1qlindex.condition or "")

class GetAllQueryIndexOptions(OptionBlockTimeOut):
    pass

class CreateQueryIndexOptions(OptionBlockTimeOut):
    @overload
    def __init__(self,
                 timeout=None,           # type: timedelta
                 ignore_if_exists=None,  # type: bool
                 num_replicas=None,      # type: int
                 deferred=None,          # type: bool
                 ):
        pass

    def __init__(self, **kwargs):
        if 'ignore_if_exists' not in kwargs:
            kwargs['ignore_if_exists'] = False
        super(CreateQueryIndexOptions, self).__init__(**kwargs)

class CreatePrimaryQueryIndexOptions(CreateQueryIndexOptions):
    @overload
    def __init__(self,
                 index_name=None,        # type: str
                 timeout=None,           # type: timedelta
                 ignore_if_exists=None,  # type: bool
                 num_replicas=None,      # type: int
                 deferred=None,          # type: bool
                 ):
        pass

    def __init__(self, **kwargs):
        super(CreatePrimaryQueryIndexOptions, self).__init__(**kwargs)

class DropQueryIndexOptions(OptionBlockTimeOut):
    @overload
    def __init__(self,
                 ignore_if_not_exists=None,  # type: bool
                 timeout=None                # type: timedelta
                 ):
        pass

    def __init__(self, **kwargs):
        super(DropQueryIndexOptions, self).__init__(**kwargs)

class DropPrimaryQueryIndexOptions(OptionBlockTimeOut):
    @overload
    def __init__(self,
                 index_name=None,            # str
                 ignore_if_not_exists=None,  # type: bool
                 timeout=None                # type: timedelta
                 ):
        pass

    def __init__(self, **kwargs):
        super(DropPrimaryQueryIndexOptions, self).__init__(**kwargs)

class WatchQueryIndexOptions(OptionBlock):
    @overload
    def __init__(self,
                 watch_primary=None  # type: bool
                 ):
        pass

    def __init__(self, **kwargs):
        super(WatchQueryIndexOptions, self).__init__(**kwargs)

class BuildDeferredQueryIndexOptions(OptionBlockTimeOut):
    pass
<|endoftext|>