<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# This software is under a BSD license.  See LICENSE.txt for details.

from DTRegion2D import DTRegion2D
from DTMask import DTMask
import numpy as np

class DTTriangularMesh2D(object):
    """2D triangular mesh object."""

    def __init__(self, grid, values):
        super(DTTriangularMesh2D, self).__init__()
        """Create a new 2D triangular mesh.

        Arguments:
        grid -- DTTriangularGrid2D object
        values -- vector or list of values

        """

        values = np.squeeze(values)
        assert grid.number_of_points() == len(values)
        self._grid = grid
        self._values = values

    def __dt_type__(self):
        return "2D Triangular Mesh"

    def grid(self):
        """returns DTTriangularGrid2D"""
        return self._grid

    def bounding_box(self):
        return self._grid.bounding_box()

    def write_with_shared_grid(self, datafile, name, grid_name, time, time_index):
        if grid_name not in datafile:
            datafile.write_anonymous(self._grid, grid_name)
            datafile.write_anonymous(self.__dt_type__(), "Seq_" + name)

        varname = "%s_%d" % (name, time_index)
        datafile.write_anonymous(grid_name, varname)
        datafile.write_anonymous(self._values, varname + "_V")
        datafile.write_anonymous(np.array((time,)), varname + "_time")

    def __str__(self):
        return self.__dt_type__() + ":\n  grid = " + str(self._grid)

    def __dt_write__(self, datafile, name):
        datafile.write_anonymous(self._values, name + "_V")
        datafile.write_anonymous(self._grid, name)

    @classmethod
    def from_data_file(self, datafile, name):

        name = datafile.resolve_name(name)
        values = datafile[name + "_V"]
        grid = datafile[name]
        assert values != None, "DTTriangularMesh2D: no such variable %s in %s" % (name + "_V", datafile.path())
        assert grid != None, "DTTriangularMesh2D: no such variable %s in %s" % (name, datafile)
        return DTStructuredMesh2D(grid, values)

<commit_msg>unused import<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# This software is under a BSD license.  See LICENSE.txt for details.

import numpy as np

class DTTriangularMesh2D(object):
    """2D triangular mesh object."""

    def __init__(self, grid, values):
        super(DTTriangularMesh2D, self).__init__()
        """Create a new 2D triangular mesh.

        Arguments:
        grid -- DTTriangularGrid2D object
        values -- vector or list of values

        """

        values = np.squeeze(values)
        assert grid.number_of_points() == len(values)
        self._grid = grid
        self._values = values

    def __dt_type__(self):
        return "2D Triangular Mesh"

    def grid(self):
        """returns DTTriangularGrid2D"""
        return self._grid

    def bounding_box(self):
        return self._grid.bounding_box()

    def write_with_shared_grid(self, datafile, name, grid_name, time, time_index):
        if grid_name not in datafile:
            datafile.write_anonymous(self._grid, grid_name)
            datafile.write_anonymous(self.__dt_type__(), "Seq_" + name)

        varname = "%s_%d" % (name, time_index)
        datafile.write_anonymous(grid_name, varname)
        datafile.write_anonymous(self._values, varname + "_V")
        datafile.write_anonymous(np.array((time,)), varname + "_time")

    def __str__(self):
        return self.__dt_type__() + ":\n  grid = " + str(self._grid)

    def __dt_write__(self, datafile, name):
        datafile.write_anonymous(self._values, name + "_V")
        datafile.write_anonymous(self._grid, name)

    @classmethod
    def from_data_file(self, datafile, name):

        name = datafile.resolve_name(name)
        values = datafile[name + "_V"]
        grid = datafile[name]
        assert values != None, "DTTriangularMesh2D: no such variable %s in %s" % (name + "_V", datafile.path())
        assert grid != None, "DTTriangularMesh2D: no such variable %s in %s" % (name, datafile)
        return DTStructuredMesh2D(grid, values)

<|endoftext|>


<commit_before>from sympy import MatrixSymbol, Q, ask, Identity, ZeroMatrix, Trace
from sympy.utilities.pytest import XFAIL

X = MatrixSymbol('X', 2, 2)
Y = MatrixSymbol('Y', 2, 3)
Z = MatrixSymbol('Z', 2, 2)

def test_invertible():
    assert ask(Q.invertible(X), Q.invertible(X))
    assert ask(Q.invertible(Y)) is False
    assert ask(Q.invertible(X*Y), Q.invertible(X)) is False
    assert ask(Q.invertible(X*Z), Q.invertible(X)) is None
    assert ask(Q.invertible(X*Z), Q.invertible(X) & Q.invertible(Z)) is True
    assert ask(Q.invertible(X.T)) is None
    assert ask(Q.invertible(X.T), Q.invertible(X)) is True
    assert ask(Q.invertible(X.I)) is True
    assert ask(Q.invertible(Identity(3))) is True
    assert ask(Q.invertible(ZeroMatrix(3, 3))) is False

def test_symmetric():
    assert ask(Q.symmetric(X), Q.symmetric(X))
    assert ask(Q.symmetric(X*Z), Q.symmetric(X)) is None
    assert ask(Q.symmetric(X*Z), Q.symmetric(X) & Q.symmetric(Z)) is True
    assert ask(Q.symmetric(X + Z), Q.symmetric(X) & Q.symmetric(Z)) is True
    assert ask(Q.symmetric(Y)) is False
    assert ask(Q.symmetric(Y*Y.T)) is True
    assert ask(Q.symmetric(Y.T*X*Y)) is None
    assert ask(Q.symmetric(Y.T*X*Y), Q.symmetric(X)) is True
    assert ask(Q.symmetric(X*X*X*X*X*X*X*X*X*X), Q.symmetric(X)) is True

def test_orthogonal():
    assert ask(Q.orthogonal(X), Q.orthogonal(X))
    assert ask(Q.orthogonal(X.T), Q.orthogonal(X)) is True
    assert ask(Q.orthogonal(X.I), Q.orthogonal(X)) is True
    assert ask(Q.orthogonal(Y)) is False
    assert ask(Q.orthogonal(X)) is None
    assert ask(Q.orthogonal(X*Z*X), Q.orthogonal(X) & Q.orthogonal(Z)) is True
    assert ask(Q.orthogonal(Identity(3))) is True
    assert ask(Q.orthogonal(ZeroMatrix(3, 3))) is False
    assert ask(Q.invertible(X), Q.orthogonal(X))
    assert not ask(Q.orthogonal(X + Z), Q.orthogonal(X) & Q.orthogonal(Z))

def test_positive_definite():
    assert ask(Q.positive_definite(X), Q.positive_definite(X))
    assert ask(Q.positive_definite(X.T), Q.positive_definite(X)) is True
    assert ask(Q.positive_definite(X.I), Q.positive_definite(X)) is True
    assert ask(Q.positive_definite(Y)) is False
    assert ask(Q.positive_definite(X)) is None
    assert ask(Q.positive_definite(X*Z*X),
            Q.positive_definite(X) & Q.positive_definite(Z)) is True
    assert ask(Q.positive_definite(X), Q.orthogonal(X))
    assert ask(Q.positive_definite(Y.T*X*Y),
            Q.positive_definite(X) & Q.orthogonal(Y)) is True
    assert ask(Q.positive_definite(Identity(3))) is True
    assert ask(Q.positive_definite(ZeroMatrix(3, 3))) is False
    assert ask(Q.positive_definite(X + Z), Q.positive_definite(X) &
            Q.positive_definite(Z)) is True
    assert not ask(Q.positive_definite(-X), Q.positive_definite(X))

def test_triangular():
    assert ask(Q.upper_triangular(X + Z.T + Identity(2)), Q.upper_triangular(X) &
            Q.lower_triangular(Z)) is True
    assert ask(Q.upper_triangular(X*Z.T), Q.upper_triangular(X) &
            Q.lower_triangular(Z)) is True
    assert ask(Q.lower_triangular(Identity(3))) is True
    assert ask(Q.lower_triangular(ZeroMatrix(3, 3))) is True
    assert ask(Q.triangular(X), Q.unit_triangular(X))

def test_diagonal():
    assert ask(Q.diagonal(X + Z.T + Identity(2)), Q.diagonal(X) &
               Q.diagonal(Z)) is True
    assert ask(Q.diagonal(ZeroMatrix(3, 3)))
    assert ask(Q.lower_triangular(X) & Q.upper_triangular(X), Q.diagonal(X))
    assert ask(Q.diagonal(X), Q.lower_triangular(X) & Q.upper_triangular(X))
    assert ask(Q.diagonal(X), Q.symmetric(X))
    assert ask(Q.diagonal(X), Q.triangular(X))

def test_non_atoms():
    assert ask(Q.real(Trace(X)), Q.positive(Trace(X)))

@XFAIL
def test_non_trivial_implies():
    X = MatrixSymbol('X', 3, 3)
    Y = MatrixSymbol('Y', 3, 3)
    assert ask(Q.lower_triangular(X+Y), Q.lower_triangular(X) &
               Q.lower_triangular(Y))
    assert ask(Q.triangular(X), Q.lower_triangular(X))
    assert ask(Q.triangular(X+Y), Q.lower_triangular(X) &
               Q.lower_triangular(Y))
<commit_msg>correct diagonal tests in assumptions<commit_after>from sympy import MatrixSymbol, Q, ask, Identity, ZeroMatrix, Trace
from sympy.utilities.pytest import XFAIL

X = MatrixSymbol('X', 2, 2)
Y = MatrixSymbol('Y', 2, 3)
Z = MatrixSymbol('Z', 2, 2)

def test_invertible():
    assert ask(Q.invertible(X), Q.invertible(X))
    assert ask(Q.invertible(Y)) is False
    assert ask(Q.invertible(X*Y), Q.invertible(X)) is False
    assert ask(Q.invertible(X*Z), Q.invertible(X)) is None
    assert ask(Q.invertible(X*Z), Q.invertible(X) & Q.invertible(Z)) is True
    assert ask(Q.invertible(X.T)) is None
    assert ask(Q.invertible(X.T), Q.invertible(X)) is True
    assert ask(Q.invertible(X.I)) is True
    assert ask(Q.invertible(Identity(3))) is True
    assert ask(Q.invertible(ZeroMatrix(3, 3))) is False

def test_symmetric():
    assert ask(Q.symmetric(X), Q.symmetric(X))
    assert ask(Q.symmetric(X*Z), Q.symmetric(X)) is None
    assert ask(Q.symmetric(X*Z), Q.symmetric(X) & Q.symmetric(Z)) is True
    assert ask(Q.symmetric(X + Z), Q.symmetric(X) & Q.symmetric(Z)) is True
    assert ask(Q.symmetric(Y)) is False
    assert ask(Q.symmetric(Y*Y.T)) is True
    assert ask(Q.symmetric(Y.T*X*Y)) is None
    assert ask(Q.symmetric(Y.T*X*Y), Q.symmetric(X)) is True
    assert ask(Q.symmetric(X*X*X*X*X*X*X*X*X*X), Q.symmetric(X)) is True

def test_orthogonal():
    assert ask(Q.orthogonal(X), Q.orthogonal(X))
    assert ask(Q.orthogonal(X.T), Q.orthogonal(X)) is True
    assert ask(Q.orthogonal(X.I), Q.orthogonal(X)) is True
    assert ask(Q.orthogonal(Y)) is False
    assert ask(Q.orthogonal(X)) is None
    assert ask(Q.orthogonal(X*Z*X), Q.orthogonal(X) & Q.orthogonal(Z)) is True
    assert ask(Q.orthogonal(Identity(3))) is True
    assert ask(Q.orthogonal(ZeroMatrix(3, 3))) is False
    assert ask(Q.invertible(X), Q.orthogonal(X))
    assert not ask(Q.orthogonal(X + Z), Q.orthogonal(X) & Q.orthogonal(Z))

def test_positive_definite():
    assert ask(Q.positive_definite(X), Q.positive_definite(X))
    assert ask(Q.positive_definite(X.T), Q.positive_definite(X)) is True
    assert ask(Q.positive_definite(X.I), Q.positive_definite(X)) is True
    assert ask(Q.positive_definite(Y)) is False
    assert ask(Q.positive_definite(X)) is None
    assert ask(Q.positive_definite(X*Z*X),
            Q.positive_definite(X) & Q.positive_definite(Z)) is True
    assert ask(Q.positive_definite(X), Q.orthogonal(X))
    assert ask(Q.positive_definite(Y.T*X*Y),
            Q.positive_definite(X) & Q.orthogonal(Y)) is True
    assert ask(Q.positive_definite(Identity(3))) is True
    assert ask(Q.positive_definite(ZeroMatrix(3, 3))) is False
    assert ask(Q.positive_definite(X + Z), Q.positive_definite(X) &
            Q.positive_definite(Z)) is True
    assert not ask(Q.positive_definite(-X), Q.positive_definite(X))

def test_triangular():
    assert ask(Q.upper_triangular(X + Z.T + Identity(2)), Q.upper_triangular(X) &
            Q.lower_triangular(Z)) is True
    assert ask(Q.upper_triangular(X*Z.T), Q.upper_triangular(X) &
            Q.lower_triangular(Z)) is True
    assert ask(Q.lower_triangular(Identity(3))) is True
    assert ask(Q.lower_triangular(ZeroMatrix(3, 3))) is True
    assert ask(Q.triangular(X), Q.unit_triangular(X))

def test_diagonal():
    assert ask(Q.diagonal(X + Z.T + Identity(2)), Q.diagonal(X) &
               Q.diagonal(Z)) is True
    assert ask(Q.diagonal(ZeroMatrix(3, 3)))
    assert ask(Q.lower_triangular(X) & Q.upper_triangular(X), Q.diagonal(X))
    assert ask(Q.diagonal(X), Q.lower_triangular(X) & Q.upper_triangular(X))
    assert ask(Q.symmetric(X), Q.diagonal(X))
    assert ask(Q.triangular(X), Q.diagonal(X))

def test_non_atoms():
    assert ask(Q.real(Trace(X)), Q.positive(Trace(X)))

@XFAIL
def test_non_trivial_implies():
    X = MatrixSymbol('X', 3, 3)
    Y = MatrixSymbol('Y', 3, 3)
    assert ask(Q.lower_triangular(X+Y), Q.lower_triangular(X) &
               Q.lower_triangular(Y))
    assert ask(Q.triangular(X), Q.lower_triangular(X))
    assert ask(Q.triangular(X+Y), Q.lower_triangular(X) &
               Q.lower_triangular(Y))
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import re

from wtforms.compat import string_types, text_type

__all__ = (
    'DataRequired', 'data_required', 'Email', 'email', 'EqualTo', 'equal_to',
    'IPAddress', 'ip_address', 'InputRequired', 'input_required', 'Length',
    'length', 'NumberRange', 'number_range', 'Optional', 'optional',
    'Required', 'required', 'Regexp', 'regexp', 'URL', 'url', 'AnyOf',
    'any_of', 'NoneOf', 'none_of', 'MacAddress', 'mac_address', 'UUID'
)

class ValidationError(ValueError):
    """
    Raised when a validator fails to validate its input.
    """
    def __init__(self, message='', *args, **kwargs):
        ValueError.__init__(self, message, *args, **kwargs)

class StopValidation(Exception):
    """
    Causes the validation chain to stop.

    If StopValidation is raised, no more validators in the validation chain are
    called. If raised with a message, the message will be added to the errors
    list.
    """
    def __init__(self, message='', *args, **kwargs):
        Exception.__init__(self, message, *args, **kwargs)

class EqualTo(object):
    """
    Compares the values of two fields.

    :param fieldname:
        The name of the other field to compare to.
    :param message:
        Error message to raise in case of a validation error. Can be
        interpolated with `%(other_label)s` and `%(other_name)s` to provide a
        more helpful error.
    """
    def __init__(self, fieldname, message=None):
        self.fieldname = fieldname
        self.message = message

    def __call__(self, form, field):
        try:
            other = form[self.fieldname]
        except KeyError:
            raise ValidationError(field.gettext("Invalid field name '%s'.") % self.fieldname)
        if field.data != other.data:
            d = {
                'other_label': hasattr(other, 'label') and other.label.text or self.fieldname,
                'other_name': self.fieldname
            }
            if self.message is None:
                self.message = field.gettext('Field must be equal to %(other_name)s.')

            raise ValidationError(self.message % d)

class Length(object):
    """
    Validates the length of a string.

    :param min:
        The minimum required length of the string. If not provided, minimum
        length will not be checked.
    :param max:
        The maximum length of the string. If not provided, maximum length
        will not be checked.
    :param message:
        Error message to raise in case of a validation error. Can be
        interpolated using `%(min)d` and `%(max)d` if desired. Useful defaults
        are provided depending on the existence of min and max.
    """
    def __init__(self, min=-1, max=-1, message=None):
        assert min != -1 or max!=-1, 'At least one of `min` or `max` must be specified.'
        assert max == -1 or min <= max, '`min` cannot be more than `max`.'
        self.min = min
        self.max = max
        self.message = message

    def __call__(self, form, field):
        l = field.data and len(field.data) or 0
        if l < self.min or self.max != -1 and l > self.max:
            if self.message is None:
                if self.max == -1:
                    self.message = field.ngettext('Field must be at least %(min)d character long.',
                                                  'Field must be at least %(min)d characters long.', self.min)
                elif self.min == -1:
                    self.message = field.ngettext('Field cannot be longer than %(max)d character.',
                                                  'Field cannot be longer than %(max)d characters.', self.max)
                else:
                    self.message = field.gettext('Field must be between %(min)d and %(max)d characters long.')

            raise ValidationError(self.message % dict(min=self.min, max=self.max))

class NumberRange(object):
    """
    Validates that a number is of a minimum and/or maximum value, inclusive.
    This will work with any comparable number type, such as floats and
    decimals, not just integers.

    :param min:
        The minimum required value of the number. If not provided, minimum
        value will not be checked.
    :param max:
        The maximum value of the number. If not provided, maximum value
        will not be checked.
    :param message:
        Error message to raise in case of a validation error. Can be
        interpolated using `%(min)s` and `%(max)s` if desired. Useful defaults
        are provided depending on the existence of min and max.
    """
    def __init__(self, min=None, max=None, message=None):
        self.min = min
        self.max = max
        self.message = message

    def __call__(self, form, field):
        data = field.data
        if data is None or (self.min is not None and data < self.min) or \
            (self.max is not None and data > self.max):
            if self.message is None:
                # we use %(min)s interpolation to support floats, None, and
                # Decimals without throwing a formatting exception.
                if self.max is None:
                    self.message = field.gettext('Number must be at least %(min)s.')
                elif self.min is None:
                    self.message = field.gettext('Number must be at most %(max)s.')
                else:
                    self.message = field.gettext('Number must be between %(min)s and %(max)s.')

            raise ValidationError(self.message % dict(min=self.min, max=self.max))

class Optional(object):
    """
    Allows empty input and stops the validation chain from continuing.

    If input is empty, also removes prior errors (such as processing errors)
    from the field.

    :param strip_whitespace:
        If True (the default) also stop the validation chain on input which
        consists of only whitespace.
    """
    field_flags = ('optional', )

    def __init__(self, strip_whitespace=True):
        if strip_whitespace:
            self.string_check = lambda s: s.strip()
        else:
            self.string_check = lambda s: s

    def __call__(self, form, field):
        if not field.raw_data or isinstance(field.raw_data[0], string_types) and not self.string_check(field.raw_data[0]):
            field.data = None
            field.errors[:] = []
            raise StopValidation()

class DataRequired(object):
    """
    Validates that the field contains data. This validator will stop the
    validation chain on error.

    If the data is empty, also removes prior errors (such as processing errors)
    from the field.

    :param message:
        Error message to raise in case of a validation error.
    """
    field_flags = ('required', )

    def __init__(self, message=None):
        self.message = message

    def __call__(self, form, field):
        if not field.data or isinstance(field.data, string_types) and not field.data.strip():
            if self.message is None:
                self.message = field.gettext('This field is required.')

            field.errors[:] = []
            raise StopValidation(self.message)

class Required(DataRequired):
    """
    Legacy alias for DataRequired.

    This is needed over simple aliasing for those who require that the
    class-name of required be 'Required.'

    This class will start throwing deprecation warnings in WTForms 1.1 and be removed by 1.2.
    """

class InputRequired(object):
    """
    Validates that input was provided for this field.

    Note there is a distinction between this and DataRequired in that
    InputRequired looks that form-input data was provided, and DataRequired
    looks at the post-coercion data.
    """
    field_flags = ('required', )

    def __init__(self, message=None):
        self.message = message

    def __call__(self, form, field):
        if not field.raw_data or not field.raw_data[0]:
            if self.message is None:
                self.message = field.gettext('This field is required.')

            field.errors[:] = []
            raise StopValidation(self.message)

class Regexp(object):
    """
    Validates the field against a user provided regexp.

    :param regex:
        The regular expression string to use. Can also be a compiled regular
        expression pattern.
    :param flags:
        The regexp flags to use, for example re.IGNORECASE. Ignored if
        `regex` is not a string.
    :param message:
        Error message to raise in case of a validation error.
    """
    def __init__(self, regex, flags=0, message=None):
        if isinstance(regex, string_types):
            regex = re.compile(regex, flags)
        self.regex = regex
        self.message = message

    def __call__(self, form, field):
        if not self.regex.match(field.data or ''):
            if self.message is None:
                self.message = field.gettext('Invalid input.')

            raise ValidationError(self.message)

class Email(Regexp):
    """
    Validates an email address. Note that this uses a very primitive regular
    expression and should only be used in instances where you later verify by
    other means, such as email activation or lookups.

    :param message:
        Error message to raise in case of a validation error.
    """
    def __init__(self, message=None):
        super(Email, self).__init__(r'^.+@[^.].*\.[a-z]{2,10}$', re.IGNORECASE, message)

    def __call__(self, form, field):
        if self.message is None:
            self.message = field.gettext('Invalid email address.')

        super(Email, self).__call__(form, field)

class IPAddress(object):
    """
    Validates an IPv4 (IPv6 too with ipv6=True) address.

    :param ipv6:
        If True, accept IPv6 as valid also.
    :param message:
        Error message to raise in case of a validation error.
    """
    def __init__(self, ipv6=False, message=None):
        self.ipv6 = ipv6
        self.message = message

    def __call__(self, form, field):
        value = field.data
        valid = False
        if value:
            valid = self.check_ipv4(value)

            if not valid and self.ipv6:
                valid = self.check_ipv6(value)

        if not valid:
            if self.message is None:
                self.message = field.gettext('Invalid IP address.')
            raise ValidationError(self.message)

    def check_ipv4(self, value):
        parts = value.split('.')
        if len(parts) == 4 and all(x.isdigit() for x in parts):
            numbers = list(int(x) for x in parts)
            return all(num >= 0 and num < 256 for num in numbers)
        return False

    def check_ipv6(self, value):
        parts = value.split(':')
        if len(parts) > 8:
            return False

        num_blank = 0
        for part in parts:
            if not part:
                num_blank += 1
            else:
                try:
                    value = int(part, 16)
                except ValueError:
                    return False
                else:
                    if value < 0 or value >= 65536:
                        return False

        if num_blank < 2:
            return True
        elif num_blank == 2 and not parts[0] and not parts[1]:
            return True
        return False

class MacAddress(Regexp):
    """
    Validates a MAC address.

    :param message:
        Error message to raise in case of a validation error.
    """
    def __init__(self, message=None):
        pattern = r'^(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}$'
        super(MacAddress, self).__init__(pattern, message=message)

    def __call__(self, form, field):
        if self.message is None:
            self.message = field.gettext('Invalid Mac address.')

        super(MacAddress, self).__call__(form, field)

class URL(Regexp):
    """
    Simple regexp based url validation. Much like the email validator, you
    probably want to validate the url later by other means if the url must
    resolve.

    :param require_tld:
        If true, then the domain-name portion of the URL must contain a .tld
        suffix.  Set this to false if you want to allow domains like
        `localhost`.
    :param message:
        Error message to raise in case of a validation error.
    """
    def __init__(self, require_tld=True, message=None):
        tld_part = (require_tld and r'\.[a-z]{2,10}' or '')
        regex = r'^[a-z]+://([^/:]+%s|([0-9]{1,3}\.){3}[0-9]{1,3})(:[0-9]+)?(\/.*)?$' % tld_part
        super(URL, self).__init__(regex, re.IGNORECASE, message)

    def __call__(self, form, field):
        if self.message is None:
            self.message = field.gettext('Invalid URL.')

        super(URL, self).__call__(form, field)

class UUID(Regexp):
    """
    Validates a UUID.

    :param message:
        Error message to raise in case of a validation error.
    """
    def __init__(self, message=None):
        pattern = r'^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$'
        super(UUID, self).__init__(pattern, message=message)

    def __call__(self, form, field):
        if self.message is None:
            self.message = field.gettext('Invalid UUID.')

        super(UUID, self).__call__(form, field)

class AnyOf(object):
    """
    Compares the incoming data to a sequence of valid inputs.

    :param values:
        A sequence of valid inputs.
    :param message:
        Error message to raise in case of a validation error. `%(values)s`
        contains the list of values.
    :param values_formatter:
        Function used to format the list of values in the error message.
    """
    def __init__(self, values, message=None, values_formatter=None):
        self.values = values
        self.message = message
        if values_formatter is None:
            values_formatter = lambda v: ', '.join(text_type(x) for x in v)
        self.values_formatter = values_formatter

    def __call__(self, form, field):
        if field.data not in self.values:
            if self.message is None:
                self.message = field.gettext('Invalid value, must be one of: %(values)s.')

            raise ValidationError(self.message % dict(values=self.values_formatter(self.values)))

class NoneOf(object):
    """
    Compares the incoming data to a sequence of invalid inputs.

    :param values:
        A sequence of invalid inputs.
    :param message:
        Error message to raise in case of a validation error. `%(values)s`
        contains the list of values.
    :param values_formatter:
        Function used to format the list of values in the error message.
    """
    def __init__(self, values, message=None, values_formatter=None):
        self.values = values
        self.message = message
        if values_formatter is None:
            values_formatter = lambda v: ', '.join(text_type(x) for x in v)
        self.values_formatter = values_formatter

    def __call__(self, form, field):
        if field.data in self.values:
            if self.message is None:
                self.message = field.gettext('Invalid value, can\'t be any of: %(values)s.')

            raise ValidationError(self.message % dict(values=self.values_formatter(self.values)))

email = Email
equal_to = EqualTo
ip_address = IPAddress
mac_address = MacAddress
length = Length
number_range = NumberRange
optional = Optional
required = Required
input_required = InputRequired
data_required = DataRequired
regexp = Regexp
url = URL
any_of = AnyOf
none_of = NoneOf
<commit_msg>dont set field to None on optional<commit_after>from __future__ import unicode_literals

import re

from wtforms.compat import string_types, text_type

__all__ = (
    'DataRequired', 'data_required', 'Email', 'email', 'EqualTo', 'equal_to',
    'IPAddress', 'ip_address', 'InputRequired', 'input_required', 'Length',
    'length', 'NumberRange', 'number_range', 'Optional', 'optional',
    'Required', 'required', 'Regexp', 'regexp', 'URL', 'url', 'AnyOf',
    'any_of', 'NoneOf', 'none_of', 'MacAddress', 'mac_address', 'UUID'
)

class ValidationError(ValueError):
    """
    Raised when a validator fails to validate its input.
    """
    def __init__(self, message='', *args, **kwargs):
        ValueError.__init__(self, message, *args, **kwargs)

class StopValidation(Exception):
    """
    Causes the validation chain to stop.

    If StopValidation is raised, no more validators in the validation chain are
    called. If raised with a message, the message will be added to the errors
    list.
    """
    def __init__(self, message='', *args, **kwargs):
        Exception.__init__(self, message, *args, **kwargs)

class EqualTo(object):
    """
    Compares the values of two fields.

    :param fieldname:
        The name of the other field to compare to.
    :param message:
        Error message to raise in case of a validation error. Can be
        interpolated with `%(other_label)s` and `%(other_name)s` to provide a
        more helpful error.
    """
    def __init__(self, fieldname, message=None):
        self.fieldname = fieldname
        self.message = message

    def __call__(self, form, field):
        try:
            other = form[self.fieldname]
        except KeyError:
            raise ValidationError(field.gettext("Invalid field name '%s'.") % self.fieldname)
        if field.data != other.data:
            d = {
                'other_label': hasattr(other, 'label') and other.label.text or self.fieldname,
                'other_name': self.fieldname
            }
            if self.message is None:
                self.message = field.gettext('Field must be equal to %(other_name)s.')

            raise ValidationError(self.message % d)

class Length(object):
    """
    Validates the length of a string.

    :param min:
        The minimum required length of the string. If not provided, minimum
        length will not be checked.
    :param max:
        The maximum length of the string. If not provided, maximum length
        will not be checked.
    :param message:
        Error message to raise in case of a validation error. Can be
        interpolated using `%(min)d` and `%(max)d` if desired. Useful defaults
        are provided depending on the existence of min and max.
    """
    def __init__(self, min=-1, max=-1, message=None):
        assert min != -1 or max!=-1, 'At least one of `min` or `max` must be specified.'
        assert max == -1 or min <= max, '`min` cannot be more than `max`.'
        self.min = min
        self.max = max
        self.message = message

    def __call__(self, form, field):
        l = field.data and len(field.data) or 0
        if l < self.min or self.max != -1 and l > self.max:
            if self.message is None:
                if self.max == -1:
                    self.message = field.ngettext('Field must be at least %(min)d character long.',
                                                  'Field must be at least %(min)d characters long.', self.min)
                elif self.min == -1:
                    self.message = field.ngettext('Field cannot be longer than %(max)d character.',
                                                  'Field cannot be longer than %(max)d characters.', self.max)
                else:
                    self.message = field.gettext('Field must be between %(min)d and %(max)d characters long.')

            raise ValidationError(self.message % dict(min=self.min, max=self.max))

class NumberRange(object):
    """
    Validates that a number is of a minimum and/or maximum value, inclusive.
    This will work with any comparable number type, such as floats and
    decimals, not just integers.

    :param min:
        The minimum required value of the number. If not provided, minimum
        value will not be checked.
    :param max:
        The maximum value of the number. If not provided, maximum value
        will not be checked.
    :param message:
        Error message to raise in case of a validation error. Can be
        interpolated using `%(min)s` and `%(max)s` if desired. Useful defaults
        are provided depending on the existence of min and max.
    """
    def __init__(self, min=None, max=None, message=None):
        self.min = min
        self.max = max
        self.message = message

    def __call__(self, form, field):
        data = field.data
        if data is None or (self.min is not None and data < self.min) or \
            (self.max is not None and data > self.max):
            if self.message is None:
                # we use %(min)s interpolation to support floats, None, and
                # Decimals without throwing a formatting exception.
                if self.max is None:
                    self.message = field.gettext('Number must be at least %(min)s.')
                elif self.min is None:
                    self.message = field.gettext('Number must be at most %(max)s.')
                else:
                    self.message = field.gettext('Number must be between %(min)s and %(max)s.')

            raise ValidationError(self.message % dict(min=self.min, max=self.max))

class Optional(object):
    """
    Allows empty input and stops the validation chain from continuing.

    If input is empty, also removes prior errors (such as processing errors)
    from the field.

    :param strip_whitespace:
        If True (the default) also stop the validation chain on input which
        consists of only whitespace.
    """
    field_flags = ('optional', )

    def __init__(self, strip_whitespace=True):
        if strip_whitespace:
            self.string_check = lambda s: s.strip()
        else:
            self.string_check = lambda s: s

    def __call__(self, form, field):
        if not field.raw_data or isinstance(field.raw_data[0], string_types) and not self.string_check(field.raw_data[0]):
            field.errors[:] = []
            raise StopValidation()

class DataRequired(object):
    """
    Validates that the field contains data. This validator will stop the
    validation chain on error.

    If the data is empty, also removes prior errors (such as processing errors)
    from the field.

    :param message:
        Error message to raise in case of a validation error.
    """
    field_flags = ('required', )

    def __init__(self, message=None):
        self.message = message

    def __call__(self, form, field):
        if not field.data or isinstance(field.data, string_types) and not field.data.strip():
            if self.message is None:
                self.message = field.gettext('This field is required.')

            field.errors[:] = []
            raise StopValidation(self.message)

class Required(DataRequired):
    """
    Legacy alias for DataRequired.

    This is needed over simple aliasing for those who require that the
    class-name of required be 'Required.'

    This class will start throwing deprecation warnings in WTForms 1.1 and be removed by 1.2.
    """

class InputRequired(object):
    """
    Validates that input was provided for this field.

    Note there is a distinction between this and DataRequired in that
    InputRequired looks that form-input data was provided, and DataRequired
    looks at the post-coercion data.
    """
    field_flags = ('required', )

    def __init__(self, message=None):
        self.message = message

    def __call__(self, form, field):
        if not field.raw_data or not field.raw_data[0]:
            if self.message is None:
                self.message = field.gettext('This field is required.')

            field.errors[:] = []
            raise StopValidation(self.message)

class Regexp(object):
    """
    Validates the field against a user provided regexp.

    :param regex:
        The regular expression string to use. Can also be a compiled regular
        expression pattern.
    :param flags:
        The regexp flags to use, for example re.IGNORECASE. Ignored if
        `regex` is not a string.
    :param message:
        Error message to raise in case of a validation error.
    """
    def __init__(self, regex, flags=0, message=None):
        if isinstance(regex, string_types):
            regex = re.compile(regex, flags)
        self.regex = regex
        self.message = message

    def __call__(self, form, field):
        if not self.regex.match(field.data or ''):
            if self.message is None:
                self.message = field.gettext('Invalid input.')

            raise ValidationError(self.message)

class Email(Regexp):
    """
    Validates an email address. Note that this uses a very primitive regular
    expression and should only be used in instances where you later verify by
    other means, such as email activation or lookups.

    :param message:
        Error message to raise in case of a validation error.
    """
    def __init__(self, message=None):
        super(Email, self).__init__(r'^.+@[^.].*\.[a-z]{2,10}$', re.IGNORECASE, message)

    def __call__(self, form, field):
        if self.message is None:
            self.message = field.gettext('Invalid email address.')

        super(Email, self).__call__(form, field)

class IPAddress(object):
    """
    Validates an IPv4 (IPv6 too with ipv6=True) address.

    :param ipv6:
        If True, accept IPv6 as valid also.
    :param message:
        Error message to raise in case of a validation error.
    """
    def __init__(self, ipv6=False, message=None):
        self.ipv6 = ipv6
        self.message = message

    def __call__(self, form, field):
        value = field.data
        valid = False
        if value:
            valid = self.check_ipv4(value)

            if not valid and self.ipv6:
                valid = self.check_ipv6(value)

        if not valid:
            if self.message is None:
                self.message = field.gettext('Invalid IP address.')
            raise ValidationError(self.message)

    def check_ipv4(self, value):
        parts = value.split('.')
        if len(parts) == 4 and all(x.isdigit() for x in parts):
            numbers = list(int(x) for x in parts)
            return all(num >= 0 and num < 256 for num in numbers)
        return False

    def check_ipv6(self, value):
        parts = value.split(':')
        if len(parts) > 8:
            return False

        num_blank = 0
        for part in parts:
            if not part:
                num_blank += 1
            else:
                try:
                    value = int(part, 16)
                except ValueError:
                    return False
                else:
                    if value < 0 or value >= 65536:
                        return False

        if num_blank < 2:
            return True
        elif num_blank == 2 and not parts[0] and not parts[1]:
            return True
        return False

class MacAddress(Regexp):
    """
    Validates a MAC address.

    :param message:
        Error message to raise in case of a validation error.
    """
    def __init__(self, message=None):
        pattern = r'^(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}$'
        super(MacAddress, self).__init__(pattern, message=message)

    def __call__(self, form, field):
        if self.message is None:
            self.message = field.gettext('Invalid Mac address.')

        super(MacAddress, self).__call__(form, field)

class URL(Regexp):
    """
    Simple regexp based url validation. Much like the email validator, you
    probably want to validate the url later by other means if the url must
    resolve.

    :param require_tld:
        If true, then the domain-name portion of the URL must contain a .tld
        suffix.  Set this to false if you want to allow domains like
        `localhost`.
    :param message:
        Error message to raise in case of a validation error.
    """
    def __init__(self, require_tld=True, message=None):
        tld_part = (require_tld and r'\.[a-z]{2,10}' or '')
        regex = r'^[a-z]+://([^/:]+%s|([0-9]{1,3}\.){3}[0-9]{1,3})(:[0-9]+)?(\/.*)?$' % tld_part
        super(URL, self).__init__(regex, re.IGNORECASE, message)

    def __call__(self, form, field):
        if self.message is None:
            self.message = field.gettext('Invalid URL.')

        super(URL, self).__call__(form, field)

class UUID(Regexp):
    """
    Validates a UUID.

    :param message:
        Error message to raise in case of a validation error.
    """
    def __init__(self, message=None):
        pattern = r'^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$'
        super(UUID, self).__init__(pattern, message=message)

    def __call__(self, form, field):
        if self.message is None:
            self.message = field.gettext('Invalid UUID.')

        super(UUID, self).__call__(form, field)

class AnyOf(object):
    """
    Compares the incoming data to a sequence of valid inputs.

    :param values:
        A sequence of valid inputs.
    :param message:
        Error message to raise in case of a validation error. `%(values)s`
        contains the list of values.
    :param values_formatter:
        Function used to format the list of values in the error message.
    """
    def __init__(self, values, message=None, values_formatter=None):
        self.values = values
        self.message = message
        if values_formatter is None:
            values_formatter = lambda v: ', '.join(text_type(x) for x in v)
        self.values_formatter = values_formatter

    def __call__(self, form, field):
        if field.data not in self.values:
            if self.message is None:
                self.message = field.gettext('Invalid value, must be one of: %(values)s.')

            raise ValidationError(self.message % dict(values=self.values_formatter(self.values)))

class NoneOf(object):
    """
    Compares the incoming data to a sequence of invalid inputs.

    :param values:
        A sequence of invalid inputs.
    :param message:
        Error message to raise in case of a validation error. `%(values)s`
        contains the list of values.
    :param values_formatter:
        Function used to format the list of values in the error message.
    """
    def __init__(self, values, message=None, values_formatter=None):
        self.values = values
        self.message = message
        if values_formatter is None:
            values_formatter = lambda v: ', '.join(text_type(x) for x in v)
        self.values_formatter = values_formatter

    def __call__(self, form, field):
        if field.data in self.values:
            if self.message is None:
                self.message = field.gettext('Invalid value, can\'t be any of: %(values)s.')

            raise ValidationError(self.message % dict(values=self.values_formatter(self.values)))

email = Email
equal_to = EqualTo
ip_address = IPAddress
mac_address = MacAddress
length = Length
number_range = NumberRange
optional = Optional
required = Required
input_required = InputRequired
data_required = DataRequired
regexp = Regexp
url = URL
any_of = AnyOf
none_of = NoneOf
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
import re
import hashlib
# Use cPickle in python 2 and pickle in python 3
try:
    import cPickle as pickle
except ImportError:
    import pickle

from funcy import wraps, str_join, walk_values, filter
from termcolor import colored, cprint
import numpy as np
import pandas as pd

from funcy import print_durations

DEFAULT_CACHE_DIR = '/tmp/debug_cache'
# DEFAULT_CACHE_TIMEOUT = 60*60*24*30

__all__ = ('cache', 'cached', 'CacheMiss', 'DebugCache')

FLOAT_PRECISION = 0.002
# FLOAT_PRECISION = 0.00001

def _series_equal(a, b):
    if a.dtype == 'float64':
        anan = np.isnan(a)
        bnan = np.isnan(b)
        return anan.equals(bnan) and (abs(a[~anan] - b[~bnan]) <= FLOAT_PRECISION).all()
    else:
        return a.equals(b)

def compare(a, b):
    if isinstance(a, pd.DataFrame) and isinstance(b, pd.DataFrame):
        return a.columns.equals(b.columns) and a.dtypes.equals(b.dtypes) and len(a) == len(b) \
            and all(_series_equal(a[col], b[col]) for col in a.columns)
    elif isinstance(a, pd.Series) and isinstance(b, pd.Series):
        return _series_equal(a, b)
    elif isinstance(a, (list, tuple, set, dict)) and isinstance(b, (list, tuple, set, dict)):
        return a == b
    else:
        return pickle.dumps(a) == pickle.dumps(b)

# Borrowed these from pytest.assertion.util

import pprint, py

def _compare_eq_sequence(left, right, verbose=False):
    explanation = []
    for i in range(min(len(left), len(right))):
        if left[i] != right[i]:
            explanation += ['At index %s diff: %r != %r'
                            % (i, left[i], right[i])]
            break
    if len(left) > len(right):
        explanation += ['Left contains more items, first extra item: %s'
                        % py.io.saferepr(left[len(right)],)]
    elif len(left) < len(right):
        explanation += [
            'Right contains more items, first extra item: %s' %
            py.io.saferepr(right[len(left)],)]
    return explanation  # + _diff_text(pprint.pformat(left),
                        #              pprint.pformat(right))

def _compare_eq_set(left, right, verbose=False):
    explanation = []
    diff_left = left - right
    diff_right = right - left
    if diff_left:
        explanation.append('Extra items in the left set:')
        for item in diff_left:
            explanation.append(py.io.saferepr(item))
    if diff_right:
        explanation.append('Extra items in the right set:')
        for item in diff_right:
            explanation.append(py.io.saferepr(item))
    return explanation

def _compare_eq_dict(left, right, verbose=False):
    explanation = []
    common = set(left).intersection(set(right))
    same = dict((k, left[k]) for k in common if left[k] == right[k])
    if same and not verbose:
        explanation += ['Omitting %s identical items, use -v to show' %
                        len(same)]
    elif same:
        explanation += ['Common items:']
        explanation += pprint.pformat(same).splitlines()
    diff = set(k for k in common if left[k] != right[k])
    if diff:
        explanation += ['Differing items:']
        for k in diff:
            explanation += [py.io.saferepr({k: left[k]}) + ' != ' +
                            py.io.saferepr({k: right[k]})]
    extra_left = set(left) - set(right)
    if extra_left:
        explanation.append('Left contains more items:')
        explanation.extend(pprint.pformat(
            dict((k, left[k]) for k in extra_left)).splitlines())
    extra_right = set(right) - set(left)
    if extra_right:
        explanation.append('Right contains more items:')
        explanation.extend(pprint.pformat(
            dict((k, right[k]) for k in extra_right)).splitlines())
    return explanation

# end utils from pytest.assertion.util

from funcy import joining, print_errors

@print_errors
@joining('\n')
def explain_frame_diff(a, b):
    assert isinstance(a, pd.DataFrame) and isinstance(b, pd.DataFrame)

    res = []

    # Compare column sets
    a_cols = set(a.columns)
    b_cols = set(b.columns)
    if a_cols != b_cols:
        # return _compare_eq_set(a_cols, b_cols)
        if a_cols - b_cols:
            res.append('Removed columns: %s' % ', '.join(a_cols - b_cols))
        if b_cols - a_cols:
            res.append('Added columns: %s' % ', '.join(b_cols - a_cols))
        return res

    # Compare column types
    if not a.dtypes.equals(b.dtypes):
        for name in a.columns:
            if a.dtype != b.dtype:
                res.append('Column %s changed type from %s to %s' % (name, a.dtype, b.dtype))
        return res

    # Compare len
    if len(a) != len(b):
        res.append('Data length changed from %d to %d' % (len(a), len(b)))
        return res

    # Compare indexes
    if not a.index.equals(b.index):
        return ['Indexes mismatch:'] + _compare_eq_sequence(a.index, b.index)

    # Compare values
    for name in a.columns:
        acol = a[name]
        bcol = b[name]
        if not _series_equal(acol, bcol):
            if acol.dtype == 'float64':
                diff = acol.index[abs(acol - bcol) > FLOAT_PRECISION]
            else:
                diff = acol.index[acol != bcol]
            res.append('Column %s has %d diffrences,' % (name, len(diff)))
            res.append('  first one is at index %s, changed from %s to %s'
                        % (diff[0], acol[diff[0]], bcol[diff[0]]))
    return res

from collections import Mapping

def explain_diff(a, b):
    if isinstance(a, pd.DataFrame) and isinstance(b, pd.DataFrame):
        return explain_frame_diff(a, b)
    elif isinstance(a, Mapping) and isinstance(b, Mapping):
        return '\n'.join(_compare_eq_dict(a, b))
    else:
        raise NotImplementedError("Don't know how tp compare %s to %s "
                                  % (a.__class__.__name__, b.__class__.__name__))

class CacheMiss(Exception):
    pass

class DebugCache(object):
    """
    A file cache which fixes bugs and misdesign in django default one.
    Uses mtimes in the future to designate expire time. This makes unnecessary
    reading stale files.
    """
    def __init__(self, path=DEFAULT_CACHE_DIR):
        self._path = path

    def _call_info(self, func, args, kwargs):
        serialized_args = map(serialize, args)
        serialized_kwargs = walk_values(serialize, kwargs)

        parts = []
        parts.extend(smart_str(a) for a in args)
        parts.extend('%s=%s' % (k, smart_str(v)) for k, v in sorted(kwargs.items()))
        parts.append(hash_args(serialized_args, serialized_kwargs))
        dirname = '%s/%s' % (func.__name__, '.'.join(parts))

        return dirname, serialized_args, serialized_kwargs

    def _load_call_info(self, dirname):
        path = os.path.join(self._path, dirname)
        files = os.listdir(path)

        arg_files = sorted(filter(r'^a', files))
        args = tuple(map(self._read_data, (os.path.join(path, f) for f in arg_files)))

        kwarg_files = filter(r'^k', files)
        kwarg_files = {filename[1:]: os.path.join(path, filename) for filename in kwarg_files}
        kwargs = walk_values(self._read_data, kwarg_files)

        return args, kwargs

    def cached(self, func):
        """
        A decorator for caching function calls
        """
        @wraps(func)
        def wrapper(*args, **kwargs):
            dirname, serialized_args, serialized_kwargs = self._call_info(func, args, kwargs)

            try:
                result = self._get(dirname)
            except CacheMiss:
                result = func(*args, **kwargs)
                self._set(dirname, serialized_args, serialized_kwargs, result)

            return result

        # def invalidate(*args, **kwargs):
        #     cache_key = key_func(func, args, kwargs)
        #     self.delete(cache_key)
        # wrapper.invalidate = invalidate

        return wrapper

    def checked(self, func=None, strict=True, subs=None):
        """
        Checks that function output doesn't change for same input.
        """
        # Allow using @checked without parentheses
        if callable(func):
            return self.checked()(func)

        subs = subs or {}

        def decorator(func):
            prepend = lambda s: '%s/%s' % (func.__name__, s)
            _subs = {prepend(k): prepend(v) for k, v in subs.items()}

            @wraps(func)
            def wrapper(*args, **kwargs):
                dirname, serialized_args, serialized_kwargs = self._call_info(func, args, kwargs)
                if dirname in _subs:
                    cprint('Using substition in check', 'green')
                    dirname = _subs[dirname]
                    args, kwargs = self._load_call_info(dirname)

                try:
                    print 'check', dirname
                    saved_result = self._get(dirname)
                    result = func(*args, **kwargs)
                except CacheMiss:
                    if strict:
                        cprint('No result for %s' % dirname, 'red')
                        try:
                            import ipdb; ipdb.set_trace()
                        except ImportError:
                            import pdb; pdb.set_trace()
                    else:
                        result = func(*args, **kwargs)
                        self._set(dirname, serialized_args, serialized_kwargs, result)
                else:
                    if not compare(saved_result, result):
                        cprint('Result change in %s' % dirname, 'red')
                        print explain_diff(saved_result, result)
                        # print ''
                        print 'hey'
                        try:
                            import ipdb; ipdb.set_trace()
                        except ImportError:
                            import pdb; pdb.set_trace()

                return result

            return wrapper
        return decorator

    def checked_call(self, func, state): #, strict=True):
        dirname = '%s/%s' % (func.__name__, state)
        print dirname
        args, kwargs = self._load_call_info(dirname)
        # self.checked(strict=strict)(func)(*args, **kwargs)

        try:
            saved_result = self._get(dirname)
            result = func(*args, **kwargs)
        except CacheMiss:
            raise LookupError('No recored call %s' % dirname, 'red')
        else:
            if not compare(saved_result, result):
                cprint('Result change in %s' % dirname, 'red')
                print explain_diff(saved_result, result)
                try:
                    import ipdb; ipdb.set_trace()
                except ImportError:
                    import pdb; pdb.set_trace()

    # @print_durations
    def _get(self, dirname):
        filename = os.path.join(self._path, dirname, 'out')
        try:
            # return self._read_data(filename)
            with open(filename, 'rb') as f:
                return pickle.load(f)
        except (IOError, OSError, EOFError, pickle.PickleError):
            raise CacheMiss

    # @print_durations
    def _set(self, dirname, serialized_args, serialized_kwargs, result):
        path = os.path.join(self._path, dirname)

        if not os.path.exists(path):
            os.makedirs(path)

        for i, value in enumerate(serialized_args):
            self._write_file(os.path.join(path, 'a%d' % i), value)
        for name, value in serialized_kwargs.items():
            self._write_file(os.path.join(path, 'k' + name), value)

        self._write_file(os.path.join(path, 'out'), serialize(result))

    # def _read_file(self, filename):
    #     with open(filename, 'rb') as f:
    #         return f.read()

    def _read_data(self, filename):
        with open(filename, 'rb') as f:
            return pickle.load(f)

    def _write_file(self, filename, data):
        # Use open with exclusive rights to prevent data corruption
        f = os.open(filename, os.O_EXCL | os.O_WRONLY | os.O_CREAT)
        try:
            os.write(f, data)
        finally:
            os.close(f)

    # def set(self, key, data, timeout=None):
    #     filename = self._key_to_filename(key)
    #     dirname = os.path.dirname(filename)

    #     if timeout is None:
    #         timeout = self._default_timeout

    #     try:
    #         if not os.path.exists(dirname):
    #             os.makedirs(dirname)

    #         # Use open with exclusive rights to prevent data corruption
    #         print 'w0', filename
    #         f = os.open(filename, os.O_EXCL | os.O_WRONLY | os.O_CREAT)
    #         try:
    #             print 'w1'
    #             os.write(f, pickle.dumps(data, pickle.HIGHEST_PROTOCOL))
    #             print 'w2'
    #         finally:
    #             os.close(f)

    #         # Set mtime to expire time
    #         os.utime(filename, (0, time.time() + timeout))
    #     except (IOError, OSError):
    #         pass

    def delete(self, fname):
        try:
            os.remove(fname)
            # Trying to remove directory in case it's empty
            dirname = os.path.dirname(fname)
            os.rmdir(dirname)
        except (IOError, OSError):
            pass

    # def _key_to_filename(self, key):
    #     """
    #     Returns a filename corresponding to cache key
    #     """
    #     return os.path.join(self._path, key)

cache = DebugCache()

def smart_str(value, max_len=20):
    s = str(value).strip()
    s = re.sub(r'\s+', ' ', s)
    if max_len and len(s) > max_len:
        s = s[:max_len-1] + '*'
    return s

def serialize(value):
    try:
        return pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
    except pickle.PickleError:
        return pickle.dumps(value)

def deserialize(value):
    return pickle.loads(value)

def md5hex(s):
    return hashlib.md5(s).hexdigest()

# @print_durations
def hash_args(serialized_args, serialized_kwargs):
    hash_sum = hashlib.md5()
    for ha in serialized_args:
        hash_sum.update(ha)
    for k, v in sorted(serialized_kwargs.items()):
        hash_sum.update(k)
        hash_sum.update(v)
    return hash_sum.hexdigest()
<commit_msg>Add some cache introspection features<commit_after># -*- coding: utf-8 -*-
import os
import re
import hashlib
# Use cPickle in python 2 and pickle in python 3
try:
    import cPickle as pickle
except ImportError:
    import pickle

from funcy import wraps, str_join, walk_values, filter
from termcolor import colored, cprint
import numpy as np
import pandas as pd

from funcy import print_durations

DEFAULT_CACHE_DIR = '/tmp/debug_cache'
# DEFAULT_CACHE_TIMEOUT = 60*60*24*30

__all__ = ('cache', 'cached', 'CacheMiss', 'DebugCache')

FLOAT_PRECISION = 0.002
# FLOAT_PRECISION = 0.00001

def _series_equal(a, b):
    if a.dtype == 'float64':
        anan = np.isnan(a)
        bnan = np.isnan(b)
        return anan.equals(bnan) and (abs(a[~anan] - b[~bnan]) <= FLOAT_PRECISION).all()
    else:
        return a.equals(b)

def compare(a, b):
    if isinstance(a, pd.DataFrame) and isinstance(b, pd.DataFrame):
        return a.columns.equals(b.columns) and a.dtypes.equals(b.dtypes) and len(a) == len(b) \
            and all(_series_equal(a[col], b[col]) for col in a.columns)
    elif isinstance(a, pd.Series) and isinstance(b, pd.Series):
        return _series_equal(a, b)
    elif isinstance(a, (list, tuple, set, dict)) and isinstance(b, (list, tuple, set, dict)):
        return a == b
    else:
        return pickle.dumps(a) == pickle.dumps(b)

# Borrowed these from pytest.assertion.util

import pprint, py

def _compare_eq_sequence(left, right, verbose=False):
    explanation = []
    for i in range(min(len(left), len(right))):
        if left[i] != right[i]:
            explanation += ['At index %s diff: %r != %r'
                            % (i, left[i], right[i])]
            break
    if len(left) > len(right):
        explanation += ['Left contains more items, first extra item: %s'
                        % py.io.saferepr(left[len(right)],)]
    elif len(left) < len(right):
        explanation += [
            'Right contains more items, first extra item: %s' %
            py.io.saferepr(right[len(left)],)]
    return explanation  # + _diff_text(pprint.pformat(left),
                        #              pprint.pformat(right))

def _compare_eq_set(left, right, verbose=False):
    explanation = []
    diff_left = left - right
    diff_right = right - left
    if diff_left:
        explanation.append('Extra items in the left set:')
        for item in diff_left:
            explanation.append(py.io.saferepr(item))
    if diff_right:
        explanation.append('Extra items in the right set:')
        for item in diff_right:
            explanation.append(py.io.saferepr(item))
    return explanation

def _compare_eq_dict(left, right, verbose=False):
    explanation = []
    common = set(left).intersection(set(right))
    same = dict((k, left[k]) for k in common if left[k] == right[k])
    if same and not verbose:
        explanation += ['Omitting %s identical items, use -v to show' %
                        len(same)]
    elif same:
        explanation += ['Common items:']
        explanation += pprint.pformat(same).splitlines()
    diff = set(k for k in common if left[k] != right[k])
    if diff:
        explanation += ['Differing items:']
        for k in diff:
            explanation += [py.io.saferepr({k: left[k]}) + ' != ' +
                            py.io.saferepr({k: right[k]})]
    extra_left = set(left) - set(right)
    if extra_left:
        explanation.append('Left contains more items:')
        explanation.extend(pprint.pformat(
            dict((k, left[k]) for k in extra_left)).splitlines())
    extra_right = set(right) - set(left)
    if extra_right:
        explanation.append('Right contains more items:')
        explanation.extend(pprint.pformat(
            dict((k, right[k]) for k in extra_right)).splitlines())
    return explanation

# end utils from pytest.assertion.util

from funcy import joining, print_errors

@print_errors
@joining('\n')
def explain_frame_diff(a, b):
    assert isinstance(a, pd.DataFrame) and isinstance(b, pd.DataFrame)

    res = []

    # Compare column sets
    a_cols = set(a.columns)
    b_cols = set(b.columns)
    if a_cols != b_cols:
        # return _compare_eq_set(a_cols, b_cols)
        if a_cols - b_cols:
            res.append('Removed columns: %s' % ', '.join(a_cols - b_cols))
        if b_cols - a_cols:
            res.append('Added columns: %s' % ', '.join(b_cols - a_cols))
        return res

    # Compare column types
    if not a.dtypes.equals(b.dtypes):
        for name in a.columns:
            if a.dtype != b.dtype:
                res.append('Column %s changed type from %s to %s' % (name, a.dtype, b.dtype))
        return res

    # Compare len
    if len(a) != len(b):
        res.append('Data length changed from %d to %d' % (len(a), len(b)))
        return res

    # Compare indexes
    if not a.index.equals(b.index):
        return ['Indexes mismatch:'] + _compare_eq_sequence(a.index, b.index)

    # Compare values
    for name in a.columns:
        acol = a[name]
        bcol = b[name]
        if not _series_equal(acol, bcol):
            if acol.dtype == 'float64':
                diff = acol.index[abs(acol - bcol) > FLOAT_PRECISION]
            else:
                diff = acol.index[acol != bcol]
            res.append('Column %s has %d diffrences,' % (name, len(diff)))
            res.append('  first one is at index %s, changed from %s to %s'
                        % (diff[0], acol[diff[0]], bcol[diff[0]]))
    return res

from collections import Mapping

def explain_diff(a, b):
    if isinstance(a, pd.DataFrame) and isinstance(b, pd.DataFrame):
        return explain_frame_diff(a, b)
    elif isinstance(a, Mapping) and isinstance(b, Mapping):
        return '\n'.join(_compare_eq_dict(a, b))
    else:
        raise NotImplementedError("Don't know how tp compare %s to %s "
                                  % (a.__class__.__name__, b.__class__.__name__))

class CacheMiss(Exception):
    pass

class DebugCache(object):
    """
    A file cache which fixes bugs and misdesign in django default one.
    Uses mtimes in the future to designate expire time. This makes unnecessary
    reading stale files.
    """
    def __init__(self, path=DEFAULT_CACHE_DIR):
        self._path = path

    def _call_info(self, func, args, kwargs):
        serialized_args = map(serialize, args)
        serialized_kwargs = walk_values(serialize, kwargs)

        parts = []
        parts.extend(smart_str(a) for a in args)
        parts.extend('%s=%s' % (k, smart_str(v)) for k, v in sorted(kwargs.items()))
        parts.append(hash_args(serialized_args, serialized_kwargs))
        dirname = '%s/%s' % (func.__name__, '.'.join(parts))

        return dirname, serialized_args, serialized_kwargs

    def _load_call_info(self, dirname):
        path = os.path.join(self._path, dirname)
        files = os.listdir(path)

        arg_files = sorted(filter(r'^a', files))
        args = tuple(map(self._read_data, (os.path.join(path, f) for f in arg_files)))

        kwarg_files = filter(r'^k', files)
        kwarg_files = {filename[1:]: os.path.join(path, filename) for filename in kwarg_files}
        kwargs = walk_values(self._read_data, kwarg_files)

        return args, kwargs

    def load_call_info(self, func, state):
        dirname = '%s/%s' % (func.__name__, state)
        return self._load_call_info(dirname)

    def cached(self, func):
        """
        A decorator for caching function calls
        """
        @wraps(func)
        def wrapper(*args, **kwargs):
            dirname, serialized_args, serialized_kwargs = self._call_info(func, args, kwargs)

            try:
                result = self._get(dirname)
            except CacheMiss:
                result = func(*args, **kwargs)
                self._set(dirname, serialized_args, serialized_kwargs, result)

            return result

        # def invalidate(*args, **kwargs):
        #     cache_key = key_func(func, args, kwargs)
        #     self.delete(cache_key)
        # wrapper.invalidate = invalidate

        return wrapper

    def checked(self, func=None, strict=True, subs=None):
        """
        Checks that function output doesn't change for same input.
        """
        # Allow using @checked without parentheses
        if callable(func):
            return self.checked()(func)

        subs = subs or {}

        def decorator(func):
            prepend = lambda s: '%s/%s' % (func.__name__, s)
            _subs = {prepend(k): prepend(v) for k, v in subs.items()}

            @wraps(func)
            def wrapper(*args, **kwargs):
                dirname, serialized_args, serialized_kwargs = self._call_info(func, args, kwargs)
                if dirname in _subs:
                    cprint('Using substition in check', 'green')
                    dirname = _subs[dirname]
                    args, kwargs = self._load_call_info(dirname)

                try:
                    print 'check', dirname
                    saved_result = self._get(dirname)
                    result = func(*args, **kwargs)
                except CacheMiss:
                    if strict:
                        cprint('No result for %s' % dirname, 'red')
                        try:
                            import ipdb; ipdb.set_trace()
                        except ImportError:
                            import pdb; pdb.set_trace()
                    else:
                        result = func(*args, **kwargs)
                        self._set(dirname, serialized_args, serialized_kwargs, result)
                else:
                    if not compare(saved_result, result):
                        cprint('Result change in %s' % dirname, 'red')
                        print explain_diff(saved_result, result)
                        # print ''
                        print 'hey'
                        try:
                            import ipdb; ipdb.set_trace()
                        except ImportError:
                            import pdb; pdb.set_trace()

                return result

            return wrapper
        return decorator

    def checked_call(self, func, state): #, strict=True):
        dirname = '%s/%s' % (func.__name__, state)
        print dirname
        args, kwargs = self._load_call_info(dirname)
        # self.checked(strict=strict)(func)(*args, **kwargs)

        try:
            saved_result = self._get(dirname)
            result = func(*args, **kwargs)
        except CacheMiss:
            raise LookupError('No recored call %s' % dirname, 'red')
        else:
            if not compare(saved_result, result):
                cprint('Result change in %s' % dirname, 'red')
                print explain_diff(saved_result, result)
                try:
                    import ipdb; ipdb.set_trace()
                except ImportError:
                    import pdb; pdb.set_trace()

    def liststates(self, func):
        path = os.path.join(self._path, func.__name__)
        return os.listdir(path)

    # @print_durations
    def _get(self, dirname):
        filename = os.path.join(self._path, dirname, 'out')
        try:
            # return self._read_data(filename)
            with open(filename, 'rb') as f:
                return pickle.load(f)
        except (IOError, OSError, EOFError, pickle.PickleError):
            raise CacheMiss

    # @print_durations
    def _set(self, dirname, serialized_args, serialized_kwargs, result):
        path = os.path.join(self._path, dirname)

        if not os.path.exists(path):
            os.makedirs(path)

        for i, value in enumerate(serialized_args):
            self._write_file(os.path.join(path, 'a%d' % i), value)
        for name, value in serialized_kwargs.items():
            self._write_file(os.path.join(path, 'k' + name), value)

        self._write_file(os.path.join(path, 'out'), serialize(result))

    # def _read_file(self, filename):
    #     with open(filename, 'rb') as f:
    #         return f.read()

    def _read_data(self, filename):
        with open(filename, 'rb') as f:
            return pickle.load(f)

    def _write_file(self, filename, data):
        # Use open with exclusive rights to prevent data corruption
        f = os.open(filename, os.O_EXCL | os.O_WRONLY | os.O_CREAT)
        try:
            os.write(f, data)
        finally:
            os.close(f)

    # def set(self, key, data, timeout=None):
    #     filename = self._key_to_filename(key)
    #     dirname = os.path.dirname(filename)

    #     if timeout is None:
    #         timeout = self._default_timeout

    #     try:
    #         if not os.path.exists(dirname):
    #             os.makedirs(dirname)

    #         # Use open with exclusive rights to prevent data corruption
    #         print 'w0', filename
    #         f = os.open(filename, os.O_EXCL | os.O_WRONLY | os.O_CREAT)
    #         try:
    #             print 'w1'
    #             os.write(f, pickle.dumps(data, pickle.HIGHEST_PROTOCOL))
    #             print 'w2'
    #         finally:
    #             os.close(f)

    #         # Set mtime to expire time
    #         os.utime(filename, (0, time.time() + timeout))
    #     except (IOError, OSError):
    #         pass

    def delete(self, fname):
        try:
            os.remove(fname)
            # Trying to remove directory in case it's empty
            dirname = os.path.dirname(fname)
            os.rmdir(dirname)
        except (IOError, OSError):
            pass

    # def _key_to_filename(self, key):
    #     """
    #     Returns a filename corresponding to cache key
    #     """
    #     return os.path.join(self._path, key)

cache = DebugCache()

def smart_str(value, max_len=20):
    s = str(value).strip()
    s = re.sub(r'\s+', ' ', s)
    if max_len and len(s) > max_len:
        s = s[:max_len-1] + '*'
    return s

def serialize(value):
    try:
        return pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
    except pickle.PickleError:
        return pickle.dumps(value)

def deserialize(value):
    return pickle.loads(value)

def md5hex(s):
    return hashlib.md5(s).hexdigest()

# @print_durations
def hash_args(serialized_args, serialized_kwargs):
    hash_sum = hashlib.md5()
    for ha in serialized_args:
        hash_sum.update(ha)
    for k, v in sorted(serialized_kwargs.items()):
        hash_sum.update(k)
        hash_sum.update(v)
    return hash_sum.hexdigest()
<|endoftext|>


<commit_before><commit_msg>Tools: [skip ci] move import of requests module into try/except block<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# (c) 2015, René Moser <mail@renemoser.net>
#
# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

try:
    from cs import CloudStack, CloudStackException, read_config
    has_lib_cs = True
except ImportError:
    has_lib_cs = False

class AnsibleCloudStack:

    def __init__(self, module):
        if not has_lib_cs:
            module.fail_json(msg="python library cs required: pip install cs")

        self.module = module
        self._connect()

        self.project = None
        self.ip_address = None
        self.zone = None
        self.vm = None
        self.os_type = None
        self.hypervisor = None

    def _connect(self):
        api_key = self.module.params.get('api_key')
        api_secret = self.module.params.get('secret_key')
        api_url = self.module.params.get('api_url')
        api_http_method = self.module.params.get('api_http_method')

        if api_key and api_secret and api_url:
            self.cs = CloudStack(
                endpoint=api_url,
                key=api_key,
                secret=api_secret,
                method=api_http_method
                )
        else:
            self.cs = CloudStack(**read_config())

    def _has_changed(self, want_dict, current_dict, only_keys=None):
        for key, value in want_dict.iteritems():

            # Optionally limit by a list of keys
            if only_keys and key not in only_keys:
                continue;

            if key in current_dict:

                # API returns string for int in some cases, just to make sure
                if isinstance(value, int):
                    current_dict[key] = int(current_dict[key])
                elif isinstance(value, str):
                    current_dict[key] = str(current_dict[key])

                # Only need to detect a singe change, not every item
                if value != current_dict[key]:
                    return True
        return False

    def _get_by_key(self, key=None, my_dict={}):
        if key:
            if key in my_dict:
                return my_dict[key]
            self.module.fail_json(msg="Something went wrong: %s not found" % key)
        return my_dict

    # TODO: for backward compatibility only, remove if not used anymore
    def get_project_id(self):
        return self.get_project(key='id')

    def get_project(self, key=None):
        if self.project:
            return self._get_by_key(key, self.project)

        project = self.module.params.get('project')
        if not project:
            return None

        projects = self.cs.listProjects(listall=True)
        if projects:
            for p in projects['project']:
                if project in [ p['name'], p['displaytext'], p['id'] ]:
                    self.project = p
                    return self._get_by_key(key, self.project)
        self.module.fail_json(msg="project '%s' not found" % project)

    # TODO: for backward compatibility only, remove if not used anymore
    def get_ip_address_id(self):
        return self.get_ip_address(key='id')

    def get_ip_address(self, key=None):
        if self.ip_address:
            return self._get_by_key(key, self.ip_address)

        ip_address = self.module.params.get('ip_address')
        if not ip_address:
            self.module.fail_json(msg="IP address param 'ip_address' is required")

        args = {}
        args['ipaddress'] = ip_address
        args['projectid'] = self.get_project(key='id')
        ip_addresses = self.cs.listPublicIpAddresses(**args)

        if not ip_addresses:
            self.module.fail_json(msg="IP address '%s' not found" % args['ipaddress'])

        self.ip_address = ip_addresses['publicipaddress'][0]
        return self._get_by_key(key, self.ip_address)

    # TODO: for backward compatibility only, remove if not used anymore
    def get_vm_id(self):
        return self.get_vm(key='id')

    def get_vm(self, key=None):
        if self.vm:
            return self._get_by_key(key, self.vm)

        vm = self.module.params.get('vm')
        if not vm:
            self.module.fail_json(msg="Virtual machine param 'vm' is required")

        args = {}
        args['projectid'] = self.get_project(key='id')
        vms = self.cs.listVirtualMachines(**args)
        if vms:
            for v in vms['virtualmachine']:
                if vm in [ v['name'], v['displayname'], v['id'] ]:
                    self.vm = v
                    return self._get_by_key(key, self.vm)
        self.module.fail_json(msg="Virtual machine '%s' not found" % vm)

    # TODO: for backward compatibility only, remove if not used anymore
    def get_zone_id(self):
        return self.get_zone(key='id')

    def get_zone(self, key=None):
        if self.zone:
            return self._get_by_key(key, self.zone)

        zone = self.module.params.get('zone')
        zones = self.cs.listZones()

        # use the first zone if no zone param given
        if not zone:
            self.zone = zones['zone'][0]
            return self._get_by_key(key, self.zone)

        if zones:
            for z in zones['zone']:
                if zone in [ z['name'], z['id'] ]:
                    self.zone = z
                    return self._get_by_key(key, self.zone)
        self.module.fail_json(msg="zone '%s' not found" % zone)

    # TODO: for backward compatibility only, remove if not used anymore
    def get_os_type_id(self):
        return self.get_os_type(key='id')

    def get_os_type(self, key=None):
        if self.os_type:
            return self._get_by_key(key, self.zone)

        os_type = self.module.params.get('os_type')
        if not os_type:
            return None

        os_types = self.cs.listOsTypes()
        if os_types:
            for o in os_types['ostype']:
                if os_type in [ o['description'], o['id'] ]:
                    self.os_type = o
                    return self._get_by_key(key, self.os_type)
        self.module.fail_json(msg="OS type '%s' not found" % os_type)

    def get_hypervisor(self):
        if self.hypervisor:
            return self.hypervisor

        hypervisor = self.module.params.get('hypervisor')
        hypervisors = self.cs.listHypervisors()

        # use the first hypervisor if no hypervisor param given
        if not hypervisor:
            self.hypervisor = hypervisors['hypervisor'][0]['name']
            return self.hypervisor

        for h in hypervisors['hypervisor']:
            if hypervisor.lower() == h['name'].lower():
                self.hypervisor = h['name']
                return self.hypervisor
        self.module.fail_json(msg="Hypervisor '%s' not found" % hypervisor)

    def _poll_job(self, job=None, key=None):
        if 'jobid' in job:
            while True:
                res = self.cs.queryAsyncJobResult(jobid=job['jobid'])
                if res['jobstatus'] != 0 and 'jobresult' in res:
                    if 'errortext' in res['jobresult']:
                        self.module.fail_json(msg="Failed: '%s'" % res['jobresult']['errortext'])
                    if key and key in res['jobresult']:
                        job = res['jobresult'][key]
                    break
                time.sleep(2)
        return job
<commit_msg>cloudstack: add method to to get infos of API<commit_after># -*- coding: utf-8 -*-
#
# (c) 2015, René Moser <mail@renemoser.net>
#
# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

try:
    from cs import CloudStack, CloudStackException, read_config
    has_lib_cs = True
except ImportError:
    has_lib_cs = False

class AnsibleCloudStack:

    def __init__(self, module):
        if not has_lib_cs:
            module.fail_json(msg="python library cs required: pip install cs")

        self.module = module
        self._connect()

        self.project = None
        self.ip_address = None
        self.zone = None
        self.vm = None
        self.os_type = None
        self.hypervisor = None
        self.capabilities = None

    def _connect(self):
        api_key = self.module.params.get('api_key')
        api_secret = self.module.params.get('secret_key')
        api_url = self.module.params.get('api_url')
        api_http_method = self.module.params.get('api_http_method')

        if api_key and api_secret and api_url:
            self.cs = CloudStack(
                endpoint=api_url,
                key=api_key,
                secret=api_secret,
                method=api_http_method
                )
        else:
            self.cs = CloudStack(**read_config())

    def _has_changed(self, want_dict, current_dict, only_keys=None):
        for key, value in want_dict.iteritems():

            # Optionally limit by a list of keys
            if only_keys and key not in only_keys:
                continue;

            if key in current_dict:

                # API returns string for int in some cases, just to make sure
                if isinstance(value, int):
                    current_dict[key] = int(current_dict[key])
                elif isinstance(value, str):
                    current_dict[key] = str(current_dict[key])

                # Only need to detect a singe change, not every item
                if value != current_dict[key]:
                    return True
        return False

    def _get_by_key(self, key=None, my_dict={}):
        if key:
            if key in my_dict:
                return my_dict[key]
            self.module.fail_json(msg="Something went wrong: %s not found" % key)
        return my_dict

    # TODO: for backward compatibility only, remove if not used anymore
    def get_project_id(self):
        return self.get_project(key='id')

    def get_project(self, key=None):
        if self.project:
            return self._get_by_key(key, self.project)

        project = self.module.params.get('project')
        if not project:
            return None

        projects = self.cs.listProjects(listall=True)
        if projects:
            for p in projects['project']:
                if project in [ p['name'], p['displaytext'], p['id'] ]:
                    self.project = p
                    return self._get_by_key(key, self.project)
        self.module.fail_json(msg="project '%s' not found" % project)

    # TODO: for backward compatibility only, remove if not used anymore
    def get_ip_address_id(self):
        return self.get_ip_address(key='id')

    def get_ip_address(self, key=None):
        if self.ip_address:
            return self._get_by_key(key, self.ip_address)

        ip_address = self.module.params.get('ip_address')
        if not ip_address:
            self.module.fail_json(msg="IP address param 'ip_address' is required")

        args = {}
        args['ipaddress'] = ip_address
        args['projectid'] = self.get_project(key='id')
        ip_addresses = self.cs.listPublicIpAddresses(**args)

        if not ip_addresses:
            self.module.fail_json(msg="IP address '%s' not found" % args['ipaddress'])

        self.ip_address = ip_addresses['publicipaddress'][0]
        return self._get_by_key(key, self.ip_address)

    # TODO: for backward compatibility only, remove if not used anymore
    def get_vm_id(self):
        return self.get_vm(key='id')

    def get_vm(self, key=None):
        if self.vm:
            return self._get_by_key(key, self.vm)

        vm = self.module.params.get('vm')
        if not vm:
            self.module.fail_json(msg="Virtual machine param 'vm' is required")

        args = {}
        args['projectid'] = self.get_project(key='id')
        vms = self.cs.listVirtualMachines(**args)
        if vms:
            for v in vms['virtualmachine']:
                if vm in [ v['name'], v['displayname'], v['id'] ]:
                    self.vm = v
                    return self._get_by_key(key, self.vm)
        self.module.fail_json(msg="Virtual machine '%s' not found" % vm)

    # TODO: for backward compatibility only, remove if not used anymore
    def get_zone_id(self):
        return self.get_zone(key='id')

    def get_zone(self, key=None):
        if self.zone:
            return self._get_by_key(key, self.zone)

        zone = self.module.params.get('zone')
        zones = self.cs.listZones()

        # use the first zone if no zone param given
        if not zone:
            self.zone = zones['zone'][0]
            return self._get_by_key(key, self.zone)

        if zones:
            for z in zones['zone']:
                if zone in [ z['name'], z['id'] ]:
                    self.zone = z
                    return self._get_by_key(key, self.zone)
        self.module.fail_json(msg="zone '%s' not found" % zone)

    # TODO: for backward compatibility only, remove if not used anymore
    def get_os_type_id(self):
        return self.get_os_type(key='id')

    def get_os_type(self, key=None):
        if self.os_type:
            return self._get_by_key(key, self.zone)

        os_type = self.module.params.get('os_type')
        if not os_type:
            return None

        os_types = self.cs.listOsTypes()
        if os_types:
            for o in os_types['ostype']:
                if os_type in [ o['description'], o['id'] ]:
                    self.os_type = o
                    return self._get_by_key(key, self.os_type)
        self.module.fail_json(msg="OS type '%s' not found" % os_type)

    def get_hypervisor(self):
        if self.hypervisor:
            return self.hypervisor

        hypervisor = self.module.params.get('hypervisor')
        hypervisors = self.cs.listHypervisors()

        # use the first hypervisor if no hypervisor param given
        if not hypervisor:
            self.hypervisor = hypervisors['hypervisor'][0]['name']
            return self.hypervisor

        for h in hypervisors['hypervisor']:
            if hypervisor.lower() == h['name'].lower():
                self.hypervisor = h['name']
                return self.hypervisor
        self.module.fail_json(msg="Hypervisor '%s' not found" % hypervisor)

    def get_capabilities(self, key=None):
        if self.capabilities:
            return self._get_by_key(key, self.capabilities)
        capabilities = self.cs.listCapabilities()
        self.capabilities = capabilities['capability']
        return self._get_by_key(key, self.capabilities)

    def _poll_job(self, job=None, key=None):
        if 'jobid' in job:
            while True:
                res = self.cs.queryAsyncJobResult(jobid=job['jobid'])
                if res['jobstatus'] != 0 and 'jobresult' in res:
                    if 'errortext' in res['jobresult']:
                        self.module.fail_json(msg="Failed: '%s'" % res['jobresult']['errortext'])
                    if key and key in res['jobresult']:
                        job = res['jobresult'][key]
                    break
                time.sleep(2)
        return job
<|endoftext|>


<commit_before>#
# pep257.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Aparajita Fishman
#
# Project: https://github.com/SublimeLinter/SublimeLinter-contrib-pep257
# License: MIT
#

"""Exports the Pep257 linter class."""

import os

# If possible, import pep257 and use it directly for better performance
try:
    from pep257 import check_source
except ImportError:
    check_source = None

from SublimeLinter.lint import highlight, PythonLinter, Registrar

class Pep257Meta(Registrar):

    """
    Metaclass for Pep257 that dynamically sets the 'cmd' attribute.

    If a linter can work both using an executable and built in code,
    the best way to deal with that is to set the cmd class attribute
    during class construction. This allows the linter to take advantage
    of the rest of the SublimeLinter machinery for everything but run().

    """

    def __init__(cls, name, bases, attrs):
        super().__init__(name, bases, attrs)

        # If pep257 could not be imported, use the executable
        if check_source is None:
            setattr(cls, 'cmd', ('pep257@python', '-'))
        else:
            setattr(cls, 'cmd', None)

class Pep257(PythonLinter, metaclass=Pep257Meta):

    """Provide an interface to the pep257 python script."""

    language = 'python'
    regex = r'^.+?:(?P<line>\d+):(?P<col>\d+): (?P<message>.+)'
    default_type = highlight.WARNING
    line_col_base = (1, 0)  # pep257 uses one-based line and zero-based column numbers

    def run(self, cmd, code):
        """Run pep257 on the source and return the output."""
        if check_source is None:
            return super().run(cmd, code)
        else:
            errors = check_source(code, os.path.basename(self.filename))
            return '\n'.join([str(e) for e in errors])
<commit_msg>Be sure to catch exceptions when checking directly.<commit_after>#
# pep257.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Aparajita Fishman
#
# Project: https://github.com/SublimeLinter/SublimeLinter-contrib-pep257
# License: MIT
#

"""Exports the Pep257 linter class."""

import os

# If possible, import pep257 and use it directly for better performance
try:
    from pep257 import check_source
except ImportError:
    check_source = None

from SublimeLinter.lint import highlight, PythonLinter, Registrar

class Pep257Meta(Registrar):

    """
    Metaclass for Pep257 that dynamically sets the 'cmd' attribute.

    If a linter can work both using an executable and built in code,
    the best way to deal with that is to set the cmd class attribute
    during class construction. This allows the linter to take advantage
    of the rest of the SublimeLinter machinery for everything but run().

    """

    def __init__(cls, name, bases, attrs):
        super().__init__(name, bases, attrs)

        # If pep257 could not be imported, use the executable
        if check_source is None:
            setattr(cls, 'cmd', ('pep257@python', '-'))
        else:
            setattr(cls, 'cmd', None)

class Pep257(PythonLinter, metaclass=Pep257Meta):

    """Provide an interface to the pep257 python script."""

    language = 'python'
    regex = r'^.+?:(?P<line>\d+):(?P<col>\d+): (?P<message>.+)'
    default_type = highlight.WARNING
    line_col_base = (1, 0)  # pep257 uses one-based line and zero-based column numbers

    def run(self, cmd, code):
        """Run pep257 on the source and return the output."""
        if check_source is None:
            return super().run(cmd, code)
        else:
            try:
                errors = check_source(code, os.path.basename(self.filename))
            except:
                errors = []

            return '\n'.join([str(e) for e in errors])
<|endoftext|>


<commit_before>"""
    wtforms.validators
    ~~~~~~~~~~~~~~~~~~

    TODO

    :copyright: 2007-2008 by James Crasta, Thomas Johansson.
    :license: MIT, see LICENSE.txt for details.
"""
import re

class ValidationError(ValueError):
    pass

def email(message=u'Invalid email address.'):
    def _email(form, field):
        if not re.match(r'^.+@[^.].*\.[a-z]{2,4}$', field.data, re.IGNORECASE):
            raise ValidationError(message)
    return _email

def equal_to(fieldname, message=u'Must be equal to %s'):
    def _equal_to(form, field):
        other = getattr(form, fieldname, None)
        if not other:
            raise ValidationError(u"Invalid field name '%s'" % fieldname)
        elif field.data != other.data:
            raise ValidationError(message % fieldname)
    return _equal_to

def ip_address(message=u'Invalid IP address.'):
    def _ip_address(form, field):
        if not re.match(r'^([0-9]{1,3}\.){3}[0-9]{1,3}$', field.data):
            raise ValidationError(message)
    return _ip_address

def is_checked(message=u'Field must tbe checked.'):
    def _is_checked(form, field):
        if not field.data:
            raise ValidationError(message)
    return _is_checked

def length(min=-1, max=-1, message=u'Field must be between %(min)i and %(max)i characters long.'):
    fmt_args = {'min': min, 'max': max}
    def _length(form, field):
        l = field.data and len(field.data) or 0
        if l < min or max != -1 and l > max:
            raise ValidationError(message % fmt_args)
    return _length

def not_empty(message=u'Field must not be empty.'):
    def _not_empty(form, field):
        if not field.data or not field.data.strip():
            raise ValidationError(message)
    return _not_empty

def url(allow_blank=False, message=u'Invalid URL.'):
    def _url(form, field):
        if allow_blank and not field.data:
            return
        if not re.match(r'[a-z]+://.*\.[a-z]{2,4}(\/.*)?', field.data, re.I):
            raise ValidationError(message)
    return _url

__all__ = ('ValidationError', 'email', 'equal_to', 'ip_address', 'is_checked', 'length', 'not_empty', 'url')
<commit_msg>Remove use of substitution in standard validators. It forced usage of the values with custom messages.<commit_after>"""
    wtforms.validators
    ~~~~~~~~~~~~~~~~~~

    TODO

    :copyright: 2007-2008 by James Crasta, Thomas Johansson.
    :license: MIT, see LICENSE.txt for details.
"""
import re

class ValidationError(ValueError):
    pass

def email(message=u'Invalid email address.'):
    def _email(form, field):
        if not re.match(r'^.+@[^.].*\.[a-z]{2,4}$', field.data, re.IGNORECASE):
            raise ValidationError(message)
    return _email

def equal_to(fieldname, message=None):
    if not message:
        message = u'Field must be equal to %s' % fieldname
    def _equal_to(form, field):
        other = getattr(form, fieldname, None)
        if not other:
            raise ValidationError(u"Invalid field name '%s'" % fieldname)
        elif field.data != other.data:
            raise ValidationError(message)
    return _equal_to

def ip_address(message=u'Invalid IP address.'):
    def _ip_address(form, field):
        if not re.match(r'^([0-9]{1,3}\.){3}[0-9]{1,3}$', field.data):
            raise ValidationError(message)
    return _ip_address

def is_checked(message=u'Field must tbe checked.'):
    def _is_checked(form, field):
        if not field.data:
            raise ValidationError(message)
    return _is_checked

def length(min=-1, max=-1, message=None):
    if not message:
        message = u'Field must be between %i and %i characters long.' % (min, max)
    def _length(form, field):
        l = field.data and len(field.data) or 0
        if l < min or max != -1 and l > max:
            raise ValidationError(message)
    return _length

def not_empty(message=u'Field must not be empty.'):
    def _not_empty(form, field):
        if not field.data or not field.data.strip():
            raise ValidationError(message)
    return _not_empty

def url(allow_blank=False, message=u'Invalid URL.'):
    def _url(form, field):
        if allow_blank and not field.data:
            return
        if not re.match(r'[a-z]+://.*\.[a-z]{2,4}(\/.*)?', field.data, re.I):
            raise ValidationError(message)
    return _url

__all__ = ('ValidationError', 'email', 'equal_to', 'ip_address', 'is_checked', 'length', 'not_empty', 'url')
<|endoftext|>