<commit_before>class NodeTree(object):
    """
    Tree of node objects, responsible for turning operation nodes into C code.
    """

    def __init__(self, node_root):

        if not isinstance(node_root, (ConcatNode, ExtractNode)):
            raise ValueError("only ConcatNodes or ExtractNodes can be tree roots")

        self.root = node_root
        self.created_vars = set()

    @staticmethod
    def _to_byte_idx(idx):
        return 4095 - idx / 8

    def _find_node(self, tree, node_cls):

        if isinstance(tree, node_cls):
            return tree

        if isinstance(tree, BinOpNode):
            res = self._find_node(tree.arg1, node_cls)
            if res is not None:
                return res
            res = self._find_node(tree.arg2, node_cls)

        if isinstance(tree, ReverseNode):
            return self._find_node(tree.arg, node_cls)

        if isinstance(tree, ExtractNode):
            return self._find_node(tree.arg, node_cls)

        if isinstance(tree, (BVSNode, BVVNode)):
            return None

    def to_c(self):
        """
        Convert a C expression.
        """

        c_code = None
        if isinstance(self.root, ConcatNode):
            c_code = self._concat_to_c()
        elif isinstance(self.root, ExtractNode):
            c_code = self._extract_to_c()
        else:
            assert False, "unrecognized node type %s as op" % self.root.__class__

        return c_code

    def _single_sided(self, op):
        """
        Check if a an AST contains the flag page on only one sided
        :param op: Node representing the operation to test
        :return: boolean whether symbolic data is only on a single side
        """

    def _concat_to_c(self):

        need_vars = [ ]
        statements = [ ]

        max_op_size = 0

        for size, op in self.root.operands:

            statement = None
            # we can get operations where flag data is on both sides of arithmetic operation
            # these are not always impossible to reverse (for example flag_data + flag_data)
            # TODO: but im too lazy to special case these at the moment
            if isinstance(op, BVVNode) or op._symbolic_sides() > 1:
                # read and throw away
                statements.append("blank_receive(0, {});".format(size / 8))
            else:
                max_op_size = max(max_op_size, size / 8)
                # find extract to determine which flag byte
                statements.append("receive(0, {}, {}, NULL);\n".format('root', size / 8))

                enode = self._find_node(op, ExtractNode)
                start_byte = NodeTree._to_byte_idx(enode.end_index)
                end_byte = NodeTree._to_byte_idx(enode.start_index) + 1

                statement = op.to_statement()
                if statement != "":
                    statements.append(statement + ";")

                for r_i, i in enumerate(range(start_byte, end_byte)):
                    statement = ""
                    if not i in self.created_vars:
                        statement += "uint8_t "
                        self.created_vars.add(i)
                    # hack! this will assume that if the operation size is larger than leaked bytes
                    # the leaked bytes will be on the `right` of the leaked int
                    statement += "flag_byte_%d = root[%d] & 0xff;" % (i, r_i + (size / 8) - (end_byte - start_byte))
                    statements.append(statement)

        statements = ["\nchar root[%d];" % max_op_size, "int flag = 0;"] + statements
        return '\n'.join(statements) + "\n" + self._concat_combine_bytes(need_vars)

    def _extract_to_c(self):

        # if it's an extract statement we already know it needs to have all the bytes

        statements = ["\nchar root[%d];" % (self.root.size / 8), "int flag = 0;"]
        statements.append("receive(0, {}, {}, NULL);".format('root', self.root.size / 8))

        statements.append(self.root.to_statement() + ";")
        for i in range(self.root.size / 8):
            statements.append("uint8_t flag_byte_%d = root[%d] & 0xff;" % (i, i))

        for i in range(min(self.root.size / 8, 4)):
            statements.append("flag |= flag_byte_%d << %d;" % (i, 24 - (i * 8)))
        return "\n".join(statements)

    def leaked_bytes(self):
        """
        Determine which bytes were leaked
        :returns: list of tuples of (byte_index, operation)
        """

        byte_list = [ ]
        if isinstance(self.root, ConcatNode):
            byte_list = self._concat_leaked_bytes()
        elif isinstance(self.root, ExtractNode):
            byte_list = self._extract_leaked_bytes()

        return byte_list

    def _concat_leaked_bytes(self):
        """
        Traverse tree and determine which byte indices of the flag page were leaked.
        :returns: list of tuples of (byte_index, operation)
        """

        lbytes = [ ]
        op = None # silence pylint
        for _, op in self.root.operands:

            node = self._find_node(op, ExtractNode)
            if node is not None:
                start_byte = NodeTree._to_byte_idx(node.end_index)
                end_byte = NodeTree._to_byte_idx(node.start_index)

                bs = range(start_byte, end_byte+1)

                lbytes  += map(lambda y: (y, op), bs)

        return lbytes

    def _extract_leaked_bytes(self):
        """
        Simple for extract, just do operations based off the indices
        """

        start_byte = NodeTree._to_byte_idx(self.root.end_index)
        end_byte = NodeTree._to_byte_idx(self.root.start_index)

        return map(lambda y: (y, self.root), [start_byte] + range(start_byte + 1, end_byte + 1))

    def _to_single_byte_vars(self, need_vars):

        created_singletons = set()
        for varset in self.created_vars:
            if len(varset) == 1:
                created_singletons.add(varset[0])

        statements = [ ]

        for varname, varset in need_vars:
            for i, var in enumerate(varset):

                if not var in created_singletons:
                    statement  = "int flag_byte_%d = " % var
                    statement += "(%s & (0xff << %d)) >> %d;" % (varname, 24 - i * 8, 24 - i * 8)
                    statements.append(statement)
                    created_singletons.add(var)

        return statements

    def _concat_combine_bytes(self, need_vars):

        #statements = self._to_single_byte_vars(need_vars)

        statements = [ ]
        ordered_bytes = dict(self.leaked_bytes())
        for current_byte in ordered_bytes:
            # check if the next four bytes leak the subsequent bytes
            current_byte_idx = current_byte

            next_consec = True

            for j in range(1,4):
                if not current_byte+j in ordered_bytes:
                    next_consec = False
                    break

            # try again
            if not next_consec:
                continue

            # found four consecutive bytes
            for j in range(0, 4):
                statements.append("flag |= " + "flag_byte_{} << {};".format(current_byte_idx + j, 24 - 8 * j))

            break

        if len(statements) == 0:
            raise ValueError("no consecutive four bytes")

        return '\n'.join(statements)

class Node(object):

    def __init__(self, size):
        self.size = size

    def _symbolic_sides(self):
        raise NotImplementedError("It is the responsibilty of subclasses to implement this method")

class UnOp(Node):

    def __init__(self, arg, size):
        super(UnOp, self).__init__(size)
        self.arg = arg

    def _symbolic_sides(self):
        return self.arg._symbolic_sides()

class BVVNode(UnOp):
    def __init__(self, arg, size):
        super(BVVNode, self).__init__(arg, size)

    def to_statement(self):
        return "{0:#x}".format(self.arg)

    def _symbolic_sides(self):
        return 0

class BVSNode(UnOp):
    def __init__(self, arg, size):
        super(BVSNode, self).__init__(arg, size)

    def to_statement(self):
        return self.arg

    def _symbolic_sides(self):
        return 1

class BinOpNode(Node):

    def __init__(self, op_str, arg1, arg2, size):
        super(BinOpNode, self).__init__(size)
        self.arg1 = arg1
        self.arg2 = arg2
        self.op_str = op_str

    def to_statement(self):
        a1_t = self.arg1.to_statement()
        a2_t = self.arg2.to_statement()
        return "{1}({0}, {2}, {3})".format(a1_t, self.op_str, a2_t, self.size / 8)

    def _symbolic_sides(self):
        return self.arg1._symbolic_sides() + self.arg2._symbolic_sides()

class AddNode(BinOpNode):

    def __init__(self, arg1, arg2, size):
        super(AddNode, self).__init__('add', arg1, arg2, size)

class SubNode(BinOpNode):

    def __init__(self, arg1, arg2, size):
        super(SubNode, self).__init__('sub', arg1, arg2, size)

class XorNode(BinOpNode):

    def __init__(self, arg1, arg2, size):
        super(XorNode, self).__init__('xor', arg1, arg2, size)

class AndNode(BinOpNode):
    def __init__(self, arg1, arg2, size):
        super(AndNode, self).__init__('and', arg1, arg2, size)

class ExtractNode(UnOp):

    def __init__(self, arg, start_index, end_index, size):
        super(ExtractNode, self).__init__(arg, size)
        self.start_index = start_index
        self.end_index = end_index

    def to_statement(self):
        """
        ExtractNodes are assumed to be top-level
        """

        a_t = self.arg.to_statement()

        if isinstance(self.arg, BVSNode):
            return ""

        return "{0}".format(a_t)

class ReverseNode(UnOp):

    def __init__(self, arg, size):
        super(ReverseNode, self).__init__(arg, size)

    def to_statement(self):
        a_t = self.arg.to_statement()
        return "reverse({0}, {1})".format(a_t, self.size / 8)

class ConcatNode(Node):

    def __init__(self, operands, size):
        super(ConcatNode, self).__init__(size)
        self.operands = operands

    def to_statement(self):
        raise NotImplementedError, "this should not be called"

    def _symbolic_sides(self):
        """symbolicness does not matter in this case"""
        return 0
<commit_msg>Remove unused variables and unused functions<commit_after>class NodeTree(object):
    """
    Tree of node objects, responsible for turning operation nodes into C code.
    """

    def __init__(self, node_root):

        if not isinstance(node_root, (ConcatNode, ExtractNode)):
            raise ValueError("only ConcatNodes or ExtractNodes can be tree roots")

        self.root = node_root
        self.created_vars = set()

    @staticmethod
    def _to_byte_idx(idx):
        return 4095 - idx / 8

    def _find_node(self, tree, node_cls):

        if isinstance(tree, node_cls):
            return tree

        if isinstance(tree, BinOpNode):
            res = self._find_node(tree.arg1, node_cls)
            if res is not None:
                return res
            res = self._find_node(tree.arg2, node_cls)

        if isinstance(tree, ReverseNode):
            return self._find_node(tree.arg, node_cls)

        if isinstance(tree, ExtractNode):
            return self._find_node(tree.arg, node_cls)

        if isinstance(tree, (BVSNode, BVVNode)):
            return None

    def to_c(self):
        """
        Convert a C expression.
        """

        c_code = None
        if isinstance(self.root, ConcatNode):
            c_code = self._concat_to_c()
        elif isinstance(self.root, ExtractNode):
            c_code = self._extract_to_c()
        else:
            assert False, "unrecognized node type %s as op" % self.root.__class__

        return c_code

    def _single_sided(self, op):
        """
        Check if a an AST contains the flag page on only one sided
        :param op: Node representing the operation to test
        :return: boolean whether symbolic data is only on a single side
        """

    def _concat_to_c(self):

        statements = [ ]

        max_op_size = 0

        for size, op in self.root.operands:

            statement = None
            # we can get operations where flag data is on both sides of arithmetic operation
            # these are not always impossible to reverse (for example flag_data + flag_data)
            # TODO: but im too lazy to special case these at the moment
            if isinstance(op, BVVNode) or op._symbolic_sides() > 1:
                # read and throw away
                statements.append("blank_receive(0, {});".format(size / 8))
            else:
                max_op_size = max(max_op_size, size / 8)
                # find extract to determine which flag byte
                statements.append("receive(0, {}, {}, NULL);\n".format('root', size / 8))

                enode = self._find_node(op, ExtractNode)
                start_byte = NodeTree._to_byte_idx(enode.end_index)
                end_byte = NodeTree._to_byte_idx(enode.start_index) + 1

                statement = op.to_statement()
                if statement != "":
                    statements.append(statement + ";")

                for r_i, i in enumerate(range(start_byte, end_byte)):
                    statement = ""
                    if not i in self.created_vars:
                        statement += "uint8_t "
                        self.created_vars.add(i)
                    # hack! this will assume that if the operation size is larger than leaked bytes
                    # the leaked bytes will be on the `right` of the leaked int
                    statement += "flag_byte_%d = root[%d] & 0xff;" % (i, r_i + (size / 8) - (end_byte - start_byte))
                    statements.append(statement)

        statements = ["\nchar root[%d];" % max_op_size, "int flag = 0;"] + statements
        return '\n'.join(statements) + "\n" + self._concat_combine_bytes()

    def _extract_to_c(self):

        # if it's an extract statement we already know it needs to have all the bytes

        statements = ["\nchar root[%d];" % (self.root.size / 8), "int flag = 0;"]
        statements.append("receive(0, {}, {}, NULL);".format('root', self.root.size / 8))

        statements.append(self.root.to_statement() + ";")
        for i in range(self.root.size / 8):
            statements.append("uint8_t flag_byte_%d = root[%d] & 0xff;" % (i, i))

        for i in range(min(self.root.size / 8, 4)):
            statements.append("flag |= flag_byte_%d << %d;" % (i, 24 - (i * 8)))
        return "\n".join(statements)

    def leaked_bytes(self):
        """
        Determine which bytes were leaked
        :returns: list of tuples of (byte_index, operation)
        """

        byte_list = [ ]
        if isinstance(self.root, ConcatNode):
            byte_list = self._concat_leaked_bytes()
        elif isinstance(self.root, ExtractNode):
            byte_list = self._extract_leaked_bytes()

        return byte_list

    def _concat_leaked_bytes(self):
        """
        Traverse tree and determine which byte indices of the flag page were leaked.
        :returns: list of tuples of (byte_index, operation)
        """

        lbytes = [ ]
        op = None # silence pylint
        for _, op in self.root.operands:

            node = self._find_node(op, ExtractNode)
            if node is not None:
                start_byte = NodeTree._to_byte_idx(node.end_index)
                end_byte = NodeTree._to_byte_idx(node.start_index)

                bs = range(start_byte, end_byte+1)

                lbytes  += map(lambda y: (y, op), bs)

        return lbytes

    def _extract_leaked_bytes(self):
        """
        Simple for extract, just do operations based off the indices
        """

        start_byte = NodeTree._to_byte_idx(self.root.end_index)
        end_byte = NodeTree._to_byte_idx(self.root.start_index)

        return map(lambda y: (y, self.root), [start_byte] + range(start_byte + 1, end_byte + 1))

    def _concat_combine_bytes(self):

        statements = [ ]
        ordered_bytes = dict(self.leaked_bytes())
        for current_byte in ordered_bytes:
            # check if the next four bytes leak the subsequent bytes
            current_byte_idx = current_byte

            next_consec = True

            for j in range(1,4):
                if not current_byte+j in ordered_bytes:
                    next_consec = False
                    break

            # try again
            if not next_consec:
                continue

            # found four consecutive bytes
            for j in range(0, 4):
                statements.append("flag |= " + "flag_byte_{} << {};".format(current_byte_idx + j, 24 - 8 * j))

            break

        if len(statements) == 0:
            raise ValueError("no consecutive four bytes")

        return '\n'.join(statements)

class Node(object):

    def __init__(self, size):
        self.size = size

    def _symbolic_sides(self):
        raise NotImplementedError("It is the responsibilty of subclasses to implement this method")

class UnOp(Node):

    def __init__(self, arg, size):
        super(UnOp, self).__init__(size)
        self.arg = arg

    def _symbolic_sides(self):
        return self.arg._symbolic_sides()

class BVVNode(UnOp):
    def __init__(self, arg, size):
        super(BVVNode, self).__init__(arg, size)

    def to_statement(self):
        return "{0:#x}".format(self.arg)

    def _symbolic_sides(self):
        return 0

class BVSNode(UnOp):
    def __init__(self, arg, size):
        super(BVSNode, self).__init__(arg, size)

    def to_statement(self):
        return self.arg

    def _symbolic_sides(self):
        return 1

class BinOpNode(Node):

    def __init__(self, op_str, arg1, arg2, size):
        super(BinOpNode, self).__init__(size)
        self.arg1 = arg1
        self.arg2 = arg2
        self.op_str = op_str

    def to_statement(self):
        a1_t = self.arg1.to_statement()
        a2_t = self.arg2.to_statement()
        return "{1}({0}, {2}, {3})".format(a1_t, self.op_str, a2_t, self.size / 8)

    def _symbolic_sides(self):
        return self.arg1._symbolic_sides() + self.arg2._symbolic_sides()

class AddNode(BinOpNode):

    def __init__(self, arg1, arg2, size):
        super(AddNode, self).__init__('add', arg1, arg2, size)

class SubNode(BinOpNode):

    def __init__(self, arg1, arg2, size):
        super(SubNode, self).__init__('sub', arg1, arg2, size)

class XorNode(BinOpNode):

    def __init__(self, arg1, arg2, size):
        super(XorNode, self).__init__('xor', arg1, arg2, size)

class AndNode(BinOpNode):
    def __init__(self, arg1, arg2, size):
        super(AndNode, self).__init__('and', arg1, arg2, size)

class ExtractNode(UnOp):

    def __init__(self, arg, start_index, end_index, size):
        super(ExtractNode, self).__init__(arg, size)
        self.start_index = start_index
        self.end_index = end_index

    def to_statement(self):
        """
        ExtractNodes are assumed to be top-level
        """

        a_t = self.arg.to_statement()

        if isinstance(self.arg, BVSNode):
            return ""

        return "{0}".format(a_t)

class ReverseNode(UnOp):

    def __init__(self, arg, size):
        super(ReverseNode, self).__init__(arg, size)

    def to_statement(self):
        a_t = self.arg.to_statement()
        return "reverse({0}, {1})".format(a_t, self.size / 8)

class ConcatNode(Node):

    def __init__(self, operands, size):
        super(ConcatNode, self).__init__(size)
        self.operands = operands

    def to_statement(self):
        raise NotImplementedError, "this should not be called"

    def _symbolic_sides(self):
        """symbolicness does not matter in this case"""
        return 0
<|endoftext|>


<commit_before>from __future__ import division
from functools import partial

import numpy as np
from scipy.special import gamma, gammaln

from .utils import lazy_range, izip, iteritems
from .mp_utils import progress
from .knn_search import knn_search

################################################################################
### Estimators of various divergences based on nearest-neighbor distances.
#
# The standard interface for these functions is:
#
# Function attributes:
#
#   needs_alpha: whether this function needs an alpha parameter. Default false.
#
#   self_value: The value that this function should take when comparing a
#               sample to itself: either a scalar constant or None (the
#               default), in which case the function is still called with
#               rhos = nus.
#
# Arguments:
#
#   alphas (if needs_alpha; array-like, scalar or 1d): the alpha values to use
#
#   Ks (array-like, scalar or 1d): the K values used
#
#   num_q (scalar): the number of points in the sample from q
#
#   dim (scalar): the dimension of the feature space
#
#   rhos: an array of within-bag nearest neighbor distances for a sample from p.
#         rhos[i, j] should be the distance from the ith sample from p to its
#         Ks[j]'th neighbor in the same sample. Shape: (num_p, num_Ks).
#   nus: an array of nearest neighbor distances from samples from other dists.
#        nus[i, j] should be the distance from the ith sample from p to its
#        Ks[j]'th neighbor in the sample from q. Shape: (num_p, num_Ks).
#
# Returns an array of divergence estimates. If needs_alpha, should be of shape
# (num_alphas, num_Ks); otherwise, of shape (num_Ks,).

def linear(Ks, dim, num_q, rhos, nus):
    r'''
    Estimates the linear inner product \int p q between two distributions,
    based on kNN distances.
    '''
    return _get_linear(Ks, dim)(num_q, rhos, nus)

def _get_linear(Ks, dim):
    # Estimated with alpha=0, beta=1:
    #   B_{k,d,0,1} = (k - 1) / pi^(dim/2) * gamma(dim/2 + 1)
    #   (using gamma(k) / gamma(k - 1) = k - 1)
    Ks = np.reshape(Ks, (-1,))
    Bs = (Ks - 1) / np.pi ** (dim / 2) * gamma(dim / 2 + 1)  # shape (num_Ks,)
    return partial(_linear, Bs, dim)

def _linear(Bs, dim, num_q, rhos, nus):
    # and the rest of the estimator is
    #   B / m * mean(nu ^ -dim)
    return Bs / num_q * np.mean(nus ** (-dim), axis=0)
linear.self_value = None  # have to execute it
linear.needs_alpha = False
linear.chooser_fn = _get_linear

def kl(Ks, dim, num_q, rhos, nus):
    r'''
    Estimate the KL divergence between distributions:
        \int p(x) \log (p(x) / q(x))
    using the kNN-based estimator (5) of
        Qing Wang, Sanjeev R Kulkarni, and Sergio Verdu (2009).
        Divergence Estimation for Multidimensional Densities Via
        k-Nearest-Neighbor Distances.
        IEEE Transactions on Information Theory.
        http://www.ee.princeton.edu/~verdu/reprints/WanKulVer.May2009.pdf
    which is:
        d * 1/n \sum \log (nu_k(i) / rho_k(i)) + log(m / (n - 1))

    Enforces KL >= 0.

    Returns an array of shape (num_Ks,).
    '''
    est = dim * np.mean(np.log(nus) - np.log(rhos), axis=0)
    est += np.log(num_q / (rhos.shape[0] - 1))
    np.maximum(est, 0, out=est)
    return est
kl.self_value = 0
kl.needs_alpha = False

def alpha_div(alphas, Ks, dim, num_q, rhos, nus):
    r'''
    Estimate the alpha divergence between distributions:
        \int p^\alpha q^(1-\alpha)
    based on kNN distances.

    Used in Renyi, Hellinger, Bhattacharyya, Tsallis divergences.

    Enforces that estimates are >= 0.

    Returns divergence estimates with shape (num_alphas, num_Ks).
    '''
    return _get_alpha_div(alphas, Ks)(num_q, dim, rhos, nus)

def _get_alpha_div(alphas, Ks, dim):
    alphas = np.reshape(alphas, (-1, 1))
    Ks = np.reshape(Ks, (1, -1))

    omas = 1 - alphas

    # We're estimating with alpha = alpha-1, beta = 1-alpha.
    # B constant in front:
    #   estimator's alpha = -beta, so volume of unit ball cancels out
    #   and then ratio of gamma functions
    Bs = np.exp(gammaln(Ks) * 2 - gammaln(Ks + omas) - gammaln(Ks - omas))

    return partial(_alpha_div, omas, Bs, dim)

def _alpha_div(omas, Bs, dim, num_q, rhos, nus):
    N = rhos.shape[0]

    # the actual main estimate:
    #   rho^(- dim * est alpha) nu^(- dim * est beta)
    #   = (rho / nu) ^ (dim * (1 - alpha))
    # do some reshaping trickery to get broadcasting right
    estimates = (rhos / nus)[:, np.newaxis, :]
    estimates = estimates ** (dim * omas.reshape(1, -1, 1))
    estimates = np.mean(estimates, axis=0)  # shape (n_alphas, n_Ks)

    estimates *= Bs

    # factors based on the sizes:
    #   1 / [ (n-1)^(est alpha) * m^(est beta) ] = ((n-1) / m) ^ (1 - alpha)
    estimates *= ((N - 1) / num_q) ** omas

    np.maximum(estimates, 0, out=estimates)
    return estimates

alpha_div.self_value = 1
alpha_div.needs_alpha = True
alpha_div.chooser_fn = _get_alpha_div

################################################################################

def _estimate_cross_divs(features, indices, rhos,
                         mask, funcs, Ks, specs, n_meta_only,
                         progressbar):
    n_bags = len(features)
    dim = features.dim
    max_K = np.max(Ks)

    outputs = np.empty((n_bags, n_bags, len(specs) + n_meta_only, len(Ks)),
                       dtype=np.float32)
    outputs.fill(np.nan)

    # TODO: should just call functions that need self up here with rhos
    #       instead of computing nus and then throwing them out below
    any_run_self = False
    all_bags = lazy_range(n_bags)
    for func, info in iteritems(funcs):
        self_val = getattr(func, 'self_value', None)
        if self_val is not None:
            pos = np.reshape(info.pos, (-1, 1))
            outputs[all_bags, all_bags, pos, :] = self_val
        else:
            any_run_self = True

    indices_loop = progress()(indices) if progressbar else indices
    for i, index in enumerate(indices_loop):
        # Loop over rows of the output array.
        #
        # We want to search from most(?) of the other bags to this one, as
        # determined by mask and to avoid repeating nus.
        #
        # But we don't want to waste memory copying almost all of the features.
        #
        # So instead we'll run a separate NN search for each contiguous
        # subarray of the features. If they're too small, of course, this hurts
        # the parallelizability.
        #
        # TODO: is there a better scheme than this? use a custom version of
        #       nanoflann or something?
        #
        # TODO: Is cythonning this file/this function worth it?

        num_q = features._n_pts[i]

        # make a boolean array of whether we want to do the ith bag
        do_bag = mask[i]
        if not any_run_self:
            do_bag = do_bag.copy()
            do_bag[i] = False

        # loop over contiguous sections where do_bag is True
        change_pts = np.hstack([0, np.diff(do_bag).nonzero()[0] + 1, n_bags])
        s = 0 if do_bag[0] else 1
        for start, end in izip(change_pts[s::2], change_pts[s+1::2]):
            boundaries = features._boundaries[start:end+1]
            feats = features._features[boundaries[0]:boundaries[-1]]

            # find the nearest neighbors in features[i] from each of these bags
            neighbors = knn_search(max_K, feats, index=index)[:, Ks - 1]

            # TODO: parallelize this bit?
            for j in lazy_range(start, end):
                rho = rhos[j]
                nu = neighbors[boundaries[j]:boundaries[j+1]]

                if i == j:
                    for func, info in iteritems(funcs):
                        if getattr(func, 'self_value', None) is None:
                            outputs[j, i, info.pos, :] = func(num_q, rho, rho)
                            # otherwise, already set it above
                else:
                    for func, info in iteritems(funcs):
                        outputs[j, i, info.pos, :] = func(num_q, rho, nu)
    return outputs
<commit_msg>remove dead code<commit_after>from __future__ import division
from functools import partial

import numpy as np
from scipy.special import gamma, gammaln

from .utils import lazy_range, izip, iteritems
from .mp_utils import progress
from .knn_search import knn_search

################################################################################
### Estimators of various divergences based on nearest-neighbor distances.
#
# The standard interface for these functions is:
#
# Function attributes:
#
#   needs_alpha: whether this function needs an alpha parameter. Default false.
#
#   self_value: The value that this function should take when comparing a
#               sample to itself: either a scalar constant or None (the
#               default), in which case the function is still called with
#               rhos = nus.
#
# Arguments:
#
#   alphas (if needs_alpha; array-like, scalar or 1d): the alpha values to use
#
#   Ks (array-like, scalar or 1d): the K values used
#
#   num_q (scalar): the number of points in the sample from q
#
#   dim (scalar): the dimension of the feature space
#
#   rhos: an array of within-bag nearest neighbor distances for a sample from p.
#         rhos[i, j] should be the distance from the ith sample from p to its
#         Ks[j]'th neighbor in the same sample. Shape: (num_p, num_Ks).
#   nus: an array of nearest neighbor distances from samples from other dists.
#        nus[i, j] should be the distance from the ith sample from p to its
#        Ks[j]'th neighbor in the sample from q. Shape: (num_p, num_Ks).
#
# Returns an array of divergence estimates. If needs_alpha, should be of shape
# (num_alphas, num_Ks); otherwise, of shape (num_Ks,).

def linear(Ks, dim, num_q, rhos, nus):
    r'''
    Estimates the linear inner product \int p q between two distributions,
    based on kNN distances.
    '''
    return _get_linear(Ks, dim)(num_q, rhos, nus)

def _get_linear(Ks, dim):
    # Estimated with alpha=0, beta=1:
    #   B_{k,d,0,1} = (k - 1) / pi^(dim/2) * gamma(dim/2 + 1)
    #   (using gamma(k) / gamma(k - 1) = k - 1)
    Ks = np.reshape(Ks, (-1,))
    Bs = (Ks - 1) / np.pi ** (dim / 2) * gamma(dim / 2 + 1)  # shape (num_Ks,)
    return partial(_linear, Bs, dim)

def _linear(Bs, dim, num_q, rhos, nus):
    # and the rest of the estimator is
    #   B / m * mean(nu ^ -dim)
    return Bs / num_q * np.mean(nus ** (-dim), axis=0)
linear.self_value = None  # have to execute it
linear.needs_alpha = False
linear.chooser_fn = _get_linear

def kl(Ks, dim, num_q, rhos, nus):
    r'''
    Estimate the KL divergence between distributions:
        \int p(x) \log (p(x) / q(x))
    using the kNN-based estimator (5) of
        Qing Wang, Sanjeev R Kulkarni, and Sergio Verdu (2009).
        Divergence Estimation for Multidimensional Densities Via
        k-Nearest-Neighbor Distances.
        IEEE Transactions on Information Theory.
        http://www.ee.princeton.edu/~verdu/reprints/WanKulVer.May2009.pdf
    which is:
        d * 1/n \sum \log (nu_k(i) / rho_k(i)) + log(m / (n - 1))

    Enforces KL >= 0.

    Returns an array of shape (num_Ks,).
    '''
    est = dim * np.mean(np.log(nus) - np.log(rhos), axis=0)
    est += np.log(num_q / (rhos.shape[0] - 1))
    np.maximum(est, 0, out=est)
    return est
kl.self_value = 0
kl.needs_alpha = False

def alpha_div(alphas, Ks, dim, num_q, rhos, nus):
    r'''
    Estimate the alpha divergence between distributions:
        \int p^\alpha q^(1-\alpha)
    based on kNN distances.

    Used in Renyi, Hellinger, Bhattacharyya, Tsallis divergences.

    Enforces that estimates are >= 0.

    Returns divergence estimates with shape (num_alphas, num_Ks).
    '''
    return _get_alpha_div(alphas, Ks)(num_q, dim, rhos, nus)

def _get_alpha_div(alphas, Ks, dim):
    alphas = np.reshape(alphas, (-1, 1))
    Ks = np.reshape(Ks, (1, -1))

    omas = 1 - alphas

    # We're estimating with alpha = alpha-1, beta = 1-alpha.
    # B constant in front:
    #   estimator's alpha = -beta, so volume of unit ball cancels out
    #   and then ratio of gamma functions
    Bs = np.exp(gammaln(Ks) * 2 - gammaln(Ks + omas) - gammaln(Ks - omas))

    return partial(_alpha_div, omas, Bs, dim)

def _alpha_div(omas, Bs, dim, num_q, rhos, nus):
    N = rhos.shape[0]

    # the actual main estimate:
    #   rho^(- dim * est alpha) nu^(- dim * est beta)
    #   = (rho / nu) ^ (dim * (1 - alpha))
    # do some reshaping trickery to get broadcasting right
    estimates = (rhos / nus)[:, np.newaxis, :]
    estimates = estimates ** (dim * omas.reshape(1, -1, 1))
    estimates = np.mean(estimates, axis=0)  # shape (n_alphas, n_Ks)

    estimates *= Bs

    # factors based on the sizes:
    #   1 / [ (n-1)^(est alpha) * m^(est beta) ] = ((n-1) / m) ^ (1 - alpha)
    estimates *= ((N - 1) / num_q) ** omas

    np.maximum(estimates, 0, out=estimates)
    return estimates

alpha_div.self_value = 1
alpha_div.needs_alpha = True
alpha_div.chooser_fn = _get_alpha_div

################################################################################

def _estimate_cross_divs(features, indices, rhos,
                         mask, funcs, Ks, specs, n_meta_only,
                         progressbar):
    n_bags = len(features)
    max_K = np.max(Ks)

    outputs = np.empty((n_bags, n_bags, len(specs) + n_meta_only, len(Ks)),
                       dtype=np.float32)
    outputs.fill(np.nan)

    # TODO: should just call functions that need self up here with rhos
    #       instead of computing nus and then throwing them out below
    any_run_self = False
    all_bags = lazy_range(n_bags)
    for func, info in iteritems(funcs):
        self_val = getattr(func, 'self_value', None)
        if self_val is not None:
            pos = np.reshape(info.pos, (-1, 1))
            outputs[all_bags, all_bags, pos, :] = self_val
        else:
            any_run_self = True

    indices_loop = progress()(indices) if progressbar else indices
    for i, index in enumerate(indices_loop):
        # Loop over rows of the output array.
        #
        # We want to search from most(?) of the other bags to this one, as
        # determined by mask and to avoid repeating nus.
        #
        # But we don't want to waste memory copying almost all of the features.
        #
        # So instead we'll run a separate NN search for each contiguous
        # subarray of the features. If they're too small, of course, this hurts
        # the parallelizability.
        #
        # TODO: is there a better scheme than this? use a custom version of
        #       nanoflann or something?
        #
        # TODO: Is cythonning this file/this function worth it?

        num_q = features._n_pts[i]

        # make a boolean array of whether we want to do the ith bag
        do_bag = mask[i]
        if not any_run_self:
            do_bag = do_bag.copy()
            do_bag[i] = False

        # loop over contiguous sections where do_bag is True
        change_pts = np.hstack([0, np.diff(do_bag).nonzero()[0] + 1, n_bags])
        s = 0 if do_bag[0] else 1
        for start, end in izip(change_pts[s::2], change_pts[s+1::2]):
            boundaries = features._boundaries[start:end+1]
            feats = features._features[boundaries[0]:boundaries[-1]]

            # find the nearest neighbors in features[i] from each of these bags
            neighbors = knn_search(max_K, feats, index=index)[:, Ks - 1]

            # TODO: parallelize this bit?
            for j in lazy_range(start, end):
                rho = rhos[j]
                nu = neighbors[boundaries[j]:boundaries[j+1]]

                if i == j:
                    for func, info in iteritems(funcs):
                        if getattr(func, 'self_value', None) is None:
                            outputs[j, i, info.pos, :] = func(num_q, rho, rho)
                            # otherwise, already set it above
                else:
                    for func, info in iteritems(funcs):
                        outputs[j, i, info.pos, :] = func(num_q, rho, nu)
    return outputs
<|endoftext|>


<commit_before>import os

# python 2.4 does not have os.SEEK_*
try:
    os.SEEK_SET
except AttributeError:
    os.SEEK_SET, os.SEEK_CUR, os.SEEK_END = range(3)

import logging

log = logging.getLogger('flvlib')
log.setLevel(logging.NOTSET)

handler = logging.StreamHandler()
handler.setLevel(logging.NOTSET)

formatter = logging.Formatter("%(levelname)-7s %(name)-20s "
                              "%(message)s (%(pathname)s:%(lineno)d)")
handler.setFormatter(formatter)

log.addHandler(handler)

__version__ = (0, 1, 6)
__versionstr__ = '.'.join([str(n) for n in __version__])
<commit_msg>Back to development release 0.1.6.1<commit_after>import os

# python 2.4 does not have os.SEEK_*
try:
    os.SEEK_SET
except AttributeError:
    os.SEEK_SET, os.SEEK_CUR, os.SEEK_END = range(3)

import logging

log = logging.getLogger('flvlib')
log.setLevel(logging.NOTSET)

handler = logging.StreamHandler()
handler.setLevel(logging.NOTSET)

formatter = logging.Formatter("%(levelname)-7s %(name)-20s "
                              "%(message)s (%(pathname)s:%(lineno)d)")
handler.setFormatter(formatter)

log.addHandler(handler)

__version__ = (0, 1, 6, 1)
__versionstr__ = '.'.join([str(n) for n in __version__])
<|endoftext|>


<commit_before>'''This is a unit test of the Actor system.  It creates a crazy-quilt
of actors which route a request through the network based on the
request type (addEvery, addOdd, addEven, addFives, addSevens, etc.).
The intent is to ensure that various paths using different passed
ActorAddresses work, and that multiple request can be injected into
the system at once and still be handled correctly.'''

import sys
from thespian.actors import *
from thespian.test import *
from datetime import timedelta

max_wait=timedelta(seconds=20)

class addEvery(object):
    "Message causing the enclosed value to be incremented by one by each Actor."
    def __init__(self, value=0, asker=None, actorList=[]):
        self.value = value
        self.asker = asker
        self.step = 1
        self.actorList = actorList

class DropResults(Actor):
    def receiveMessage(self, msg, sender):
        pass

class AddOne(Actor):
    "This actor adds one to the input value and then passes it to the next actor."
    def receiveMessage(self, msg, sender):
#        if isinstance(msg, ActorAddress):
#            self.next_adder = msg   # next actor
        if isinstance(msg, addEvery):
            if not msg.asker: msg.asker = sender  # start of chain, remember asker
            msg.value = msg.value + 1
            if msg.actorList:
                next_addr = msg.actorList.pop()
                self.send(next_addr, msg)
            else:
                self.send(msg.asker, msg)

class AddTen(Actor):
    def __init__(self):
        self.next_adder = {}
        self.adders = None
    def receiveMessage(self, msg, sender):
        if not self.adders:
            # n.b. would prefer to do this initialization in __init__, but that's not supported ATM.  - KWQ 2013.10.30
            # Create 10 AddOne Actors
            self.adders = [ self.createActor(AddOne) for x in range(10) ]
        if isinstance(msg, type((1,2))):
            self.next_adder[msg[0]] = msg[1]
        elif isinstance(msg, addEvery):
            if not msg.asker: msg.asker = sender  # start of chain, remember asker
            if hasattr(msg, 'parentmsg'):
                # completion chain is returning here
                parmsg = msg.parentmsg
                parmsg.value = msg.value
                self.send(self.next_adder.get(parmsg.step, parmsg.asker), parmsg)
            else:
                # start of sub-adder chain
                submsg = addEvery(msg.value, actorList = self.adders[1:])
                submsg.parentmsg = msg
                self.send(self.adders[0], submsg)
        elif msg == "Names of Children?":
            self.send(sender, self.adders)

class TestFuncActorAdder(object):

    def test00_ready(self, asys):
        pass

    def test01_TenEvery(self, asys):
        unstable_test(asys, 'multiprocQueueBase')
        addOnes = [ asys.createActor(AddOne) for x in range(10) ]
        for (frmA, toA) in zip(addOnes, addOnes[1:]):
            asys.tell(frmA, (1, toA))
        sum = asys.ask(addOnes[0], addEvery(actorList = addOnes[1:]), max_wait)
        assert sum.value == 10
        sum2 = asys.ask(addOnes[0], addEvery(actorList = addOnes[1:]), max_wait)
        assert sum2.value == 10

    def test02_TenEvens(self, asys):
        unstable_test(asys, 'multiprocQueueBase')
        if 'pypy' in sys.executable:
            pytest.skip("Pypy adder processes will consume all available system"
                        " memory for no good reason")
        addOnes = [ asys.createActor(AddOne) for x in range(10) ]
        for (frmA, toA) in zip(addOnes, addOnes[1:]):
            asys.tell(frmA, (1, toA))
        for (frmA, toA) in zip(addOnes[0:], addOnes[2:]):
            asys.tell(frmA, (2, toA))
        sum = asys.ask(addOnes[0], addEvery(actorList = addOnes[1:]), max_wait)
        assert sum.value == 10
        sum2 = asys.ask(addOnes[0],
                        addEvery(actorList = [addOnes[I]
                                              for I in range(2, len(addOnes),2)]),
                        max_wait)
        assert sum2.value == 5
        sum3 = asys.ask(addOnes[1],
                        addEvery(actorList = [addOnes[I]
                                              for I in range(3, len(addOnes),2)]),
                        max_wait)
        assert sum3.value == 5
        sum4 = asys.ask(addOnes[4],
                        addEvery(actorList = [addOnes[I]
                                              for I in range(6, len(addOnes),2)]),
                        max_wait)
        assert sum4.value == 3

    def test03_LotsOfActorsEvery(self, asys):
        unstable_test(asys, 'multiprocQueueBase')
        if 'pypy' in sys.executable:
            pytest.skip("Pypy adder processes will consume all available system"
                        " memory for no good reason")
        addOnes = [ asys.createActor(AddOne) for x in range(50) ]
        for (frmA, toA) in zip(addOnes, addOnes[1:]):
            asys.tell(frmA, (1, toA))
        sum = asys.ask(addOnes[0], addEvery(actorList = addOnes[1:]), max_wait)
        assert sum.value == 50
        sum2 = asys.ask(addOnes[0], addEvery(actorList = addOnes[1:]), max_wait)
        assert sum2.value == 50
        for killA in addOnes:
            asys.tell(killA, ActorExitRequest())

    def test04_LotsOfActorsEveryTen(self, asys):
        unstable_test(asys, 'multiprocQueueBase')
        if 'pypy' in sys.executable:
            pytest.skip("Pypy adder processes will consume all available system"
                        " memory for no good reason")
        # Create a set of addTens
        addTens = [ asys.createActor(AddTen) for x in range(0, 50, 10) ]
        # Point each addTens to the next one to make a chain.  Each
        # addTens will automatically make 10 addOnes children.
        for (frmA, toA) in zip(addTens, addTens[1:]):
            asys.tell(frmA, (1, toA))

        sum = asys.ask(addTens[0], addEvery(), max_wait)
        assert sum.value == 50
        sum2 = asys.ask(addTens[0], addEvery(), max_wait)
        assert sum2.value == 50
        for killA in addTens:
            asys.tell(killA, ActorExitRequest())

    def test05_UniqueAddresses(self, asys):
        unstable_test(asys, 'multiprocQueueBase')
        if 'pypy' in sys.executable:
            pytest.skip("Pypy adder processes will consume all available system"
                        " memory for no good reason")
        addTen = asys.createActor(AddTen)
        children = asys.ask(addTen, "Names of Children?", max_wait)
        uniqueAddresses = [addTen]
        for each in children:
            assert each not in uniqueAddresses
            uniqueAddresses.append(each)
        assert 11 == len(uniqueAddresses)
        #Not needed: asys.tell(addTen, ActorExitRequest())

    def test06_TenEveryTen(self, asys):
        unstable_test(asys, 'multiprocQueueBase')
        if 'pypy' in sys.executable:
            pytest.skip("Pypy adder processes will consume all available system"
                        " memory for no good reason")
        addTen = asys.createActor(AddTen)
        sum = asys.ask(addTen, addEvery(), max_wait)
        assert sum.value == 10

    def test07_LotsOfActorsEveryTenWithBackground(self, asys):
        unstable_test(asys, 'multiprocQueueBase')
        if 'pypy' in sys.executable:
            pytest.skip("Pypy adder processes will consume all available system"
                        " memory for no good reason")
        addTens = [ asys.createActor(AddTen) for x in range(0, 50, 10) ]
        for (frmA, toA) in zip(addTens, addTens[1:]):
            asys.tell(frmA, (1, toA))
        drop = asys.createActor(DropResults)
        asys.tell(drop, addEvery)  # verify it can be used
        import random
        for num in range(random.randint(0, 100)):
            start = random.choice(addTens)
            asys.tell(start, addEvery(asker=drop))  # drop results
        asys.tell(addTens[-1], addEvery(asker=drop))  # drop results
        sum = asys.ask(addTens[0], addEvery(), max_wait)
        assert sum.value == 50
        for killA in addTens:
            asys.tell(killA, ActorExitRequest())
<commit_msg>Remove old debug comment in actorAdder test.<commit_after>'''This is a unit test of the Actor system.  It creates a crazy-quilt
of actors which route a request through the network based on the
request type (addEvery, addOdd, addEven, addFives, addSevens, etc.).
The intent is to ensure that various paths using different passed
ActorAddresses work, and that multiple request can be injected into
the system at once and still be handled correctly.'''

import sys
from thespian.actors import *
from thespian.test import *
from datetime import timedelta

max_wait=timedelta(seconds=20)

class addEvery(object):
    "Message causing the enclosed value to be incremented by one by each Actor."
    def __init__(self, value=0, asker=None, actorList=[]):
        self.value = value
        self.asker = asker
        self.step = 1
        self.actorList = actorList

class DropResults(Actor):
    def receiveMessage(self, msg, sender):
        pass

class AddOne(Actor):
    "This actor adds one to the input value and then passes it to the next actor."
    def receiveMessage(self, msg, sender):
#        if isinstance(msg, ActorAddress):
#            self.next_adder = msg   # next actor
        if isinstance(msg, addEvery):
            if not msg.asker: msg.asker = sender  # start of chain, remember asker
            msg.value = msg.value + 1
            if msg.actorList:
                next_addr = msg.actorList.pop()
                self.send(next_addr, msg)
            else:
                self.send(msg.asker, msg)

class AddTen(Actor):
    def __init__(self):
        self.next_adder = {}
        self.adders = None
    def receiveMessage(self, msg, sender):
        if not self.adders:
            # Create 10 AddOne Actors
            self.adders = [ self.createActor(AddOne) for x in range(10) ]
        if isinstance(msg, type((1,2))):
            self.next_adder[msg[0]] = msg[1]
        elif isinstance(msg, addEvery):
            if not msg.asker: msg.asker = sender  # start of chain, remember asker
            if hasattr(msg, 'parentmsg'):
                # completion chain is returning here
                parmsg = msg.parentmsg
                parmsg.value = msg.value
                self.send(self.next_adder.get(parmsg.step, parmsg.asker), parmsg)
            else:
                # start of sub-adder chain
                submsg = addEvery(msg.value, actorList = self.adders[1:])
                submsg.parentmsg = msg
                self.send(self.adders[0], submsg)
        elif msg == "Names of Children?":
            self.send(sender, self.adders)

class TestFuncActorAdder(object):

    def test00_ready(self, asys):
        pass

    def test01_TenEvery(self, asys):
        unstable_test(asys, 'multiprocQueueBase')
        addOnes = [ asys.createActor(AddOne) for x in range(10) ]
        for (frmA, toA) in zip(addOnes, addOnes[1:]):
            asys.tell(frmA, (1, toA))
        sum = asys.ask(addOnes[0], addEvery(actorList = addOnes[1:]), max_wait)
        assert sum.value == 10
        sum2 = asys.ask(addOnes[0], addEvery(actorList = addOnes[1:]), max_wait)
        assert sum2.value == 10

    def test02_TenEvens(self, asys):
        unstable_test(asys, 'multiprocQueueBase')
        if 'pypy' in sys.executable:
            pytest.skip("Pypy adder processes will consume all available system"
                        " memory for no good reason")
        addOnes = [ asys.createActor(AddOne) for x in range(10) ]
        for (frmA, toA) in zip(addOnes, addOnes[1:]):
            asys.tell(frmA, (1, toA))
        for (frmA, toA) in zip(addOnes[0:], addOnes[2:]):
            asys.tell(frmA, (2, toA))
        sum = asys.ask(addOnes[0], addEvery(actorList = addOnes[1:]), max_wait)
        assert sum.value == 10
        sum2 = asys.ask(addOnes[0],
                        addEvery(actorList = [addOnes[I]
                                              for I in range(2, len(addOnes),2)]),
                        max_wait)
        assert sum2.value == 5
        sum3 = asys.ask(addOnes[1],
                        addEvery(actorList = [addOnes[I]
                                              for I in range(3, len(addOnes),2)]),
                        max_wait)
        assert sum3.value == 5
        sum4 = asys.ask(addOnes[4],
                        addEvery(actorList = [addOnes[I]
                                              for I in range(6, len(addOnes),2)]),
                        max_wait)
        assert sum4.value == 3

    def test03_LotsOfActorsEvery(self, asys):
        unstable_test(asys, 'multiprocQueueBase')
        if 'pypy' in sys.executable:
            pytest.skip("Pypy adder processes will consume all available system"
                        " memory for no good reason")
        addOnes = [ asys.createActor(AddOne) for x in range(50) ]
        for (frmA, toA) in zip(addOnes, addOnes[1:]):
            asys.tell(frmA, (1, toA))
        sum = asys.ask(addOnes[0], addEvery(actorList = addOnes[1:]), max_wait)
        assert sum.value == 50
        sum2 = asys.ask(addOnes[0], addEvery(actorList = addOnes[1:]), max_wait)
        assert sum2.value == 50
        for killA in addOnes:
            asys.tell(killA, ActorExitRequest())

    def test04_LotsOfActorsEveryTen(self, asys):
        unstable_test(asys, 'multiprocQueueBase')
        if 'pypy' in sys.executable:
            pytest.skip("Pypy adder processes will consume all available system"
                        " memory for no good reason")
        # Create a set of addTens
        addTens = [ asys.createActor(AddTen) for x in range(0, 50, 10) ]
        # Point each addTens to the next one to make a chain.  Each
        # addTens will automatically make 10 addOnes children.
        for (frmA, toA) in zip(addTens, addTens[1:]):
            asys.tell(frmA, (1, toA))

        sum = asys.ask(addTens[0], addEvery(), max_wait)
        assert sum.value == 50
        sum2 = asys.ask(addTens[0], addEvery(), max_wait)
        assert sum2.value == 50
        for killA in addTens:
            asys.tell(killA, ActorExitRequest())

    def test05_UniqueAddresses(self, asys):
        unstable_test(asys, 'multiprocQueueBase')
        if 'pypy' in sys.executable:
            pytest.skip("Pypy adder processes will consume all available system"
                        " memory for no good reason")
        addTen = asys.createActor(AddTen)
        children = asys.ask(addTen, "Names of Children?", max_wait)
        uniqueAddresses = [addTen]
        for each in children:
            assert each not in uniqueAddresses
            uniqueAddresses.append(each)
        assert 11 == len(uniqueAddresses)
        #Not needed: asys.tell(addTen, ActorExitRequest())

    def test06_TenEveryTen(self, asys):
        unstable_test(asys, 'multiprocQueueBase')
        if 'pypy' in sys.executable:
            pytest.skip("Pypy adder processes will consume all available system"
                        " memory for no good reason")
        addTen = asys.createActor(AddTen)
        sum = asys.ask(addTen, addEvery(), max_wait)
        assert sum.value == 10

    def test07_LotsOfActorsEveryTenWithBackground(self, asys):
        unstable_test(asys, 'multiprocQueueBase')
        if 'pypy' in sys.executable:
            pytest.skip("Pypy adder processes will consume all available system"
                        " memory for no good reason")
        addTens = [ asys.createActor(AddTen) for x in range(0, 50, 10) ]
        for (frmA, toA) in zip(addTens, addTens[1:]):
            asys.tell(frmA, (1, toA))
        drop = asys.createActor(DropResults)
        asys.tell(drop, addEvery)  # verify it can be used
        import random
        for num in range(random.randint(0, 100)):
            start = random.choice(addTens)
            asys.tell(start, addEvery(asker=drop))  # drop results
        asys.tell(addTens[-1], addEvery(asker=drop))  # drop results
        sum = asys.ask(addTens[0], addEvery(), max_wait)
        assert sum.value == 50
        for killA in addTens:
            asys.tell(killA, ActorExitRequest())
<|endoftext|>


<commit_before>from django.shortcuts import render
from django.http import HttpResponse

# Create your views here.

def index(request):
	return HttpResponse('Hello world')<commit_msg>Return actual HTML<commit_after>from django.shortcuts import render
from django.http import HttpResponse

# Create your views here.

def index(request):
	return render(request, 'search/index.html', {})<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import re
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.http import HttpResponse
from django.http import JsonResponse
from django.shortcuts import render, get_object_or_404
from data_center.models import Course, Department
from data_center.const import DEPT_CHOICE, GEC_CHOICE, GE_CHOICE
from django.views.decorators.cache import cache_page
from django import forms

from haystack.query import SearchQuerySet
from haystack.inputs import AutoQuery

def group_words(s):
    """Split Chinese token for better search result"""
    regex = []

    # Match a whole word:
    regex += [r'\w+']

    # Match a single CJK character:
    regex += [r'[\u4e00-\ufaff]']

    # Match one of anything else, except for spaces:
    regex += [r'[^\s]']

    regex = "|".join(regex)
    r = re.compile(regex)

    return r.findall(s)

def search(request):
    result = {}
    q = request.GET.get('q', '')
    q = ' '.join(group_words(q))
    page = request.GET.get('page', '')
    size = request.GET.get('size', '')
    code = request.GET.get('code', '')
    dept_required = request.GET.get('dept_required', '')
    sortby_param = request.GET.get('sort', '')
    reverse_param = request.GET.get('reverse', '')
    ys = request.GET.get('ys', '104|20')

    page_size = size or 10
    sortby = sortby_param or 'time_token'
    reverse = True if reverse_param == 'true' else False
    rev_sortby = '-' + sortby if reverse else sortby

    courses = SearchQuerySet().filter()

    if dept_required:
        try:
            courses = Department.objects.get(
                ys=ys, dept_name=dept_required).required_course.all()
        except:
            pass
        if courses:
            result['type'] = 'required'
            page_size = courses.count()
    else:
        courses = courses.filter(content=AutoQuery(q))
        if code:
            courses = courses.filter(code__contains=code)

        if code in ['GE', 'GEC']:
            core = request.GET.get(code.lower(), '')
            if core:
                courses = courses.filter(ge__contains=core)
    courses = courses.order_by(rev_sortby)
    paginator = Paginator(courses, page_size)

    try:
        courses_page = paginator.page(page)
    except PageNotAnInteger:
        # If page is not an integer, deliver first page.
        courses_page = paginator.page(1)
    except EmptyPage:
        # If page is out of range (e.g. 9999), deliver last page of results.
        courses_page = paginator.page(paginator.num_pages)

    course_list = [x.object.__dict__ for x in courses_page.object_list]
    for x in course_list:
        for key in [k for k in x.keys() if k.startswith('_')]:
            x.pop(key, None)

    result['total'] = courses.count()
    result['page'] = courses_page.number
    result['courses'] = course_list
    result['page_size'] = page_size

    return JsonResponse(result)

@cache_page(60 * 60)
def syllabus(request, no):
    course = get_object_or_404(Course, no=no)
    return render(request, 'search/syllabus.html',
                  {'course': course, 'syllabus_path': request.path})

def hit(request, no):
    course = get_object_or_404(Course, no=no)
    course.hit += 1
    course.save()
    return HttpResponse('')

def generate_dept_required_choice():
    choices = (('', '---'),)
    try:
        departments = Department.objects.all()
        for department in departments:
            dept_name = department.dept_name
            year = {'104': '一年級', '103': '二年級', '102': '三年級', '101': '四年級'}. \
                get(dept_name[4:7], '')
            degree = {'B': '大學部', 'D': '博士班', 'M': '碩士班', 'P': '專班'}. \
                get(dept_name[7], '')
            chi_dept_name = degree

            if dept_name[7] == 'B':
                chi_dept_name += year
                chi_dept_name += {'BA': '清班', 'BB': '華班', 'BC': '梅班'}. \
                    get(dept_name[7:], '')

            choices += ((dept_name, chi_dept_name),)
    except:
        pass
    return sorted(choices)

class CourseSearchForm(forms.Form):
    DEPT_REQUIRED_CHOICE = generate_dept_required_choice()
    q = forms.CharField(label='關鍵字', required=False)
    code = forms.ChoiceField(label='開課代號', choices=DEPT_CHOICE, required=False)
    ge = forms.ChoiceField(label='向度', choices=GE_CHOICE, required=False)
    gec = forms.ChoiceField(label='向度', choices=GEC_CHOICE, required=False)
    dept_required = forms.ChoiceField(
        label='必選修', choices=DEPT_REQUIRED_CHOICE, required=False)

def table(request):
    render_data = {}
    render_data['search_filter'] = CourseSearchForm(request.GET)
    return render(request, 'table/table.html', render_data)
<commit_msg>fix regex for python3<commit_after># -*- coding: utf-8 -*-
import re
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.http import HttpResponse
from django.http import JsonResponse
from django.shortcuts import render, get_object_or_404
from data_center.models import Course, Department
from data_center.const import DEPT_CHOICE, GEC_CHOICE, GE_CHOICE
from django.views.decorators.cache import cache_page
from django import forms

from haystack.query import SearchQuerySet
from haystack.inputs import AutoQuery

def group_words(s):
    """Split Chinese token for better search result"""
    regex = []

    # Match a whole word:
    regex += [r'[a-zA-Z0-9_]+']

    # Match a single CJK character:
    regex += [r'[\u4e00-\ufaff]']

    # Match one of anything else, except for spaces:
    regex += [r'[^\s]']

    regex = "|".join(regex)
    r = re.compile(regex)

    return r.findall(s)

def search(request):
    result = {}
    q = request.GET.get('q', '')
    q = ' '.join(group_words(q))
    page = request.GET.get('page', '')
    size = request.GET.get('size', '')
    code = request.GET.get('code', '')
    dept_required = request.GET.get('dept_required', '')
    sortby_param = request.GET.get('sort', '')
    reverse_param = request.GET.get('reverse', '')
    ys = request.GET.get('ys', '104|20')

    page_size = size or 10
    sortby = sortby_param or 'time_token'
    reverse = True if reverse_param == 'true' else False
    rev_sortby = '-' + sortby if reverse else sortby

    courses = SearchQuerySet().filter()

    if dept_required:
        try:
            courses = Department.objects.get(
                ys=ys, dept_name=dept_required).required_course.all()
        except:
            pass
        if courses:
            result['type'] = 'required'
            page_size = courses.count()
    else:
        courses = courses.filter(content=AutoQuery(q))
        if code:
            courses = courses.filter(code__contains=code)

        if code in ['GE', 'GEC']:
            core = request.GET.get(code.lower(), '')
            if core:
                courses = courses.filter(ge__contains=core)
    courses = courses.order_by(rev_sortby)
    paginator = Paginator(courses, page_size)

    try:
        courses_page = paginator.page(page)
    except PageNotAnInteger:
        # If page is not an integer, deliver first page.
        courses_page = paginator.page(1)
    except EmptyPage:
        # If page is out of range (e.g. 9999), deliver last page of results.
        courses_page = paginator.page(paginator.num_pages)

    course_list = [x.object.__dict__ for x in courses_page.object_list]
    for x in course_list:
        for key in [k for k in x.keys() if k.startswith('_')]:
            x.pop(key, None)

    result['total'] = courses.count()
    result['page'] = courses_page.number
    result['courses'] = course_list
    result['page_size'] = page_size

    return JsonResponse(result)

@cache_page(60 * 60)
def syllabus(request, no):
    course = get_object_or_404(Course, no=no)
    return render(request, 'search/syllabus.html',
                  {'course': course, 'syllabus_path': request.path})

def hit(request, no):
    course = get_object_or_404(Course, no=no)
    course.hit += 1
    course.save()
    return HttpResponse('')

def generate_dept_required_choice():
    choices = (('', '---'),)
    try:
        departments = Department.objects.all()
        for department in departments:
            dept_name = department.dept_name
            year = {'104': '一年級', '103': '二年級', '102': '三年級', '101': '四年級'}. \
                get(dept_name[4:7], '')
            degree = {'B': '大學部', 'D': '博士班', 'M': '碩士班', 'P': '專班'}. \
                get(dept_name[7], '')
            chi_dept_name = degree

            if dept_name[7] == 'B':
                chi_dept_name += year
                chi_dept_name += {'BA': '清班', 'BB': '華班', 'BC': '梅班'}. \
                    get(dept_name[7:], '')

            choices += ((dept_name, chi_dept_name),)
    except:
        pass
    return sorted(choices)

class CourseSearchForm(forms.Form):
    DEPT_REQUIRED_CHOICE = generate_dept_required_choice()
    q = forms.CharField(label='關鍵字', required=False)
    code = forms.ChoiceField(label='開課代號', choices=DEPT_CHOICE, required=False)
    ge = forms.ChoiceField(label='向度', choices=GE_CHOICE, required=False)
    gec = forms.ChoiceField(label='向度', choices=GEC_CHOICE, required=False)
    dept_required = forms.ChoiceField(
        label='必選修', choices=DEPT_REQUIRED_CHOICE, required=False)

def table(request):
    render_data = {}
    render_data['search_filter'] = CourseSearchForm(request.GET)
    return render(request, 'table/table.html', render_data)
<|endoftext|>


<commit_before># TODO - cleanup regex to do proper string subs
# TODO - add is_other field to repo
# TODO - add repo.user_url

from django.conf import settings
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.db import models
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _
from github2.client import Github
from package.fields import CreationDateTimeField, ModificationDateTimeField
from package.handlers import github
from package.pypi import fetch_releases
from package.utils import uniquer
from distutils.version import LooseVersion as versioner
from urllib import urlopen
import logging
import os
import re
import sys

class NoPyPiVersionFound(Exception):
    pass

class BaseModel(models.Model):
    """ Base abstract base class to give creation and modified times """
    created     = CreationDateTimeField(_('created'))
    modified    = ModificationDateTimeField(_('modified'))

    class Meta:
        abstract = True

class Category(BaseModel):

    title = models.CharField(_("Title"), max_length="50")
    slug  = models.SlugField(_("slug"))
    description = models.TextField(_("description"), blank=True)
    title_plural = models.CharField(_("Title Plural"), max_length="50", blank=True)
    show_pypi = models.BooleanField(_("Show pypi stats & version"), default=True)

    class Meta:
        ordering = ['title']
        verbose_name_plural = 'Categories'

    def __unicode__(self):
        return self.title

REPO_CHOICES = (
    ("package.handlers.unsupported", "Unsupported"),
    ("package.handlers.bitbucket", "Bitbucket"),
    ("package.handlers.github", "Github"),
    ("package.handlers.launchpad", "Launchpad"),
    ("package.handlers.sourceforge", "Sourceforge")
)

# XXX Repo stuff ???
downloads_re = re.compile(r'[0-9]{1,}	')
doap_re      = re.compile(r"/pypi\?\:action=doap\&name=[a-zA-Z0-9\.\-\_]+\&version=[a-zA-Z0-9\.\-\_]+")
version_re   = re.compile(r'<revision>[a-zA-Z0-9\.\-\_]+</revision>')

repo_url_help_text = "Enter your project repo hosting URL here.
Example: https://bitbucket.com/ubernostrum/django-registration"
pypi_url_help_text = "Leave this blank if this package does not have a PyPI release.
What PyPI uses to index your package.
Example: django-registration"
category_help_text = """

Apps is anything that is installed by placing in settings.INSTALLED_APPS.

Frameworks are large efforts that combine many python modules or apps to build things like Pinax.

Other are not installed by settings.INSTALLED_APPS, are not frameworks or sites but still help Django in some way.

Projects are individual projects such as Django Packages, DjangoProject.com, and others.

"""

class Package(BaseModel):

    title           = models.CharField(_("Title"), max_length="100")
    slug            = models.SlugField(_("Slug"), help_text="Slugs will be lowercased", unique=True)
    category        = models.ForeignKey(Category, verbose_name="Installation", help_text=category_help_text)
    repo_description= models.TextField(_("Repo Description"), blank=True)
    repo_url        = models.URLField(_("repo URL"), help_text=repo_url_help_text, blank=True,unique=True)
    repo_watchers   = models.IntegerField(_("repo watchers"), default=0)
    repo_forks      = models.IntegerField(_("repo forks"), default=0)
    repo_commits    = models.IntegerField(_("repo commits"), default=0)
    pypi_url        = models.URLField(_("PyPI slug"), help_text=pypi_url_help_text, blank=True, default='')
    pypi_downloads  = models.IntegerField(_("Pypi downloads"), default=0)
    related_packages    = models.ManyToManyField("self", blank=True)
    participants    = models.TextField(_("Participants"),
                        help_text="List of collaborats/participants on the project", blank=True)
    usage           = models.ManyToManyField(User, blank=True)
    created_by = models.ForeignKey(User, blank=True, null=True, related_name="creator")
    last_modified_by = models.ForeignKey(User, blank=True, null=True, related_name="modifier")
    pypi_home_page  = models.URLField(_("homepage on PyPI for a project"), blank=True, null=True)

    @property
    def pypi_version(self):
        string_ver_list = self.version_set.values_list('number', flat=True)
        if string_ver_list:
            vers_list = [versioner(v) for v in string_ver_list]
            latest = sorted(vers_list)[-1]
            return str(latest)
        return ''

    @property
    def pypi_name(self):
        """ return the pypi name of a package"""

        if not self.pypi_url.strip():
            return ""

        name = self.pypi_url.replace("http://pypi.python.org/pypi/","")
        if "/" in name:
            return name[:name.index("/")]
        return name

    @property
    def last_updated(self):
        last_commit = self.commit_set.latest('commit_date')
        if last_commit:
            return last_commit.commit_date
        return None

    @property
    def repo(self):
        from package.handlers import get_handler_for_repo_url
        handler = get_handler_for_repo_url(self.repo_url)
        return handler

    def active_examples(self):
        return self.packageexample_set.filter(active=True)

    def grids(self):

        return (x.grid for x in self.gridpackage_set.all())

    @property
    def repo_name(self):
        return self.repo_url.replace(self.repo.url + '/','')

    def participant_list(self):

        return self.participants.split(',')

    def commits_over_52(self):
        from package.templatetags.package_tags import commits_over_52
        return commits_over_52(self)

    def fetch_metadata(self, *args, **kwargs):

        # Get the downloads from pypi
        if self.pypi_url.strip() and self.pypi_url != "http://pypi.python.org/pypi/":

            total_downloads = 0

            for release in fetch_releases(self.pypi_name):

                version, created = Version.objects.get_or_create(
                    package = self,
                    number = release.version
                )

                # add to total downloads
                total_downloads += release.downloads

                # add to versions
                version.downloads = release.downloads
                version.license = release.license
                version.hidden = release._pypi_hidden
                version.save()

            self.pypi_downloads = total_downloads

        # Get the repo watchers number
        handler = self.repo.__module__

        self = handler.pull(self)
        self.save()

    class Meta:
        ordering = ['title']

    def __unicode__(self):

        return self.title

    @models.permalink
    def get_absolute_url(self):
        return ("package", [self.slug])

class PackageExample(BaseModel):

    package = models.ForeignKey(Package)
    title = models.CharField(_("Title"), max_length="100")
    url = models.URLField(_("URL"))
    active = models.BooleanField(_("Active"), default=True, help_text="Moderators have to approve links before they are provided")

    class Meta:
        ordering = ['title']

    def __unicode__(self):
        return self.title

class Commit(BaseModel):

    package      = models.ForeignKey(Package)
    commit_date  = models.DateTimeField(_("Commit Date"))

    class Meta:
        ordering = ['-commit_date']

    def __unicode__(self):
        return "Commit for '%s' on %s" % (self.package.title, unicode(self.commit_date))

class Version(BaseModel):

    package = models.ForeignKey(Package, blank=True, null=True)
    number = models.CharField(_("Version"), max_length="100", default="", blank="")
    downloads = models.IntegerField(_("downloads"), default=0)
    license = models.CharField(_("license"), max_length="100")
    hidden = models.BooleanField(_("hidden"), default=False)

    class Meta:
        get_latest_by = 'created'
        ordering = ['-created']

    def __unicode__(self):
        return "%s: %s" % (self.package.title, self.number)

<commit_msg>Cleanup; remove unused repo stuff<commit_after># TODO - cleanup regex to do proper string subs
# TODO - add is_other field to repo
# TODO - add repo.user_url

from django.conf import settings
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.db import models
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _
from github2.client import Github
from package.fields import CreationDateTimeField, ModificationDateTimeField
from package.handlers import github
from package.pypi import fetch_releases
from package.utils import uniquer
from distutils.version import LooseVersion as versioner
from urllib import urlopen
import logging
import os
import re
import sys

class NoPyPiVersionFound(Exception):
    pass

class BaseModel(models.Model):
    """ Base abstract base class to give creation and modified times """
    created     = CreationDateTimeField(_('created'))
    modified    = ModificationDateTimeField(_('modified'))

    class Meta:
        abstract = True

class Category(BaseModel):

    title = models.CharField(_("Title"), max_length="50")
    slug  = models.SlugField(_("slug"))
    description = models.TextField(_("description"), blank=True)
    title_plural = models.CharField(_("Title Plural"), max_length="50", blank=True)
    show_pypi = models.BooleanField(_("Show pypi stats & version"), default=True)

    class Meta:
        ordering = ['title']
        verbose_name_plural = 'Categories'

    def __unicode__(self):
        return self.title

repo_url_help_text = "Enter your project repo hosting URL here.
Example: https://bitbucket.com/ubernostrum/django-registration"
pypi_url_help_text = "Leave this blank if this package does not have a PyPI release.
What PyPI uses to index your package.
Example: django-registration"
category_help_text = """

Apps is anything that is installed by placing in settings.INSTALLED_APPS.

Frameworks are large efforts that combine many python modules or apps to build things like Pinax.

Other are not installed by settings.INSTALLED_APPS, are not frameworks or sites but still help Django in some way.

Projects are individual projects such as Django Packages, DjangoProject.com, and others.

"""

class Package(BaseModel):

    title           = models.CharField(_("Title"), max_length="100")
    slug            = models.SlugField(_("Slug"), help_text="Slugs will be lowercased", unique=True)
    category        = models.ForeignKey(Category, verbose_name="Installation", help_text=category_help_text)
    repo_description= models.TextField(_("Repo Description"), blank=True)
    repo_url        = models.URLField(_("repo URL"), help_text=repo_url_help_text, blank=True,unique=True)
    repo_watchers   = models.IntegerField(_("repo watchers"), default=0)
    repo_forks      = models.IntegerField(_("repo forks"), default=0)
    repo_commits    = models.IntegerField(_("repo commits"), default=0)
    pypi_url        = models.URLField(_("PyPI slug"), help_text=pypi_url_help_text, blank=True, default='')
    pypi_downloads  = models.IntegerField(_("Pypi downloads"), default=0)
    related_packages    = models.ManyToManyField("self", blank=True)
    participants    = models.TextField(_("Participants"),
                        help_text="List of collaborats/participants on the project", blank=True)
    usage           = models.ManyToManyField(User, blank=True)
    created_by = models.ForeignKey(User, blank=True, null=True, related_name="creator")
    last_modified_by = models.ForeignKey(User, blank=True, null=True, related_name="modifier")
    pypi_home_page  = models.URLField(_("homepage on PyPI for a project"), blank=True, null=True)

    @property
    def pypi_version(self):
        string_ver_list = self.version_set.values_list('number', flat=True)
        if string_ver_list:
            vers_list = [versioner(v) for v in string_ver_list]
            latest = sorted(vers_list)[-1]
            return str(latest)
        return ''

    @property
    def pypi_name(self):
        """ return the pypi name of a package"""

        if not self.pypi_url.strip():
            return ""

        name = self.pypi_url.replace("http://pypi.python.org/pypi/","")
        if "/" in name:
            return name[:name.index("/")]
        return name

    @property
    def last_updated(self):
        last_commit = self.commit_set.latest('commit_date')
        if last_commit:
            return last_commit.commit_date
        return None

    @property
    def repo(self):
        from package.handlers import get_handler_for_repo_url
        handler = get_handler_for_repo_url(self.repo_url)
        return handler

    def active_examples(self):
        return self.packageexample_set.filter(active=True)

    def grids(self):

        return (x.grid for x in self.gridpackage_set.all())

    @property
    def repo_name(self):
        return self.repo_url.replace(self.repo.url + '/','')

    def participant_list(self):

        return self.participants.split(',')

    def commits_over_52(self):
        from package.templatetags.package_tags import commits_over_52
        return commits_over_52(self)

    def fetch_metadata(self, *args, **kwargs):

        # Get the downloads from pypi
        if self.pypi_url.strip() and self.pypi_url != "http://pypi.python.org/pypi/":

            total_downloads = 0

            for release in fetch_releases(self.pypi_name):

                version, created = Version.objects.get_or_create(
                    package = self,
                    number = release.version
                )

                # add to total downloads
                total_downloads += release.downloads

                # add to versions
                version.downloads = release.downloads
                version.license = release.license
                version.hidden = release._pypi_hidden
                version.save()

            self.pypi_downloads = total_downloads

        # Get the repo watchers number
        handler = self.repo.__module__

        self = handler.pull(self)
        self.save()

    class Meta:
        ordering = ['title']

    def __unicode__(self):

        return self.title

    @models.permalink
    def get_absolute_url(self):
        return ("package", [self.slug])

class PackageExample(BaseModel):

    package = models.ForeignKey(Package)
    title = models.CharField(_("Title"), max_length="100")
    url = models.URLField(_("URL"))
    active = models.BooleanField(_("Active"), default=True, help_text="Moderators have to approve links before they are provided")

    class Meta:
        ordering = ['title']

    def __unicode__(self):
        return self.title

class Commit(BaseModel):

    package      = models.ForeignKey(Package)
    commit_date  = models.DateTimeField(_("Commit Date"))

    class Meta:
        ordering = ['-commit_date']

    def __unicode__(self):
        return "Commit for '%s' on %s" % (self.package.title, unicode(self.commit_date))

class Version(BaseModel):

    package = models.ForeignKey(Package, blank=True, null=True)
    number = models.CharField(_("Version"), max_length="100", default="", blank="")
    downloads = models.IntegerField(_("downloads"), default=0)
    license = models.CharField(_("license"), max_length="100")
    hidden = models.BooleanField(_("hidden"), default=False)

    class Meta:
        get_latest_by = 'created'
        ordering = ['-created']

    def __unicode__(self):
        return "%s: %s" % (self.package.title, self.number)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.11.15 on 2019-023-05 14:40
from __future__ import unicode_literals

from django.db import migrations, models
import django.db.models.deletion
import uuid

class Migration(migrations.Migration):

    dependencies = [
        ('models', '4384_adds_rerender_widget_config'),
    ]

    operations = [
        migrations.RunSQL(
            """
            update widgets as w
            set defaultconfig = jsonb_set(defaultconfig, '{displayOnlySelectedNode}', to_jsonb(false), true)
            where w.name = 'node-value-select';
            """,
            """
            update widgets as w
            set defaultconfig = defaultconfig - 'displayOnlySelectedNode'
            where w.name = 'node-value-select';
            """)
        ]
<commit_msg>updated migration dependency.re #4658<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.11.15 on 2019-023-05 14:40
from __future__ import unicode_literals

from django.db import migrations, models
import django.db.models.deletion
import uuid

class Migration(migrations.Migration):

    dependencies = [
        ('models', '4352_format_config_number_widget'),
    ]

    operations = [
        migrations.RunSQL(
            """
            update widgets as w
            set defaultconfig = jsonb_set(defaultconfig, '{displayOnlySelectedNode}', to_jsonb(false), true)
            where w.name = 'node-value-select';
            """,
            """
            update widgets as w
            set defaultconfig = defaultconfig - 'displayOnlySelectedNode'
            where w.name = 'node-value-select';
            """)
        ]
<|endoftext|>


<commit_before># _*_ coding: utf-8 _*_
"""
@copyright   Copyright (c) 2014 Submit Consulting
@author      Angel Sullon (@asullom)
@package     space

Descripcion: Registro de los modelos de la app space
"""
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.utils.text import capfirst
from django.dispatch import receiver
from django.db.models import signals
from unicodedata import normalize
from django.core.exceptions import ValidationError

# models
#from apps.params.models import Locality

# managers
#from .managers import SolutionManager

# others
#from django.utils import timezone
#from django.core.exceptions import NON_FIELD_ERRORS
#from django.template.defaultfilters import slugify

GOVERMENT = 'GOVERMENT'
PRIVATE = 'PRIVATE'
MIXED = 'MIXED'
OTHERS = 'OTHERS'
TYPE_CHOICES = (
    (GOVERMENT, _('Government')),
    (PRIVATE, _('Private')),
    (MIXED, _('Mixed')),
    (OTHERS, _('Others'))
)

def unique_name(value):
    if value == 'angel':
        raise ValidationError(u'%s is not an angel' % value)

class Solution(models.Model):

    """
    Tabla que contiene las soluciones o planes o servicios del sistema
    """
    name = models.CharField(capfirst(_('name')), max_length=50)
    description = models.TextField(_('Description'), null=True, blank=True)
    price = models.FloatField(_('Price'), null=True, blank=True)
    is_active = models.BooleanField(capfirst(_('active')), default=True)

    created_at = models.DateTimeField(_('Created at'), auto_now_add=True)
    updated_at = models.DateTimeField(_('Updated at'), auto_now=True)

    test_image = models.ImageField(
        _('Test image'), upload_to='test_images',
        default='test_images/default.png', null=True, blank=True)
    test_date = models.DateTimeField(_('Test date'),
                                     null=True, blank=True)

    class Meta:
        verbose_name = _('Solution')
        verbose_name_plural = _('Solutions')
        permissions = (
            ('solution', 'Can ALL solution'),
        )
        unique_together = ('name',)

    def __str__(self):
        return self.name

    def validate_unique(self, exclude=None):
        if normalize('NFKD', self.name).encode('ascii', 'ignore').lower() in list(
            normalize('NFKD', c['name']).encode('ascii', 'ignore').lower()
            for c in self.__class__.objects.values('name').exclude(pk=self.pk)
        ):
            raise ValidationError({
                'name':
                (_(u'%(model_name)s with this %(field_label)s already exists.') % {
                'model_name': _('Solution'),
                'field_label': capfirst(_('name')),
                },),
            })

        super(Solution, self).validate_unique(exclude=exclude)

class Association(models.Model):

    """
    Tabla que contiene las asociaciones suscritas a un plan
    """

    name = models.CharField(capfirst(_('name')), max_length=50)
    logo = models.ImageField(
        _('Logo'),
        upload_to='associations', default='associations/default.png')
    type_a = models.CharField(
        _('Type'),
        max_length=10, choices=TYPE_CHOICES, default=PRIVATE)
    is_active = models.BooleanField(capfirst(_('active')), default=True)
    is_actived = models.BooleanField(_('Actived'), default=False)

    created_at = models.DateTimeField(_('Created at'), auto_now_add=True)
    updated_at = models.DateTimeField(_('Updated at'), auto_now=True)

    solution = models.ForeignKey(
        Solution, verbose_name=_('Solution'), null=True, blank=True)

    class Meta:
        verbose_name = _('Association')
        verbose_name_plural = _('Associations')
        permissions = (
            ('association', 'Can ALL association'),
        )
        unique_together = ('name',)

    def __str__(self):
        return self.name

    def save(self, *args, **kwargs):
        if normalize('NFKD', u'%s' % self.name).encode('ascii', 'ignore').lower() in list(
                normalize('NFKD', u'%s' % c['name']).encode('ascii', 'ignore').lower() for c in Association.objects.values('name').exclude(pk=self.pk)
        ):
            raise Exception(_(u'%(model_name)s with this %(field_label)s already exists.') % {
                'model_name': _('Association'),
                'field_label': capfirst(_(u'name')),
            })
        super(Association, self).save(*args, **kwargs)

class Enterprise(models.Model):

    """
    Tabla que contiene las empresas suscritas a un plan
    """

    name = models.CharField(capfirst(_('name')), max_length=50)
    #name_sm = models.CharField(capfirst(_('siglas')), max_length=50)
    logo = models.ImageField(
        _('Logo'), upload_to='enterprises', default='enterprises/default.png')
    tax_id = models.CharField(_('Tax id'), max_length=50)
    type_e = models.CharField(
        _('Type'),
        max_length=10, choices=TYPE_CHOICES, default=PRIVATE)
    is_active = models.BooleanField(capfirst(_('active')), default=True)
    is_actived = models.BooleanField(_('Actived'), default=False)

    created_at = models.DateTimeField(_('Created at'), auto_now_add=True)
    updated_at = models.DateTimeField(_('Updated at'), auto_now=True)

    solution = models.ForeignKey(
        Solution, verbose_name=_('Solution'), null=True, blank=True)

    class Meta:
        verbose_name = _('Enterprise')
        verbose_name_plural = _('Enterprises')
        permissions = (
            ('enterprise', 'Can ALL enterprise'),
        )
        # no duplicate name, no duplicate tax_id
        unique_together = ('name', 'tax_id',)

    def __str__(self):
        return self.name

    def save(self, *args, **kwargs):
        if normalize('NFKD', u'%s' % self.name).encode('ascii', 'ignore').lower() in list(
                normalize('NFKD', u'%s' % c['name']).encode('ascii', 'ignore').lower() for c in Enterprise.objects.values('name').exclude(pk=self.pk)
        ):
            raise Exception(_(u'%(model_name)s with this %(field_label)s already exists.') % {
                'model_name': _('Enterprise'),
                'field_label': capfirst(_(u'name')),
            })
        if Enterprise.objects.exclude(pk=self.pk).filter(tax_id=self.tax_id).count() > 0:
            raise Exception(_(u'%(model_name)s with this %(field_label)s already exists.') % {
                'model_name': _('Enterprise'),
                'field_label': capfirst(_(u'Tax id')),
            })
        super(Enterprise, self).save(*args, **kwargs)

class Headquar(models.Model):

    """
    Tabla que contiene las sedes de las empresas, asociadas a una Asociación
    Un Headquar o sede o sucursal, es la unidad principal del sistema
    Los accesos del sistema serán entregados a un Headquarters
    """
    name = models.CharField(capfirst(_('name')), max_length=50)

    phone = models.CharField(_('Phone'), max_length=50, null=True, blank=True)

    address = models.TextField(_('Address'), null=True, blank=True)
    is_main = models.BooleanField(_('Main'), default=False)
    is_active = models.BooleanField(capfirst(_('active')), default=True)
    is_actived = models.BooleanField(_('Actived'), default=False)

    created_at = models.DateTimeField(_('Created at'), auto_now_add=True)
    updated_at = models.DateTimeField(_('Updated at'), auto_now=True)

    # locality = models.ForeignKey(Locality, verbose_name=_('Locality'),
    #	null=True, blank=True)
    association = models.ForeignKey(
        Association, verbose_name=_('Association'), null=True, blank=True)
    enterprise = models.ForeignKey(Enterprise, verbose_name=_('Enterprise'))

    class Meta:
        verbose_name = _('Headquar')
        verbose_name_plural = _('Headquars')
        permissions = (
            ('headquar', 'Can ALL headquar(sedes)'),
        )
        # no duplicate name by enterprise
        unique_together = (('name', 'enterprise'),)

    def __str__(self):
        return '%s %s (%s)' % (self.name, self.enterprise, self.association)

    def save(self, *args, **kwargs):
        if normalize('NFKD', u'%s' % self.name).encode('ascii', 'ignore').lower() in list(
                normalize('NFKD', u'%s' % c['name']).encode('ascii', 'ignore').lower() for c in Headquar.objects.values('name').exclude(pk=self.pk).filter(enterprise=self.enterprise)
        ):
            raise Exception(_(u'%(model_name)s with this %(field_label)s already exists.') % {
                'model_name': _('Headquar'),
                'field_label': capfirst(_(u'name')),
            })
        super(Headquar, self).save(*args, **kwargs)
<commit_msg>primera version2<commit_after># _*_ coding: utf-8 _*_
"""
@copyright   Copyright (c) 2014 Submit Consulting
@author      Angel Sullon (@asullom)
@package     space

Descripcion: Registro de los modelos de la app space
"""
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.utils.text import capfirst
from django.dispatch import receiver
from django.db.models import signals
from unicodedata import normalize
from django.core.exceptions import ValidationError

# models
#from apps.params.models import Locality

# managers
#from .managers import SolutionManager

# others
#from django.utils import timezone
#from django.core.exceptions import NON_FIELD_ERRORS
#from django.template.defaultfilters import slugify

GOVERMENT = 'GOVERMENT'
PRIVATE = 'PRIVATE'
MIXED = 'MIXED'
OTHERS = 'OTHERS'
TYPE_CHOICES = (
    (GOVERMENT, _('Government')),
    (PRIVATE, _('Private')),
    (MIXED, _('Mixed')),
    (OTHERS, _('Others'))
)

def unique_name(value):
    if value == 'angel':
        raise ValidationError(u'%s is not an angel' % value)

class Solution(models.Model):

    """
    Tabla que contiene las soluciones o planes o servicios del sistema
    """
    name = models.CharField(capfirst(_('name')), max_length=50)
    description = models.TextField(_('Description'), null=True, blank=True)
    price = models.FloatField(_('Price'), null=True, blank=True)
    is_active = models.BooleanField(capfirst(_('active')), default=True)

    created_at = models.DateTimeField(_('Created at'), auto_now_add=True)
    updated_at = models.DateTimeField(_('Updated at'), auto_now=True)

    test_image = models.ImageField(
        _('Test image'), upload_to='test_images',
        default='test_images/default.png', null=True, blank=True)
    test_date = models.DateTimeField(_('Test date'),
                                     null=True, blank=True)

    class Meta:
        verbose_name = _('Solution')
        verbose_name_plural = _('Solutions')
        permissions = (
            ('solution', 'Can ALL solution'),
        )
        unique_together = ('name',)

    def __str__(self):
        return self.name

    def validate_unique(self, exclude=None):
        if normalize('NFKD', self.name).encode('ascii', 'ignore').lower() in list(
            normalize('NFKD', c['name']).encode('ascii', 'ignore').lower()
            for c in self.__class__.objects.values('name').exclude(pk=self.pk)
        ):
            raise ValidationError({
                'name':
                (_(u'%(model_name)s with this %(field_label)s already exists.') % {
                'model_name': _('Solution'),
                'field_label': capfirst(_('name')),
                },),
            })

        super(Solution, self).validate_unique(exclude=exclude)

class Association(models.Model):

    """
    Tabla que contiene las asociaciones suscritas a un plan
    """

    name = models.CharField(capfirst(_('name')), max_length=50)
    logo = models.ImageField(
        _('Logo'),
        upload_to='associations', default='associations/default.png')
    type_a = models.CharField(
        _('Type'),
        max_length=10, choices=TYPE_CHOICES, default=PRIVATE)
    is_active = models.BooleanField(capfirst(_('active')), default=True)
    is_actived = models.BooleanField(_('Actived'), default=False)

    created_at = models.DateTimeField(_('Created at'), auto_now_add=True)
    updated_at = models.DateTimeField(_('Updated at'), auto_now=True)

    solution = models.ForeignKey(
        Solution, verbose_name=_('Solution'), null=True, blank=True)

    class Meta:
        verbose_name = _('Association')
        verbose_name_plural = _('Associations')
        permissions = (
            ('association', 'Can ALL association'),
        )
        unique_together = ('name',)

    def __str__(self):
        return self.name

    def save(self, *args, **kwargs):
        if normalize('NFKD', u'%s' % self.name).encode('ascii', 'ignore').lower() in list(
                normalize('NFKD', u'%s' % c['name']).encode('ascii', 'ignore').lower() for c in Association.objects.values('name').exclude(pk=self.pk)
        ):
            raise Exception(_(u'%(model_name)s with this %(field_label)s already exists.') % {
                'model_name': _('Association'),
                'field_label': capfirst(_(u'name')),
            })
        super(Association, self).save(*args, **kwargs)

class Enterprise(models.Model):

    """
    Tabla que contiene las empresas suscritas a un plan
    """

    name = models.CharField(capfirst(_('name')), max_length=50)
    #name_sm = models.CharField(capfirst(_('siglas')), max_length=50)
    logo = models.ImageField(
        _('Logo'), upload_to='enterprises', default='enterprises/default.png')
    tax_id = models.CharField(_('Tax id'), max_length=50)
    type_e = models.CharField(
        _('Type'),
        max_length=10, choices=TYPE_CHOICES, default=PRIVATE)
    is_active = models.BooleanField(capfirst(_('active')), default=True)
    is_actived = models.BooleanField(_('Actived'), default=False)

    created_at = models.DateTimeField(_('Created at'), auto_now_add=True) #cuando se crea
    updated_at = models.DateTimeField(_('Updated at'), auto_now=True) #cuando actualizs

    solution = models.ForeignKey(
        Solution, verbose_name=_('Solution'), null=True, blank=True)

    class Meta:
        verbose_name = _('Enterprise')
        verbose_name_plural = _('Enterprises')
        permissions = (
            ('enterprise', 'Can ALL enterprise'),
        )
        # no duplicate name, no duplicate tax_id
        unique_together = ('name', 'tax_id',)

    def __str__(self):
        return self.name

    def save(self, *args, **kwargs):
        if normalize('NFKD', u'%s' % self.name).encode('ascii', 'ignore').lower() in list(
                normalize('NFKD', u'%s' % c['name']).encode('ascii', 'ignore').lower() for c in Enterprise.objects.values('name').exclude(pk=self.pk)
        ):
            raise Exception(_(u'%(model_name)s with this %(field_label)s already exists.') % {
                'model_name': _('Enterprise'),
                'field_label': capfirst(_(u'name')),
            })
        if Enterprise.objects.exclude(pk=self.pk).filter(tax_id=self.tax_id).count() > 0:
            raise Exception(_(u'%(model_name)s with this %(field_label)s already exists.') % {
                'model_name': _('Enterprise'),
                'field_label': capfirst(_(u'Tax id')),
            })
        super(Enterprise, self).save(*args, **kwargs)

class Headquar(models.Model):

    """
    Tabla que contiene las sedes de las empresas, asociadas a una Asociación
    Un Headquar o sede o sucursal, es la unidad principal del sistema
    Los accesos del sistema serán entregados a un Headquarters
    """
    name = models.CharField(capfirst(_('name')), max_length=50)

    phone = models.CharField(_('Phone'), max_length=50, null=True, blank=True)

    address = models.TextField(_('Address'), null=True, blank=True)
    is_main = models.BooleanField(_('Main'), default=False)
    is_active = models.BooleanField(capfirst(_('active')), default=True)
    is_actived = models.BooleanField(_('Actived'), default=False)

    created_at = models.DateTimeField(_('Created at'), auto_now_add=True)
    updated_at = models.DateTimeField(_('Updated at'), auto_now=True)

    # locality = models.ForeignKey(Locality, verbose_name=_('Locality'),
    #	null=True, blank=True)
    association = models.ForeignKey(
        Association, verbose_name=_('Association'), null=True, blank=True)
    enterprise = models.ForeignKey(Enterprise, verbose_name=_('Enterprise'))

    class Meta:
        verbose_name = _('Headquar')
        verbose_name_plural = _('Headquars')
        permissions = (
            ('headquar', 'Can ALL headquar(sedes)'),
        )
        # no duplicate name by enterprise
        unique_together = (('name', 'enterprise'),)

    def __str__(self):
        return '%s %s (%s)' % (self.name, self.enterprise, self.association)

    def save(self, *args, **kwargs):
        if normalize('NFKD', u'%s' % self.name).encode('ascii', 'ignore').lower() in list(
                normalize('NFKD', u'%s' % c['name']).encode('ascii', 'ignore').lower() for c in Headquar.objects.values('name').exclude(pk=self.pk).filter(enterprise=self.enterprise)
        ):
            raise Exception(_(u'%(model_name)s with this %(field_label)s already exists.') % {
                'model_name': _('Headquar'),
                'field_label': capfirst(_(u'name')),
            })
        super(Headquar, self).save(*args, **kwargs)
<|endoftext|>