<commit_before>import re

from .primitive import Component, Creator, Translator

class SchemaArrayValidationHandler:

    def __init__(self, visitor):
        self.visitor = visitor

    def __call__(self, sequence, *args):
        errors = []
        for element in sequence:
            try:
                element.accept(self.visitor, *args)
            except AssertionError as e:
                errors.append(e.args[0])
                continue
        return errors

class ValidationVisitor:

    def __init__(self, instance):
        self.instance = instance

    def visit_empty_schema(self, schema, *args):  # pragma: no cover
        """Always passes validation."""
        return

    def visit_enumeration(self, enumeration, *args):
        self.visit_primitive(enumeration, *args)
        assert self.instance in enumeration.enum, 'instance %r is not equal to one of the elements %r' % (self.instance, enumeration)  # noqa: E501

    def visit_all_of(self, all_of, *args):
        for element in all_of:
            element.accept(self, *args)

    def visit_any_of(self, any_of, *args):
        if any_of:
            handler = SchemaArrayValidationHandler(self)
            errors = handler(any_of, *args)
            assert len(any_of) != len(errors), ', '.join(errors)

    def visit_one_of(self, one_of, *args):
        if one_of:
            handler = SchemaArrayValidationHandler(self)
            errors = handler(one_of, *args)
            assert len(errors) == 1, ', '.join(errors)

    def visit_primitive(self, primitive, *args):
        instance = self.instance
        if primitive.const is not None:
            assert instance == primitive.const, 'instance %r is not equal to %r' % (instance, primitive.const)  # noqa: E501
        if primitive.type is not None:
            assert {
                str: lambda instance: Translator.translate(instance).__name__.lower() == primitive.type,  # noqa: E501
                list: lambda instance: Translator.translate(instance).__name__.lower() in primitive.type,  # noqa: E501
            }[primitive.type.__class__](instance), 'instance %r is not in any of the sets listed %r' % (instance, primitive.type)  # noqa: E501
        primitive.allOf.accept(self, *args)
        primitive.anyOf.accept(self, *args)
        primitive.oneOf.accept(self, *args)

    def visit_boolean(self, boolean, *args):
        self.visit_primitive(boolean, *args)

    def visit_null(self, null, *args):
        self.visit_primitive(null, *args)

    def visit_numeric(self, numeric, *args):
        self.visit_primitive(numeric, *args)

        cls = args[0]
        instance = cls(self.instance)
        assert float(instance / numeric.multipleOf).is_integer(), 'instance %r division by %r is not an integer' % (instance, numeric.multipleOf)  # noqa: E501
        if numeric.maximum is not None:
            assert instance <= numeric.maximum, 'instance %r is not less than or exactly equal to %r' % (instance, numeric.maximum)  # noqa: E501
        if numeric.exclusiveMaximum is not None:
            assert instance < numeric.exclusiveMaximum, 'instance %r is not strictly less than (not equal to) %r' % (instance, numeric.exclusiveMaximum)  # noqa: E501
        if numeric.minimum is not None:
            assert instance >= numeric.minimum, 'instance %r is not greater than or exactly equal to %r' % (instance, numeric.minimum)  # noqa: E501
        if numeric.exclusiveMinimum is not None:
            assert instance > numeric.exclusiveMinimum, 'instance %r is not strictly greater than (not equal to) %r' % (instance, numeric.exclusiveMinimum)  # noqa: E501

    def visit_number(self, number, *args):
        self.visit_numeric(number, float)

    def visit_integer(self, integer, *args):
        self.visit_numeric(integer, int)

    def visit_string(self, string, *args):
        self.visit_primitive(string, *args)

        instance = self.instance
        if string.maxLength:
            assert len(instance) <= string.maxLength, 'instance %r is not less than, or equal to to %r' % (instance, string.maxLength)  # noqa: E501
        assert len(instance) >= string.minLength, 'instance %r is not greater than, or equal to %r' % (instance, string.minLength)  # noqa: E501
        if string.pattern:
            assert re.match(string.pattern, instance) is not None, 'instance %r does not match the regular expression %r' % (instance, string.pattern)  # noqa: E501

    def visit_array(self, array, *args):
        self.visit_primitive(array, *args)

        instance = self.instance
        items = [array.items] * len(instance) if isinstance(array.items, Component) else array.items  # noqa: E501
        for i, element in enumerate(instance):
            # Determine which subschemas apply to which elements of the array.
            try:
                items[i].accept(ValidationVisitor(element), *args)
            except IndexError:
                # If "items" is an array of schemas, validation succeeds if
                # every instance element at a position greater than the size of
                # "items" validates against "additionalItems".
                #
                # http://json-schema.org/latest/json-schema-validation.html#rfc.section.6.10
                array.additionalItems.accept(ValidationVisitor(element), *args)
        if array.maxItems:
            assert len(instance) <= array.maxItems, 'instance %r is not less than, or equal to %r' % (instance, array.maxItems)  # noqa: E501
        assert len(instance) >= array.minItems, 'instance %r is not greater than, or equal to %r' % (instance, array.minItems)  # noqa: E501
        if array.uniqueItems:
            assert len(set(instance)) == len(instance), 'instance %r contains duplicate elements' % (instance,)  # noqa: E501
        # TODO: contains
        array.contains.accept(self, *args)

    def visit_properties(self, properties, *args):
        instance = self.instance
        additionalProperties = args[0]
        for name, member in instance.items():
            # Validation succeeds if, for each name that appears in both the
            # instance and as a name within this keyword's value.
            try:
                properties[name].accept(ValidationVisitor(member), *args)
            except KeyError:
                # Validation with "additionalProperties" applies only to the
                # child values of instance names that do not match any names in
                # "properties", and do not match any regular expression in
                # "patternProperties".
                #
                # http://json-schema.org/latest/json-schema-validation.html#rfc.section.6.20
                additionalProperties.accept(ValidationVisitor(member), *args)

    def visit_definitions(self, definitions, *args):  # pragma: no cover
        """This keyword plays no role in validation per se. Its role is to
        provide a standardized location for schema authors to inline JSON
        Schemas into a more general schema.

        http://json-schema.org/latest/json-schema-validation.html#rfc.section.7.1
        """
        return

    def visit_object(self, obj, *args):
        self.visit_primitive(obj, *args)

        instance = self.instance
        if obj.maxProperties:
            assert len(instance) <= obj.maxProperties, 'instance %r number of properties is not less than, or equal to %r' % (instance, obj.maxProperties)  # noqa: E501
        assert len(instance) >= obj.minProperties, 'instance %r number of properties is not greater than, or equal to %r' % (instance, obj.minProperties)  # noqa: E501
        for element in obj.required:
            assert element in instance, 'instance %r is missing required property %r' % (instance, element)  # noqa: E501
        obj.properties.accept(self, obj.additionalProperties)

    def visit_reference(self, reference, *args):
        if reference.resolved:  # pragma: no cover
            reference.value.accept(self.instance, *args)

    def visit_union(self, union, *args):
        self.visit_primitive(union, *args)

class ResolveVisitor:

    def __init__(self, context):
        self.context = context

    def visit_empty_schema(self, schema, *args):  # pragma: no cover
        return

    def visit_enumeration(self, enumeration, *args):
        return self.visit_primitive(enumeration, *args)

    def visit_all_of(self, all_of, *args):
        for i, element in enumerate(all_of):
            # Resolve each member recursively.
            element = self.visit_primitive(element, *args)
            all_of[i] = element.accept(self, *args)

    def visit_any_of(self, any_of, *args):
        for i, element in enumerate(any_of):
            # Resolve each member recursively.
            element = self.visit_primitive(element, *args)
            any_of[i] = element.accept(self, *args)

    def visit_one_of(self, one_of, *args):
        for i, element in enumerate(one_of):
            # Resolve each member recursively.
            element = self.visit_primitive(element, *args)
            one_of[i] = element.accept(self, *args)

    def visit_primitive(self, primitive, *args):
        primitive.allOf.accept(self, *args)
        primitive.anyOf.accept(self, *args)
        primitive.oneOf.accept(self, *args)
        primitive.definitions.accept(self, *args)
        return primitive

    def visit_boolean(self, boolean, *args):
        return self.visit_primitive(boolean, *args)

    def visit_null(self, null, *args):
        return self.visit_primitive(null, *args)

    def visit_number(self, number, *args):
        return self.visit_primitive(number, *args)

    def visit_integer(self, integer, *args):
        return self.visit_primitive(integer, *args)

    def visit_string(self, string, *args):
        return self.visit_primitive(string, *args)

    def visit_array(self, array, *args):
        array = self.visit_primitive(array, *args)
        if isinstance(array.items, Component):
            array.items = array.items.accept(self, *args)
        else:
            array.items = [element.accept(self, *args) for element in array.items]  # noqa: E501
        array.additionalItems = array.additionalItems.accept(self, *args)
        array.contains = array.contains.accept(self, *args)
        return array

    def visit_properties(self, properties, *args):
        for name, member in properties.items():
            # Resolve each member recursively.
            member = self.visit_primitive(member, *args)
            properties[name] = member.accept(self, *args)

    def visit_definitions(self, definitions, *args):
        for name, member in definitions.items():
            # Resolve each member recursively.
            member = self.visit_primitive(member, *args)
            definitions[name] = member.accept(self, *args)

    def visit_object(self, obj, *args):
        obj = self.visit_primitive(obj, *args)
        obj.properties.accept(self, *args)
        return obj

    def visit_reference(self, reference, *args):
        if reference.resolved:  # pragma: no cover
            return reference
        definition = reference.address.split('/')[-1]
        schema = self.context['definitions'][definition]
        reference.value = Creator.create(schema.get('type')).unmarshal(schema)
        # Resolve the value returned from the Creator.
        self.visit_primitive(reference.value, *args)
        reference.resolved = True
        return reference

    def visit_union(self, union, *args):
        return self.visit_primitive(union, *args)
<commit_msg>Use Component.accept to resolve reference value<commit_after>import re

from .primitive import Component, Creator, Translator

class SchemaArrayValidationHandler:

    def __init__(self, visitor):
        self.visitor = visitor

    def __call__(self, sequence, *args):
        errors = []
        for element in sequence:
            try:
                element.accept(self.visitor, *args)
            except AssertionError as e:
                errors.append(e.args[0])
                continue
        return errors

class ValidationVisitor:

    def __init__(self, instance):
        self.instance = instance

    def visit_empty_schema(self, schema, *args):  # pragma: no cover
        """Always passes validation."""
        return

    def visit_enumeration(self, enumeration, *args):
        self.visit_primitive(enumeration, *args)
        assert self.instance in enumeration.enum, 'instance %r is not equal to one of the elements %r' % (self.instance, enumeration)  # noqa: E501

    def visit_all_of(self, all_of, *args):
        for element in all_of:
            element.accept(self, *args)

    def visit_any_of(self, any_of, *args):
        if any_of:
            handler = SchemaArrayValidationHandler(self)
            errors = handler(any_of, *args)
            assert len(any_of) != len(errors), ', '.join(errors)

    def visit_one_of(self, one_of, *args):
        if one_of:
            handler = SchemaArrayValidationHandler(self)
            errors = handler(one_of, *args)
            assert len(errors) == 1, ', '.join(errors)

    def visit_primitive(self, primitive, *args):
        instance = self.instance
        if primitive.const is not None:
            assert instance == primitive.const, 'instance %r is not equal to %r' % (instance, primitive.const)  # noqa: E501
        if primitive.type is not None:
            assert {
                str: lambda instance: Translator.translate(instance).__name__.lower() == primitive.type,  # noqa: E501
                list: lambda instance: Translator.translate(instance).__name__.lower() in primitive.type,  # noqa: E501
            }[primitive.type.__class__](instance), 'instance %r is not in any of the sets listed %r' % (instance, primitive.type)  # noqa: E501
        primitive.allOf.accept(self, *args)
        primitive.anyOf.accept(self, *args)
        primitive.oneOf.accept(self, *args)

    def visit_boolean(self, boolean, *args):
        self.visit_primitive(boolean, *args)

    def visit_null(self, null, *args):
        self.visit_primitive(null, *args)

    def visit_numeric(self, numeric, *args):
        self.visit_primitive(numeric, *args)

        cls = args[0]
        instance = cls(self.instance)
        assert float(instance / numeric.multipleOf).is_integer(), 'instance %r division by %r is not an integer' % (instance, numeric.multipleOf)  # noqa: E501
        if numeric.maximum is not None:
            assert instance <= numeric.maximum, 'instance %r is not less than or exactly equal to %r' % (instance, numeric.maximum)  # noqa: E501
        if numeric.exclusiveMaximum is not None:
            assert instance < numeric.exclusiveMaximum, 'instance %r is not strictly less than (not equal to) %r' % (instance, numeric.exclusiveMaximum)  # noqa: E501
        if numeric.minimum is not None:
            assert instance >= numeric.minimum, 'instance %r is not greater than or exactly equal to %r' % (instance, numeric.minimum)  # noqa: E501
        if numeric.exclusiveMinimum is not None:
            assert instance > numeric.exclusiveMinimum, 'instance %r is not strictly greater than (not equal to) %r' % (instance, numeric.exclusiveMinimum)  # noqa: E501

    def visit_number(self, number, *args):
        self.visit_numeric(number, float)

    def visit_integer(self, integer, *args):
        self.visit_numeric(integer, int)

    def visit_string(self, string, *args):
        self.visit_primitive(string, *args)

        instance = self.instance
        if string.maxLength:
            assert len(instance) <= string.maxLength, 'instance %r is not less than, or equal to to %r' % (instance, string.maxLength)  # noqa: E501
        assert len(instance) >= string.minLength, 'instance %r is not greater than, or equal to %r' % (instance, string.minLength)  # noqa: E501
        if string.pattern:
            assert re.match(string.pattern, instance) is not None, 'instance %r does not match the regular expression %r' % (instance, string.pattern)  # noqa: E501

    def visit_array(self, array, *args):
        self.visit_primitive(array, *args)

        instance = self.instance
        items = [array.items] * len(instance) if isinstance(array.items, Component) else array.items  # noqa: E501
        for i, element in enumerate(instance):
            # Determine which subschemas apply to which elements of the array.
            try:
                items[i].accept(ValidationVisitor(element), *args)
            except IndexError:
                # If "items" is an array of schemas, validation succeeds if
                # every instance element at a position greater than the size of
                # "items" validates against "additionalItems".
                #
                # http://json-schema.org/latest/json-schema-validation.html#rfc.section.6.10
                array.additionalItems.accept(ValidationVisitor(element), *args)
        if array.maxItems:
            assert len(instance) <= array.maxItems, 'instance %r is not less than, or equal to %r' % (instance, array.maxItems)  # noqa: E501
        assert len(instance) >= array.minItems, 'instance %r is not greater than, or equal to %r' % (instance, array.minItems)  # noqa: E501
        if array.uniqueItems:
            assert len(set(instance)) == len(instance), 'instance %r contains duplicate elements' % (instance,)  # noqa: E501
        # TODO: contains
        array.contains.accept(self, *args)

    def visit_properties(self, properties, *args):
        instance = self.instance
        additionalProperties = args[0]
        for name, member in instance.items():
            # Validation succeeds if, for each name that appears in both the
            # instance and as a name within this keyword's value.
            try:
                properties[name].accept(ValidationVisitor(member), *args)
            except KeyError:
                # Validation with "additionalProperties" applies only to the
                # child values of instance names that do not match any names in
                # "properties", and do not match any regular expression in
                # "patternProperties".
                #
                # http://json-schema.org/latest/json-schema-validation.html#rfc.section.6.20
                additionalProperties.accept(ValidationVisitor(member), *args)

    def visit_definitions(self, definitions, *args):  # pragma: no cover
        """This keyword plays no role in validation per se. Its role is to
        provide a standardized location for schema authors to inline JSON
        Schemas into a more general schema.

        http://json-schema.org/latest/json-schema-validation.html#rfc.section.7.1
        """
        return

    def visit_object(self, obj, *args):
        self.visit_primitive(obj, *args)

        instance = self.instance
        if obj.maxProperties:
            assert len(instance) <= obj.maxProperties, 'instance %r number of properties is not less than, or equal to %r' % (instance, obj.maxProperties)  # noqa: E501
        assert len(instance) >= obj.minProperties, 'instance %r number of properties is not greater than, or equal to %r' % (instance, obj.minProperties)  # noqa: E501
        for element in obj.required:
            assert element in instance, 'instance %r is missing required property %r' % (instance, element)  # noqa: E501
        obj.properties.accept(self, obj.additionalProperties)

    def visit_reference(self, reference, *args):
        if reference.resolved:  # pragma: no cover
            reference.value.accept(self.instance, *args)

    def visit_union(self, union, *args):
        self.visit_primitive(union, *args)

class ResolveVisitor:

    def __init__(self, context):
        self.context = context

    def visit_empty_schema(self, schema, *args):  # pragma: no cover
        return

    def visit_enumeration(self, enumeration, *args):
        return self.visit_primitive(enumeration, *args)

    def visit_all_of(self, all_of, *args):
        for i, element in enumerate(all_of):
            # Resolve each member recursively.
            element = self.visit_primitive(element, *args)
            all_of[i] = element.accept(self, *args)

    def visit_any_of(self, any_of, *args):
        for i, element in enumerate(any_of):
            # Resolve each member recursively.
            element = self.visit_primitive(element, *args)
            any_of[i] = element.accept(self, *args)

    def visit_one_of(self, one_of, *args):
        for i, element in enumerate(one_of):
            # Resolve each member recursively.
            element = self.visit_primitive(element, *args)
            one_of[i] = element.accept(self, *args)

    def visit_primitive(self, primitive, *args):
        primitive.allOf.accept(self, *args)
        primitive.anyOf.accept(self, *args)
        primitive.oneOf.accept(self, *args)
        primitive.definitions.accept(self, *args)
        return primitive

    def visit_boolean(self, boolean, *args):
        return self.visit_primitive(boolean, *args)

    def visit_null(self, null, *args):
        return self.visit_primitive(null, *args)

    def visit_number(self, number, *args):
        return self.visit_primitive(number, *args)

    def visit_integer(self, integer, *args):
        return self.visit_primitive(integer, *args)

    def visit_string(self, string, *args):
        return self.visit_primitive(string, *args)

    def visit_array(self, array, *args):
        array = self.visit_primitive(array, *args)
        if isinstance(array.items, Component):
            array.items = array.items.accept(self, *args)
        else:
            array.items = [element.accept(self, *args) for element in array.items]  # noqa: E501
        array.additionalItems = array.additionalItems.accept(self, *args)
        array.contains = array.contains.accept(self, *args)
        return array

    def visit_properties(self, properties, *args):
        for name, member in properties.items():
            # Resolve each member recursively.
            member = self.visit_primitive(member, *args)
            properties[name] = member.accept(self, *args)

    def visit_definitions(self, definitions, *args):
        for name, member in definitions.items():
            # Resolve each member recursively.
            member = self.visit_primitive(member, *args)
            definitions[name] = member.accept(self, *args)

    def visit_object(self, obj, *args):
        obj = self.visit_primitive(obj, *args)
        obj.properties.accept(self, *args)
        return obj

    def visit_reference(self, reference, *args):
        if reference.resolved:  # pragma: no cover
            return reference
        definition = reference.address.split('/')[-1]
        schema = self.context['definitions'][definition]
        reference.value = Creator.create(schema.get('type')).unmarshal(schema)
        # Resolve the value returned from the Creator.
        reference.value.accept(self, *args)
        reference.resolved = True
        return reference

    def visit_union(self, union, *args):
        return self.visit_primitive(union, *args)
<|endoftext|>


<commit_before>
import numpy as np
from scipy.sparse.linalg import LinearOperator
from scipy.sparse.linalg import cg
from rlscore.utilities import sparse_kronecker_multiplication_tools_python
from rlscore.utilities import array_tools

TRAIN_LABELS = 'train_labels'
CALLBACK_FUNCTION = 'callback'

class KronSVM(object):

    '''def __init__(self, train_labels, label_row_inds, label_col_inds, regparam=1.0):
        self.Y = array_tools.as_labelmatrix(train_labels)
        self.regparam = regparam
        self.label_row_inds = label_row_inds
        self.label_col_inds = label_col_inds
        self.results = {}'''

    def __init__(self, **kwargs):
        self.resource_pool = kwargs
        Y = kwargs[TRAIN_LABELS]
        self.label_row_inds = np.array(kwargs["label_row_inds"], dtype = np.int32)
        self.label_col_inds = np.array(kwargs["label_col_inds"], dtype = np.int32)
        Y = array_tools.as_array(Y)
        self.Y = Y
        self.trained = False
        if kwargs.has_key("regparam"):
            self.regparam = kwargs["regparam"]
        else:
            self.regparam = 0.
        if kwargs.has_key("newton_iterations"):
            self.newton_it = kwargs["newton_iterations"]
        else:
            self.newton_it = 100
        if kwargs.has_key("cg_iterations"):
            self.cg_it = kwargs["cg_iterations"]
        else:
            self.cg_it = 50
        if kwargs.has_key("epsilon"):
            self.epsilon = kwargs["epsilon"]
        else:
            self.epsilon = 0.000001

    def createLearner(cls, **kwargs):
        learner = cls(**kwargs)
        return learner
    createLearner = classmethod(createLearner)

    def train(self):
        self.solve_linear(self.regparam)

    def solve_linear(self, regparam):
        lamb = regparam
        X1 = self.resource_pool['xmatrix1']
        X2 = self.resource_pool['xmatrix2']
        Y = self.Y
        rowind = self.label_row_inds
        colind = self.label_col_inds
        fdim = X1.shape[1]*X2.shape[1]
        def func(v):
            #REPLACE
            #P = np.dot(X,v)
            P = sparse_kronecker_multiplication_tools_python.x_gets_subset_of_A_kron_B_times_v(v, X2, X1.T, colind, rowind)
            z = (1. - Y*P)
            z = np.where(z>0, z, 0)
            return np.dot(z,z)+lamb*np.dot(v,v)
        def gradient(v):
            #REPLACE
            #P = np.dot(X,v)
            #P = vecProd(X1, X2, v)
            P = sparse_kronecker_multiplication_tools_python.x_gets_subset_of_A_kron_B_times_v(v, X2, X1.T, colind, rowind)
            z = (1. - Y*P)
            z = np.where(z>0, z, 0)
            sv = np.nonzero(z)[0]
            #map to rows and cols
            rows = rowind[sv]
            cols = colind[sv]
            #A = -2*np.dot(X[sv].T, Y[sv])
            A = -2 * sparse_kronecker_multiplication_tools_python.x_gets_A_kron_B_times_sparse_v(Y[sv], X1.T, X2, rows, cols)
            A = A.reshape(X2.shape[1], X1.shape[1]).T.ravel()
            #B = 2 * np.dot(X[sv].T, np.dot(X[sv],v))
            v_after = sparse_kronecker_multiplication_tools_python.x_gets_subset_of_A_kron_B_times_v(v, X2, X1.T, cols, rows)
            v_after = 2 * sparse_kronecker_multiplication_tools_python.x_gets_A_kron_B_times_sparse_v(v_after, X1.T, X2, rows, cols)
            B = v_after.reshape(X2.shape[1], X1.shape[1]).T.ravel()
            #print "FOOOBAAR"
            return A + B + lamb*v
            #return -2*np.dot(X[sv].T,Y[sv]) + 2 * np.dot(X[sv].T, np.dot(X[sv],v)) + lamb*v
        def hessian(v, p):
            #P = np.dot(X,v)
            #P = vecProd(X1, X2, v)
            P = sparse_kronecker_multiplication_tools_python.x_gets_subset_of_A_kron_B_times_v(v, X2, X1.T, colind, rowind)
            z = (1. - Y*P)
            z = np.where(z>0, z, 0)
            sv = np.nonzero(z)[0]
            #map to rows and cols
            rows = rowind[sv]
            cols = colind[sv]
            p_after = sparse_kronecker_multiplication_tools_python.x_gets_subset_of_A_kron_B_times_v(p, X2, X1.T, cols, rows)
            p_after = sparse_kronecker_multiplication_tools_python.x_gets_A_kron_B_times_sparse_v(p_after, X1.T, X2, rows, cols)
            p_after = p_after.reshape(X2.shape[1], X1.shape[1]).T.ravel()
            return 2 * p_after + lamb*p
            #return 2 * np.dot(X[sv].T, np.dot(X[sv],p)) + lamb*p
        w = np.zeros(fdim)
        #np.random.seed(1)
        #w = np.random.random(fdim)
        def mv(v):
            return hessian(w, v)
        for i in range(self.newton_it):
            g = gradient(w)
            G = LinearOperator((fdim, fdim), matvec=mv, dtype=np.float64)
            w_new = cg(G, g, maxiter=self.cg_it)[0]
            #print w_new
            #assert False
            r = G*w_new - g
            e_rel = np.linalg.norm(r)/np.linalg.norm(g)
            w = w - w_new
            #print w
            if np.linalg.norm(w_new) == 0 or e_rel < self.epsilon:
                break
        print "outer iterations", i
        self.model = LinearPairwiseModel(w, X1.shape[1], X2.shape[1])

    def getModel(self):
        if not hasattr(self, "model"):
            self.model = LinearPairwiseModel(self.W, self.X1.shape[1], self.X2.shape[1])
        return self.model

class LinearPairwiseModel(object):

    def __init__(self, W, dim1, dim2):
        """Initializes the linear model
        @param W: primal coefficient matrix
        @type W: numpy matrix"""
        self.W = W
        self.dim1, self.dim2 = dim1, dim2

    def predictWithDataMatrices(self, X1pred, X2pred):
        """Computes predictions for test examples.

        Parameters
        ----------
        X1pred: {array-like, sparse matrix}, shape = [n_samples1, n_features1]
            the first part of the test data matrix
        X2pred: {array-like, sparse matrix}, shape = [n_samples2, n_features2]
            the second part of the test data matrix

        Returns
        ----------
        P: array, shape = [n_samples1, n_samples2]
            predictions
        """
        P = np.dot(np.dot(X1pred, self.W), X2pred.T)
        return P

    def predictWithDataMatricesAlt(self, X1pred, X2pred, row_inds = None, col_inds = None):
        if row_inds == None:
            P = np.dot(np.dot(X1pred, self.W), X2pred.T)
            P = P.reshape(X1pred.shape[0] * X2pred.shape[0], 1, order = 'F')
        else:
            P = sparse_kronecker_multiplication_tools_python.x_gets_subset_of_A_kron_B_times_v(self.W.reshape((self.W.shape[0] * self.W.shape[1], 1), order = 'F'), X1pred, X2pred.T, np.array(row_inds, dtype=np.int32), np.array(col_inds, dtype=np.int32))
            #P = X1pred * self.W * X2pred.T
        return P

<commit_msg>Cleaning commented garbage lines from code<commit_after>
import numpy as np
from scipy.sparse.linalg import LinearOperator
from scipy.sparse.linalg import cg
from rlscore.utilities import sparse_kronecker_multiplication_tools_python
from rlscore.utilities import array_tools

TRAIN_LABELS = 'train_labels'
CALLBACK_FUNCTION = 'callback'

class KronSVM(object):

    def __init__(self, **kwargs):
        self.resource_pool = kwargs
        Y = kwargs[TRAIN_LABELS]
        self.label_row_inds = np.array(kwargs["label_row_inds"], dtype = np.int32)
        self.label_col_inds = np.array(kwargs["label_col_inds"], dtype = np.int32)
        Y = array_tools.as_array(Y)
        self.Y = Y
        self.trained = False
        if kwargs.has_key("regparam"):
            self.regparam = kwargs["regparam"]
        else:
            self.regparam = 0.
        if kwargs.has_key("newton_iterations"):
            self.newton_it = kwargs["newton_iterations"]
        else:
            self.newton_it = 100
        if kwargs.has_key("cg_iterations"):
            self.cg_it = kwargs["cg_iterations"]
        else:
            self.cg_it = 50
        if kwargs.has_key("epsilon"):
            self.epsilon = kwargs["epsilon"]
        else:
            self.epsilon = 0.000001

    def createLearner(cls, **kwargs):
        learner = cls(**kwargs)
        return learner
    createLearner = classmethod(createLearner)

    def train(self):
        self.solve_linear(self.regparam)

    def solve_linear(self, regparam):
        lamb = regparam
        X1 = self.resource_pool['xmatrix1']
        X2 = self.resource_pool['xmatrix2']
        Y = self.Y
        rowind = self.label_row_inds
        colind = self.label_col_inds
        fdim = X1.shape[1]*X2.shape[1]
        def func(v):
            P = sparse_kronecker_multiplication_tools_python.x_gets_subset_of_A_kron_B_times_v(v, X2, X1.T, colind, rowind)
            z = (1. - Y*P)
            z = np.where(z>0, z, 0)
            return np.dot(z,z)+lamb*np.dot(v,v)
        def gradient(v):
            P = sparse_kronecker_multiplication_tools_python.x_gets_subset_of_A_kron_B_times_v(v, X2, X1.T, colind, rowind)
            z = (1. - Y*P)
            z = np.where(z>0, z, 0)
            sv = np.nonzero(z)[0]
            #map to rows and cols
            rows = rowind[sv]
            cols = colind[sv]
            A = -2 * sparse_kronecker_multiplication_tools_python.x_gets_A_kron_B_times_sparse_v(Y[sv], X1.T, X2, rows, cols)
            A = A.reshape(X2.shape[1], X1.shape[1]).T.ravel()
            v_after = sparse_kronecker_multiplication_tools_python.x_gets_subset_of_A_kron_B_times_v(v, X2, X1.T, cols, rows)
            v_after = 2 * sparse_kronecker_multiplication_tools_python.x_gets_A_kron_B_times_sparse_v(v_after, X1.T, X2, rows, cols)
            B = v_after.reshape(X2.shape[1], X1.shape[1]).T.ravel()
            return A + B + lamb*v
        def hessian(v, p):
            P = sparse_kronecker_multiplication_tools_python.x_gets_subset_of_A_kron_B_times_v(v, X2, X1.T, colind, rowind)
            z = (1. - Y*P)
            z = np.where(z>0, z, 0)
            sv = np.nonzero(z)[0]
            #map to rows and cols
            rows = rowind[sv]
            cols = colind[sv]
            p_after = sparse_kronecker_multiplication_tools_python.x_gets_subset_of_A_kron_B_times_v(p, X2, X1.T, cols, rows)
            p_after = sparse_kronecker_multiplication_tools_python.x_gets_A_kron_B_times_sparse_v(p_after, X1.T, X2, rows, cols)
            p_after = p_after.reshape(X2.shape[1], X1.shape[1]).T.ravel()
            return 2 * p_after + lamb*p
        w = np.zeros(fdim)
        def mv(v):
            return hessian(w, v)
        for i in range(self.newton_it):
            g = gradient(w)
            G = LinearOperator((fdim, fdim), matvec=mv, dtype=np.float64)
            w_new = cg(G, g, maxiter=self.cg_it)[0]
            r = G*w_new - g
            e_rel = np.linalg.norm(r)/np.linalg.norm(g)
            w = w - w_new
            #print w
            if np.linalg.norm(w_new) == 0 or e_rel < self.epsilon:
                break
        print "outer iterations", i
        self.model = LinearPairwiseModel(w, X1.shape[1], X2.shape[1])

    def getModel(self):
        if not hasattr(self, "model"):
            self.model = LinearPairwiseModel(self.W, self.X1.shape[1], self.X2.shape[1])
        return self.model

class LinearPairwiseModel(object):

    def __init__(self, W, dim1, dim2):
        """Initializes the linear model
        @param W: primal coefficient matrix
        @type W: numpy matrix"""
        self.W = W
        self.dim1, self.dim2 = dim1, dim2

    def predictWithDataMatrices(self, X1pred, X2pred):
        """Computes predictions for test examples.

        Parameters
        ----------
        X1pred: {array-like, sparse matrix}, shape = [n_samples1, n_features1]
            the first part of the test data matrix
        X2pred: {array-like, sparse matrix}, shape = [n_samples2, n_features2]
            the second part of the test data matrix

        Returns
        ----------
        P: array, shape = [n_samples1, n_samples2]
            predictions
        """
        P = np.dot(np.dot(X1pred, self.W), X2pred.T)
        return P

    def predictWithDataMatricesAlt(self, X1pred, X2pred, row_inds = None, col_inds = None):
        if row_inds == None:
            P = np.dot(np.dot(X1pred, self.W), X2pred.T)
            P = P.reshape(X1pred.shape[0] * X2pred.shape[0], 1, order = 'F')
        else:
            P = sparse_kronecker_multiplication_tools_python.x_gets_subset_of_A_kron_B_times_v(self.W.reshape((self.W.shape[0] * self.W.shape[1], 1), order = 'F'), X1pred, X2pred.T, np.array(row_inds, dtype=np.int32), np.array(col_inds, dtype=np.int32))
            #P = X1pred * self.W * X2pred.T
        return P

<|endoftext|>


<commit_before>#!/usr/bin/python
# @lint-avoid-python-3-compatibility-imports
#
# dcsnoop   Trace directory entry cache (dcache) lookups.
#           For Linux, uses BCC, eBPF. Embedded C.
#
# USAGE: dcsnoop [-h] [-a]
#
# By default, this traces every failed dcache lookup, and shows the process
# performing the lookup and the filename requested. A -a option can be used
# to show all lookups, not just failed ones.
#
# This uses kernel dynamic tracing of the d_lookup() function, and will need
# to be modified to match kernel changes.
#
# Also see dcstat(8), for per-second summaries.
#
# Copyright 2016 Netflix, Inc.
# Licensed under the Apache License, Version 2.0 (the "License")
#
# 09-Feb-2016   Brendan Gregg   Created this.

from __future__ import print_function
from bcc import BPF
import argparse
import re

# arguments
examples = """examples:
    ./dcsnoop           # trace failed dcache lookups
    ./dcsnoop -a        # trace all dcache lookups
"""
parser = argparse.ArgumentParser(
    description="Trace directory entry cache (dcache) lookups",
    formatter_class=argparse.RawDescriptionHelpFormatter,
    epilog=examples)
parser.add_argument("-a", "--all", action="store_true",
    help="trace all lookups (default is fails only)")
args = parser.parse_args()

# define BPF program
bpf_text = """
#include <uapi/linux/ptrace.h>
#include <linux/fs.h>
#include <linux/sched.h>

#define MAX_FILE_LEN  64

struct entry_t {
    char name[MAX_FILE_LEN];
};

BPF_HASH(entrybypid, u32, struct entry_t);

/* from fs/namei.c: */
struct nameidata {
        struct path     path;
        struct qstr     last;
        // [...]
};

int trace_fast(struct pt_regs *ctx, struct nameidata *nd, struct path *path)
{
    bpf_trace_printk("R %s\\n", nd->last.name);
    return 1;
}

int kprobe__d_lookup(struct pt_regs *ctx, const struct dentry *parent,
    const struct qstr *name)
{
    u32 pid = bpf_get_current_pid_tgid();
    struct entry_t entry = {};
    const char *fname = name->name;
    if (fname) {
        bpf_probe_read(&entry.name, sizeof(entry.name), (void *)fname);
    }
    entrybypid.update(&pid, &entry);
    return 0;
}

int kretprobe__d_lookup(struct pt_regs *ctx)
{
    u32 pid = bpf_get_current_pid_tgid();
    struct entry_t *ep;
    ep = entrybypid.lookup(&pid);
    if (ep == 0) {
        return 0;   // missed entry
    }
    if (PT_REGS_RC(ctx) == 0) {
        bpf_trace_printk("M %s\\n", ep->name);
    }
    entrybypid.delete(&pid);
    return 0;
}
"""

# initialize BPF
b = BPF(text=bpf_text)
if args.all:
    b.attach_kprobe(event="lookup_fast", fn_name="trace_fast")

# header
print("%-11s %-6s %-16s %1s %s" % ("TIME(s)", "PID", "COMM", "T", "FILE"))

start_ts = 0

# format output
while 1:
    (task, pid, cpu, flags, ts, msg) = b.trace_fields()
    try:
        (type, file) = msg.split(" ", 1)
    except ValueError:
        continue

    if start_ts == 0:
        start_ts = ts

    print("%-11.6f %-6s %-16s %1s %s" % (ts - start_ts, pid, task, type, file))
<commit_msg>dcsnoop: use PERF_EVENT_OUTPUT<commit_after>#!/usr/bin/python
# @lint-avoid-python-3-compatibility-imports
#
# dcsnoop   Trace directory entry cache (dcache) lookups.
#           For Linux, uses BCC, eBPF. Embedded C.
#
# USAGE: dcsnoop [-h] [-a]
#
# By default, this traces every failed dcache lookup, and shows the process
# performing the lookup and the filename requested. A -a option can be used
# to show all lookups, not just failed ones.
#
# This uses kernel dynamic tracing of the d_lookup() function, and will need
# to be modified to match kernel changes.
#
# Also see dcstat(8), for per-second summaries.
#
# Copyright 2016 Netflix, Inc.
# Licensed under the Apache License, Version 2.0 (the "License")
#
# 09-Feb-2016   Brendan Gregg   Created this.

from __future__ import print_function
from bcc import BPF
import argparse
import ctypes as ct
import re
import time

# arguments
examples = """examples:
    ./dcsnoop           # trace failed dcache lookups
    ./dcsnoop -a        # trace all dcache lookups
"""
parser = argparse.ArgumentParser(
    description="Trace directory entry cache (dcache) lookups",
    formatter_class=argparse.RawDescriptionHelpFormatter,
    epilog=examples)
parser.add_argument("-a", "--all", action="store_true",
    help="trace all lookups (default is fails only)")
args = parser.parse_args()

# define BPF program
bpf_text = """
#include <uapi/linux/ptrace.h>
#include <linux/fs.h>
#include <linux/sched.h>

#define MAX_FILE_LEN  64

enum lookup_type {
    LOOKUP_MISS,
    LOOKUP_REFERENCE,
};

struct entry_t {
    char name[MAX_FILE_LEN];
};

BPF_HASH(entrybypid, u32, struct entry_t);

struct data_t {
    u32 pid;
    enum lookup_type type;
    char comm[TASK_COMM_LEN];
    char filename[MAX_FILE_LEN];
};

BPF_PERF_OUTPUT(events);

/* from fs/namei.c: */
struct nameidata {
        struct path     path;
        struct qstr     last;
        // [...]
};

static inline
void submit_event(struct pt_regs *ctx, void *name, int type, u32 pid)
{
    struct data_t data = {
        .pid = pid,
        .type = type,
    };
    bpf_get_current_comm(&data.comm, sizeof(data.comm));
    bpf_probe_read(&data.filename, sizeof(data.filename), name);
    events.perf_submit(ctx, &data, sizeof(data));
}

int trace_fast(struct pt_regs *ctx, struct nameidata *nd, struct path *path)
{
    u32 pid = bpf_get_current_pid_tgid();
    submit_event(ctx, (void *)nd->last.name, LOOKUP_REFERENCE, pid);
    return 1;
}

int kprobe__d_lookup(struct pt_regs *ctx, const struct dentry *parent,
    const struct qstr *name)
{
    u32 pid = bpf_get_current_pid_tgid();
    struct entry_t entry = {};
    const char *fname = name->name;
    if (fname) {
        bpf_probe_read(&entry.name, sizeof(entry.name), (void *)fname);
    }
    entrybypid.update(&pid, &entry);
    return 0;
}

int kretprobe__d_lookup(struct pt_regs *ctx)
{
    u32 pid = bpf_get_current_pid_tgid();
    struct entry_t *ep;
    ep = entrybypid.lookup(&pid);
    if (ep == 0 || PT_REGS_RC(ctx) != 0) {
        return 0;   // missed entry or lookup didn't fail
    }
    submit_event(ctx, (void *)ep->name, LOOKUP_MISS, pid);
    entrybypid.delete(&pid);
    return 0;
}
"""

TASK_COMM_LEN = 16  # linux/sched.h
MAX_FILE_LEN = 64  # see inline C

class Data(ct.Structure):
    _fields_ = [
        ("pid", ct.c_uint),
        ("type", ct.c_int),
        ("comm", ct.c_char * TASK_COMM_LEN),
        ("filename", ct.c_char * MAX_FILE_LEN),
    ]

# initialize BPF
b = BPF(text=bpf_text)
if args.all:
    b.attach_kprobe(event="lookup_fast", fn_name="trace_fast")

mode_s = {
    0: 'M',
    1: 'R',
}

start_ts = time.time()

def print_event(cpu, data, size):
    event = ct.cast(data, ct.POINTER(Data)).contents
    print("%-11.6f %-6d %-16s %1s %s" % (
            time.time() - start_ts, event.pid, event.comm, mode_s[event.type],
            event.filename))

# header
print("%-11s %-6s %-16s %1s %s" % ("TIME(s)", "PID", "COMM", "T", "FILE"))

b["events"].open_perf_buffer(print_event)
while 1:
    b.kprobe_poll()
<|endoftext|>


<commit_before># Copyright 2018, The TensorFlow Federated Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
from unittest import mock

from absl.testing import parameterized
import numpy as np
import tensorflow as tf

from tensorflow_federated.python.common_libs import test_utils
from tensorflow_federated.python.core.api import test_case
from tensorflow_federated.python.core.backends.native import execution_contexts
from tensorflow_federated.python.learning import client_weight_lib
from tensorflow_federated.python.learning import federated_sgd
from tensorflow_federated.python.learning import keras_utils
from tensorflow_federated.python.learning import model_examples
from tensorflow_federated.python.learning import model_utils
from tensorflow_federated.python.learning.framework import dataset_reduce

class FederatedSgdTest(test_case.TestCase, parameterized.TestCase):

  def dataset(self):
    # Create a dataset with 4 examples:
    dataset = tf.data.Dataset.from_tensor_slices(
        model_examples.LinearRegression.make_batch(
            x=[[1.0, 0.0], [1.0, 0.0], [1.0, 0.0], [1.0, 0.0]],
            y=[[1.0], [1.0], [1.0], [1.0]]))
    # Repeat the dataset 2 times with batches of 3 examples,
    # producing 3 minibatches (the last one with only 2 examples).
    # Note that `batch` is required for this dataset to be useable,
    # as it adds the batch dimension which is expected by the model.
    return dataset.repeat(2).batch(3)

  def model(self):
    return model_examples.LinearRegression(feature_dim=2)

  def initial_weights(self):
    return model_utils.ModelWeights(
        trainable=[
            tf.constant([[0.0], [0.0]]),
            tf.constant(0.0),
        ],
        non_trainable=[0.0])

  @parameterized.named_parameters(
      ('non-simulation_weighted', False, True),
      ('non-simulation_unweighted', False, False),
      ('simulation_weighted', True, True),
      ('simulation_unweighted', True, False),
  )
  @test_utils.skip_test_for_multi_gpu
  def test_client_tf(self, simulation, weighted):
    model = self.model()
    dataset = self.dataset()
    if weighted:
      client_weighting = client_weight_lib.ClientWeighting.NUM_EXAMPLES
    else:
      client_weighting = client_weight_lib.ClientWeighting.UNIFORM
    client_tf = federated_sgd.ClientSgd(
        model,
        client_weighting=client_weighting,
        use_experimental_simulation_loop=simulation)
    client_outputs = self.evaluate(client_tf(dataset, self.initial_weights()))

    # Both trainable parameters should have gradients, and we don't return the
    # non-trainable 'c'. Model deltas for squared error:
    self.assertAllClose(client_outputs.weights_delta, [[[1.0], [0.0]], 1.0])
    if weighted:
      self.assertAllClose(client_outputs.weights_delta_weight, 8.0)
    else:
      self.assertAllClose(client_outputs.weights_delta_weight, 1.0)

    self.assertDictContainsSubset(
        client_outputs.model_output, {
            'num_examples': 8,
            'num_examples_float': 8.0,
            'num_batches': 3,
            'loss': 0.5,
        })
    self.assertEqual(client_outputs.optimizer_output['has_non_finite_delta'], 0)

  @parameterized.named_parameters(('_inf', np.inf), ('_nan', np.nan))
  def test_non_finite_aggregation(self, bad_value):
    model = self.model()
    dataset = self.dataset()
    client_tf = federated_sgd.ClientSgd(model)
    init_weights = self.initial_weights()
    init_weights.trainable[1] = bad_value
    client_outputs = client_tf(dataset, init_weights)
    self.assertEqual(self.evaluate(client_outputs.weights_delta_weight), 0.0)
    self.assertAllClose(
        self.evaluate(client_outputs.weights_delta), [[[0.0], [0.0]], 0.0])

  @parameterized.named_parameters(('non-simulation', False),
                                  ('simulation', True))
  @mock.patch.object(
      dataset_reduce,
      '_dataset_reduce_fn',
      wraps=dataset_reduce._dataset_reduce_fn)
  @test_utils.skip_test_for_multi_gpu
  def test_client_tf_dataset_reduce_fn(self, simulation, mock_method):
    model = self.model()
    dataset = self.dataset()
    client_tf = federated_sgd.ClientSgd(
        model, use_experimental_simulation_loop=simulation)
    client_tf(dataset, self.initial_weights())
    if simulation:
      mock_method.assert_not_called()
    else:
      mock_method.assert_called()

class FederatedSGDTffTest(test_case.TestCase, parameterized.TestCase):

  @parameterized.named_parameters([
      ('unweighted', client_weight_lib.ClientWeighting.UNIFORM),
      ('example_weighted', client_weight_lib.ClientWeighting.NUM_EXAMPLES),
      ('custom_weighted', lambda _: tf.constant(1.5)),
  ])
  @test_utils.skip_test_for_multi_gpu
  def test_orchestration_execute(self, client_weighting):
    iterative_process = federated_sgd.build_federated_sgd_process(
        model_fn=model_examples.LinearRegression,
        client_weighting=client_weighting)

    # Some data points along [x_1 + 2*x_2 + 3 = y], expecting to learn
    # kernel = [1, 2], bias = [3].
    ds1 = tf.data.Dataset.from_tensor_slices(
        collections.OrderedDict(
            x=[[0.0, 0.0], [0.0, 1.0]],
            y=[[3.0], [5.0]],
        )).batch(2)
    ds2 = tf.data.Dataset.from_tensor_slices(
        collections.OrderedDict(
            x=[[1.0, 2.0], [3.0, 4.0], [1.0, 0.0], [-1.0, -1.0]],
            y=[[8.0], [14.0], [4.00], [0.0]],
        )).batch(2)
    federated_ds = [ds1, ds2]

    server_state = iterative_process.initialize()

    prev_loss = np.inf
    num_iterations = 3
    for _ in range(num_iterations):
      server_state, metric_outputs = iterative_process.next(
          server_state, federated_ds)
      train_metrics = metric_outputs['train']
      self.assertEqual(train_metrics['num_examples'],
                       num_iterations * len(federated_ds))
      loss = train_metrics['loss']
      self.assertLess(loss, prev_loss)
      prev_loss = loss

  @parameterized.named_parameters([
      ('functional_model',
       model_examples.build_linear_regression_keras_functional_model),
      ('sequential_model',
       model_examples.build_linear_regression_keras_sequential_model),
  ])
  @test_utils.skip_test_for_multi_gpu
  def test_orchestration_execute_from_keras(self, build_keras_model_fn):
    # Some data points along [x_1 + 2*x_2 + 3 = y], expecting to learn
    # kernel = [1, 2], bias = [3].
    ds1 = tf.data.Dataset.from_tensor_slices(
        collections.OrderedDict(
            x=[[0.0, 0.0], [0.0, 1.0]],
            y=[[3.0], [5.0]],
        )).batch(2)
    ds2 = tf.data.Dataset.from_tensor_slices(
        collections.OrderedDict(
            x=[[1.0, 2.0], [3.0, 4.0], [1.0, 0.0], [-1.0, -1.0]],
            y=[[8.0], [14.0], [4.00], [0.0]],
        )).batch(2)
    federated_ds = [ds1, ds2]

    def model_fn():
      # Note: we don't compile with an optimizer here; FedSGD does not use it.
      keras_model = build_keras_model_fn(feature_dims=2)
      return keras_utils.from_keras_model(
          keras_model,
          input_spec=ds1.element_spec,
          loss=tf.keras.losses.MeanSquaredError())

    iterative_process = federated_sgd.build_federated_sgd_process(
        model_fn=model_fn)

    server_state = iterative_process.initialize()
    prev_loss = np.inf
    for _ in range(0):
      server_state, metrics = iterative_process.next(server_state, federated_ds)
      self.assertLess(metrics.train.loss, prev_loss)
      prev_loss = metrics.train.loss

  @test_utils.skip_test_for_multi_gpu
  def test_execute_empty_data(self):
    iterative_process = federated_sgd.build_federated_sgd_process(
        model_fn=model_examples.LinearRegression)

    # Results in empty dataset with correct types and shapes.
    ds = tf.data.Dataset.from_tensor_slices(
        collections.OrderedDict(x=[[1.0, 2.0]], y=[[5.0]])).batch(
            5, drop_remainder=True)  # No batches of size 5 can be created.
    federated_ds = [ds] * 2

    server_state = iterative_process.initialize()
    first_state, metric_outputs = iterative_process.next(
        server_state, federated_ds)
    self.assertAllClose(
        list(first_state.model.trainable), [[[0.0], [0.0]], 0.0])
    self.assertEqual(
        list(metric_outputs.keys()),
        ['broadcast', 'aggregation', 'train', 'stat'])
    self.assertEmpty(metric_outputs['broadcast'])
    self.assertEqual(metric_outputs['aggregation'],
                     collections.OrderedDict(mean_value=(), mean_weight=()))
    self.assertEqual(metric_outputs['train']['num_examples'], 0)
    self.assertTrue(tf.math.is_nan(metric_outputs['train']['loss']))

  @test_utils.skip_test_for_multi_gpu
  def test_get_model_weights(self):
    iterative_process = federated_sgd.build_federated_sgd_process(
        model_fn=model_examples.LinearRegression)

    num_clients = 3
    ds = tf.data.Dataset.from_tensor_slices(
        collections.OrderedDict(
            x=[[1.0, 2.0], [3.0, 4.0]],
            y=[[5.0], [6.0]],
        )).batch(2)
    datasets = [ds] * num_clients

    state = iterative_process.initialize()
    self.assertIsInstance(
        iterative_process.get_model_weights(state), model_utils.ModelWeights)
    self.assertAllClose(state.model.trainable,
                        iterative_process.get_model_weights(state).trainable)

    for _ in range(3):
      state, _ = iterative_process.next(state, datasets)
      self.assertIsInstance(
          iterative_process.get_model_weights(state), model_utils.ModelWeights)
      self.assertAllClose(state.model.trainable,
                          iterative_process.get_model_weights(state).trainable)

  def test_construction_calls_model_fn(self):
    # Assert that the the process building does not call `model_fn` too many
    # times. `model_fn` can potentially be expensive (loading weights,
    # processing, etc).
    mock_model_fn = mock.Mock(side_effect=model_examples.LinearRegression)
    federated_sgd.build_federated_sgd_process(model_fn=mock_model_fn)
    # TODO(b/186451541): reduce the number of calls to model_fn.
    self.assertEqual(mock_model_fn.call_count, 4)

if __name__ == '__main__':
  execution_contexts.set_local_execution_context()
  test_case.main()
<commit_msg>Re-enables an accidentally skipped FedSGD test.<commit_after># Copyright 2018, The TensorFlow Federated Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
from unittest import mock

from absl.testing import parameterized
import numpy as np
import tensorflow as tf

from tensorflow_federated.python.common_libs import test_utils
from tensorflow_federated.python.core.api import test_case
from tensorflow_federated.python.core.backends.native import execution_contexts
from tensorflow_federated.python.learning import client_weight_lib
from tensorflow_federated.python.learning import federated_sgd
from tensorflow_federated.python.learning import keras_utils
from tensorflow_federated.python.learning import model_examples
from tensorflow_federated.python.learning import model_utils
from tensorflow_federated.python.learning.framework import dataset_reduce

class FederatedSgdTest(test_case.TestCase, parameterized.TestCase):

  def dataset(self):
    # Create a dataset with 4 examples:
    dataset = tf.data.Dataset.from_tensor_slices(
        model_examples.LinearRegression.make_batch(
            x=[[1.0, 0.0], [1.0, 0.0], [1.0, 0.0], [1.0, 0.0]],
            y=[[1.0], [1.0], [1.0], [1.0]]))
    # Repeat the dataset 2 times with batches of 3 examples,
    # producing 3 minibatches (the last one with only 2 examples).
    # Note that `batch` is required for this dataset to be useable,
    # as it adds the batch dimension which is expected by the model.
    return dataset.repeat(2).batch(3)

  def model(self):
    return model_examples.LinearRegression(feature_dim=2)

  def initial_weights(self):
    return model_utils.ModelWeights(
        trainable=[
            tf.constant([[0.0], [0.0]]),
            tf.constant(0.0),
        ],
        non_trainable=[0.0])

  @parameterized.named_parameters(
      ('non-simulation_weighted', False, True),
      ('non-simulation_unweighted', False, False),
      ('simulation_weighted', True, True),
      ('simulation_unweighted', True, False),
  )
  @test_utils.skip_test_for_multi_gpu
  def test_client_tf(self, simulation, weighted):
    model = self.model()
    dataset = self.dataset()
    if weighted:
      client_weighting = client_weight_lib.ClientWeighting.NUM_EXAMPLES
    else:
      client_weighting = client_weight_lib.ClientWeighting.UNIFORM
    client_tf = federated_sgd.ClientSgd(
        model,
        client_weighting=client_weighting,
        use_experimental_simulation_loop=simulation)
    client_outputs = self.evaluate(client_tf(dataset, self.initial_weights()))

    # Both trainable parameters should have gradients, and we don't return the
    # non-trainable 'c'. Model deltas for squared error:
    self.assertAllClose(client_outputs.weights_delta, [[[1.0], [0.0]], 1.0])
    if weighted:
      self.assertAllClose(client_outputs.weights_delta_weight, 8.0)
    else:
      self.assertAllClose(client_outputs.weights_delta_weight, 1.0)

    self.assertDictContainsSubset(
        client_outputs.model_output, {
            'num_examples': 8,
            'num_examples_float': 8.0,
            'num_batches': 3,
            'loss': 0.5,
        })
    self.assertEqual(client_outputs.optimizer_output['has_non_finite_delta'], 0)

  @parameterized.named_parameters(('_inf', np.inf), ('_nan', np.nan))
  def test_non_finite_aggregation(self, bad_value):
    model = self.model()
    dataset = self.dataset()
    client_tf = federated_sgd.ClientSgd(model)
    init_weights = self.initial_weights()
    init_weights.trainable[1] = bad_value
    client_outputs = client_tf(dataset, init_weights)
    self.assertEqual(self.evaluate(client_outputs.weights_delta_weight), 0.0)
    self.assertAllClose(
        self.evaluate(client_outputs.weights_delta), [[[0.0], [0.0]], 0.0])

  @parameterized.named_parameters(('non-simulation', False),
                                  ('simulation', True))
  @mock.patch.object(
      dataset_reduce,
      '_dataset_reduce_fn',
      wraps=dataset_reduce._dataset_reduce_fn)
  @test_utils.skip_test_for_multi_gpu
  def test_client_tf_dataset_reduce_fn(self, simulation, mock_method):
    model = self.model()
    dataset = self.dataset()
    client_tf = federated_sgd.ClientSgd(
        model, use_experimental_simulation_loop=simulation)
    client_tf(dataset, self.initial_weights())
    if simulation:
      mock_method.assert_not_called()
    else:
      mock_method.assert_called()

class FederatedSGDTffTest(test_case.TestCase, parameterized.TestCase):

  @parameterized.named_parameters([
      ('unweighted', client_weight_lib.ClientWeighting.UNIFORM),
      ('example_weighted', client_weight_lib.ClientWeighting.NUM_EXAMPLES),
      ('custom_weighted', lambda _: tf.constant(1.5)),
  ])
  @test_utils.skip_test_for_multi_gpu
  def test_orchestration_execute(self, client_weighting):
    iterative_process = federated_sgd.build_federated_sgd_process(
        model_fn=model_examples.LinearRegression,
        client_weighting=client_weighting)

    # Some data points along [x_1 + 2*x_2 + 3 = y], expecting to learn
    # kernel = [1, 2], bias = [3].
    ds1 = tf.data.Dataset.from_tensor_slices(
        collections.OrderedDict(
            x=[[0.0, 0.0], [0.0, 1.0]],
            y=[[3.0], [5.0]],
        )).batch(2)
    ds2 = tf.data.Dataset.from_tensor_slices(
        collections.OrderedDict(
            x=[[1.0, 2.0], [3.0, 4.0], [1.0, 0.0], [-1.0, -1.0]],
            y=[[8.0], [14.0], [4.00], [0.0]],
        )).batch(2)
    federated_ds = [ds1, ds2]

    server_state = iterative_process.initialize()

    prev_loss = np.inf
    num_iterations = 3
    for _ in range(num_iterations):
      server_state, metric_outputs = iterative_process.next(
          server_state, federated_ds)
      train_metrics = metric_outputs['train']
      self.assertEqual(train_metrics['num_examples'],
                       num_iterations * len(federated_ds))
      loss = train_metrics['loss']
      self.assertLess(loss, prev_loss)
      prev_loss = loss

  @parameterized.named_parameters([
      ('functional_model',
       model_examples.build_linear_regression_keras_functional_model),
      ('sequential_model',
       model_examples.build_linear_regression_keras_sequential_model),
  ])
  @test_utils.skip_test_for_multi_gpu
  def test_orchestration_execute_from_keras(self, build_keras_model_fn):
    # Some data points along [x_1 + 2*x_2 + 3 = y], expecting to learn
    # kernel = [1, 2], bias = [3].
    ds1 = tf.data.Dataset.from_tensor_slices(
        collections.OrderedDict(
            x=[[0.0, 0.0], [0.0, 1.0]],
            y=[[3.0], [5.0]],
        )).batch(2)
    ds2 = tf.data.Dataset.from_tensor_slices(
        collections.OrderedDict(
            x=[[1.0, 2.0], [3.0, 4.0], [1.0, 0.0], [-1.0, -1.0]],
            y=[[8.0], [14.0], [4.00], [0.0]],
        )).batch(2)
    federated_ds = [ds1, ds2]

    def model_fn():
      # Note: we don't compile with an optimizer here; FedSGD does not use it.
      keras_model = build_keras_model_fn(feature_dims=2)
      return keras_utils.from_keras_model(
          keras_model,
          input_spec=ds1.element_spec,
          loss=tf.keras.losses.MeanSquaredError())

    iterative_process = federated_sgd.build_federated_sgd_process(
        model_fn=model_fn)

    server_state = iterative_process.initialize()
    prev_loss = np.inf
    num_iterations = 3
    for _ in range(num_iterations):
      server_state, metrics = iterative_process.next(server_state, federated_ds)
      new_loss = metrics['train']['loss']
      self.assertLess(new_loss, prev_loss)
      prev_loss = new_loss

  @test_utils.skip_test_for_multi_gpu
  def test_execute_empty_data(self):
    iterative_process = federated_sgd.build_federated_sgd_process(
        model_fn=model_examples.LinearRegression)

    # Results in empty dataset with correct types and shapes.
    ds = tf.data.Dataset.from_tensor_slices(
        collections.OrderedDict(x=[[1.0, 2.0]], y=[[5.0]])).batch(
            5, drop_remainder=True)  # No batches of size 5 can be created.
    federated_ds = [ds] * 2

    server_state = iterative_process.initialize()
    first_state, metric_outputs = iterative_process.next(
        server_state, federated_ds)
    self.assertAllClose(
        list(first_state.model.trainable), [[[0.0], [0.0]], 0.0])
    self.assertEqual(
        list(metric_outputs.keys()),
        ['broadcast', 'aggregation', 'train', 'stat'])
    self.assertEmpty(metric_outputs['broadcast'])
    self.assertEqual(metric_outputs['aggregation'],
                     collections.OrderedDict(mean_value=(), mean_weight=()))
    self.assertEqual(metric_outputs['train']['num_examples'], 0)
    self.assertTrue(tf.math.is_nan(metric_outputs['train']['loss']))

  @test_utils.skip_test_for_multi_gpu
  def test_get_model_weights(self):
    iterative_process = federated_sgd.build_federated_sgd_process(
        model_fn=model_examples.LinearRegression)

    num_clients = 3
    ds = tf.data.Dataset.from_tensor_slices(
        collections.OrderedDict(
            x=[[1.0, 2.0], [3.0, 4.0]],
            y=[[5.0], [6.0]],
        )).batch(2)
    datasets = [ds] * num_clients

    state = iterative_process.initialize()
    self.assertIsInstance(
        iterative_process.get_model_weights(state), model_utils.ModelWeights)
    self.assertAllClose(state.model.trainable,
                        iterative_process.get_model_weights(state).trainable)

    for _ in range(3):
      state, _ = iterative_process.next(state, datasets)
      self.assertIsInstance(
          iterative_process.get_model_weights(state), model_utils.ModelWeights)
      self.assertAllClose(state.model.trainable,
                          iterative_process.get_model_weights(state).trainable)

  def test_construction_calls_model_fn(self):
    # Assert that the the process building does not call `model_fn` too many
    # times. `model_fn` can potentially be expensive (loading weights,
    # processing, etc).
    mock_model_fn = mock.Mock(side_effect=model_examples.LinearRegression)
    federated_sgd.build_federated_sgd_process(model_fn=mock_model_fn)
    # TODO(b/186451541): reduce the number of calls to model_fn.
    self.assertEqual(mock_model_fn.call_count, 4)

if __name__ == '__main__':
  execution_contexts.set_local_execution_context()
  test_case.main()
<|endoftext|>


<commit_before><commit_msg>retain script type in ocr_record class<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''An implementation of learning copying algorithm with RNN (basic RNN, LSTM,
GRU).
Input sequence length: "1 ~ 20: (1*2+1)=3 ~ (20*2+1)=41"
Input dimension: "8"
Output sequence length: equal to input sequence length.
Output dimension: equal to input dimension.
'''

from __future__ import print_function
from keras.models import Sequential
# from keras.engine.training import slice_X
from keras.layers import Activation, TimeDistributed, Dense, RepeatVector, recurrent
import numpy as np
# from six.moves import range
import dataset                               # Add by Steven Robot
import visualization                         # Add by Steven Robot
from keras.utils.visualize_util import plot  # Add by Steven Robot
import time                                  # Add by Steven Robot
from util import LossHistory                 # Add by Steven Robot
import os                                    # Add by Steven Robot

# Parameters for the model to train copying algorithm
TRAINING_SIZE = 1024000         # for 8-bits length
# TRAINING_SIZE = 128000         # for 4-bits length
# TRAINING_SIZE = 1280
INPUT_DIMENSION_SIZE = 8 + 1    # for 8-bits length
# INPUT_DIMENSION_SIZE = 4 + 1   # for 4-bits length
MAX_COPY_LENGTH = 20            # for 8-bits length
# MAX_COPY_LENGTH = 10           # for 4-bits length
MAX_INPUT_LENGTH = MAX_COPY_LENGTH + 1 + MAX_COPY_LENGTH

# Try replacing SimpleRNN, GRU, or LSTM
# RNN = recurrent.SimpleRNN
# RNN = recurrent.GRU
RNN = recurrent.LSTM
HIDDEN_SIZE = 128
# HIDDEN_SIZE = 128*4
LAYERS = 1
# BATCH_SIZE = 2048
BATCH_SIZE = 1024
# BATCH_SIZE = 512
# BATCH_SIZE = 256
# BATCH_SIZE = 128
FOLDER = "experiment_results/copy/"
if not os.path.isdir(FOLDER):
    os.makedirs(FOLDER)
    print("create folder: %s" % FOLDER)

print()
print(time.strftime('%Y-%m-%d %H:%M:%S'))
print('Generating data sets...')
train_X, train_Y = dataset.generate_copy_data_set(
    INPUT_DIMENSION_SIZE, MAX_COPY_LENGTH, TRAINING_SIZE)
valid_X, valid_Y = dataset.generate_copy_data_set(
    INPUT_DIMENSION_SIZE, MAX_COPY_LENGTH, TRAINING_SIZE/10)

print()
print(time.strftime('%Y-%m-%d %H:%M:%S'))
print('Build model...')
model = Sequential()
# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE
# note: in a situation where your input sequences have a variable length,
# use input_shape=(None, nb_feature).
hidden_layer = RNN(
    HIDDEN_SIZE,
    input_shape=(MAX_INPUT_LENGTH, INPUT_DIMENSION_SIZE),
    init='glorot_uniform',
    inner_init='orthogonal',
    activation='tanh',
    # activation='hard_sigmoid',
    # activation='sigmoid',
    W_regularizer=None,
    U_regularizer=None,
    b_regularizer=None,
    dropout_W=0.0,
    dropout_U=0.0)
model.add(hidden_layer)
# For the decoder's input, we repeat the encoded input for each time step
model.add(RepeatVector(MAX_INPUT_LENGTH))
# The decoder RNN could be multiple layers stacked or a single layer
for _ in range(LAYERS):
    model.add(RNN(HIDDEN_SIZE, return_sequences=True))

# For each of step of the output sequence, decide which character should be chosen
model.add(TimeDistributed(Dense(INPUT_DIMENSION_SIZE)))
# model.add(Activation('softmax'))
# model.add(Activation('hard_sigmoid'))
model.add(Activation('sigmoid'))

model.compile(loss='binary_crossentropy',
              # loss='mse',
              optimizer='adam',
              metrics=['accuracy'])

print()
print(time.strftime('%Y-%m-%d %H:%M:%S'))
print("Model architecture")
plot(model, show_shapes=True, to_file=FOLDER+"model_simple_rnn_for_copying.png")

print()
print(time.strftime('%Y-%m-%d %H:%M:%S'))
print("Training...")
# Train the model each generation and show predictions against the
# validation dataset
matrix_list = []
matrix_list.append(train_X[0].transpose())
matrix_list.append(train_Y[0].transpose())
matrix_list.append(train_Y[0].transpose())
name_list = []
name_list.append("Input")
name_list.append("Target")
name_list.append("Predict")
show_matrix = visualization.PlotDynamicalMatrix(matrix_list, name_list)
for iteration in range(1, 200):
    print()
    print('-' * 78)
    print(time.strftime('%Y-%m-%d %H:%M:%S'))
    print('Iteration', iteration)
    model.fit(train_X,
              train_Y,
              batch_size=BATCH_SIZE,
              nb_epoch=10,
              # nb_epoch=1,
              validation_data=(valid_X, valid_Y))
    ###
    # Select 3 samples from the validation set at random so we can
    # visualize errors
    for i in range(10):
        ind = np.random.randint(0, len(valid_X))
        # inputs = valid_X[ind]
        # outputs = valid_Y[ind]
        inputs, outputs = valid_X[np.array([ind])], valid_Y[np.array([ind])]
        predicts = model.predict(inputs, verbose=0)
        # print(inputs)
        # print(outputs)
        # print(predicts)
        matrix_list_update = []
        matrix_list_update.append(inputs[0].transpose())
        matrix_list_update.append(outputs[0].transpose())
        matrix_list_update.append(predicts[0].transpose())
        show_matrix.update(matrix_list_update, name_list)
        show_matrix.save(FOLDER+"copy_data_predict_%3d.png" % iteration)

show_matrix.close()<commit_msg>Updating hyper-parameters for learning copying.<commit_after># -*- coding: utf-8 -*-
'''An implementation of learning copying algorithm with RNN (basic RNN, LSTM,
GRU).
Input sequence length: "1 ~ 20: (1*2+1)=3 ~ (20*2+1)=41"
Input dimension: "8"
Output sequence length: equal to input sequence length.
Output dimension: equal to input dimension.
'''

from __future__ import print_function
from keras.models import Sequential
# from keras.engine.training import slice_X
from keras.layers import Activation, TimeDistributed, Dense, RepeatVector, recurrent
import numpy as np
# from six.moves import range
import dataset                               # Add by Steven Robot
import visualization                         # Add by Steven Robot
from keras.utils.visualize_util import plot  # Add by Steven Robot
import time                                  # Add by Steven Robot
from util import LossHistory                 # Add by Steven Robot
import os                                    # Add by Steven Robot

# Parameters for the model to train copying algorithm
TRAINING_SIZE = 1024000         # for 8-bits length
# TRAINING_SIZE = 128000         # for 4-bits length
# TRAINING_SIZE = 1280
INPUT_DIMENSION_SIZE = 8 + 1    # for 8-bits length
# INPUT_DIMENSION_SIZE = 4 + 1   # for 4-bits length
MAX_COPY_LENGTH = 20            # for 8-bits length
# MAX_COPY_LENGTH = 10           # for 4-bits length
MAX_INPUT_LENGTH = MAX_COPY_LENGTH + 1 + MAX_COPY_LENGTH

# Try replacing SimpleRNN, GRU, or LSTM
# RNN = recurrent.SimpleRNN
# RNN = recurrent.GRU
RNN = recurrent.LSTM
# HIDDEN_SIZE = 128
HIDDEN_SIZE = 128*4
LAYERS = 1
# BATCH_SIZE = 2048
BATCH_SIZE = 1024
# BATCH_SIZE = 512
# BATCH_SIZE = 256
# BATCH_SIZE = 128
FOLDER = "experiment_results/copy/"
if not os.path.isdir(FOLDER):
    os.makedirs(FOLDER)
    print("create folder: %s" % FOLDER)

print()
print(time.strftime('%Y-%m-%d %H:%M:%S'))
print('Generating data sets...')
train_X, train_Y = dataset.generate_copy_data_set(
    INPUT_DIMENSION_SIZE, MAX_COPY_LENGTH, TRAINING_SIZE)
valid_X, valid_Y = dataset.generate_copy_data_set(
    INPUT_DIMENSION_SIZE, MAX_COPY_LENGTH, TRAINING_SIZE/10)

print()
print(time.strftime('%Y-%m-%d %H:%M:%S'))
print('Build model...')
model = Sequential()
# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE
# note: in a situation where your input sequences have a variable length,
# use input_shape=(None, nb_feature).
hidden_layer = RNN(
    HIDDEN_SIZE,
    input_shape=(MAX_INPUT_LENGTH, INPUT_DIMENSION_SIZE),
    init='glorot_uniform',
    inner_init='orthogonal',
    activation='tanh',
    # activation='hard_sigmoid',
    # activation='sigmoid',
    W_regularizer=None,
    U_regularizer=None,
    b_regularizer=None,
    dropout_W=0.0,
    dropout_U=0.0)
model.add(hidden_layer)
# For the decoder's input, we repeat the encoded input for each time step
model.add(RepeatVector(MAX_INPUT_LENGTH))
# The decoder RNN could be multiple layers stacked or a single layer
for _ in range(LAYERS):
    model.add(RNN(HIDDEN_SIZE, return_sequences=True))

# For each of step of the output sequence, decide which character should be chosen
model.add(TimeDistributed(Dense(INPUT_DIMENSION_SIZE)))
# model.add(Activation('softmax'))
# model.add(Activation('hard_sigmoid'))
model.add(Activation('sigmoid'))

model.compile(loss='binary_crossentropy',
              # loss='mse',
              optimizer='adam',
              metrics=['accuracy'])

print()
print(time.strftime('%Y-%m-%d %H:%M:%S'))
print("Model architecture")
plot(model, show_shapes=True, to_file=FOLDER+"model_simple_rnn_for_copying.png")

print()
print(time.strftime('%Y-%m-%d %H:%M:%S'))
print("Training...")
# Train the model each generation and show predictions against the
# validation dataset
matrix_list = []
matrix_list.append(train_X[0].transpose())
matrix_list.append(train_Y[0].transpose())
matrix_list.append(train_Y[0].transpose())
name_list = []
name_list.append("Input")
name_list.append("Target")
name_list.append("Predict")
show_matrix = visualization.PlotDynamicalMatrix(matrix_list, name_list)
for iteration in range(1, 200):
    print()
    print('-' * 78)
    print(time.strftime('%Y-%m-%d %H:%M:%S'))
    print('Iteration', iteration)
    model.fit(train_X,
              train_Y,
              batch_size=BATCH_SIZE,
              # nb_epoch=10,
              nb_epoch=1,
              validation_data=(valid_X, valid_Y))
    ###
    # Select 3 samples from the validation set at random so we can
    # visualize errors
    for i in range(10):
        ind = np.random.randint(0, len(valid_X))
        # inputs = valid_X[ind]
        # outputs = valid_Y[ind]
        inputs, outputs = valid_X[np.array([ind])], valid_Y[np.array([ind])]
        predicts = model.predict(inputs, verbose=0)
        # print(inputs)
        # print(outputs)
        # print(predicts)
        matrix_list_update = []
        matrix_list_update.append(inputs[0].transpose())
        matrix_list_update.append(outputs[0].transpose())
        matrix_list_update.append(predicts[0].transpose())
        show_matrix.update(matrix_list_update, name_list)
        show_matrix.save(FOLDER+"copy_data_predict_%3d.png" % iteration)

show_matrix.close()<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Author: Konrad Zemek
Copyright (C) 2015 ACK CYFRONET AGH
This software is released under the MIT license cited in 'LICENSE.txt'

Runs oneprovider integration tests, providing Erlang's ct_run with every
environmental argument it needs for successful run. The output is put into
'test_distributed/logs'. The (init|end)_per_suite "testcases" are removed from
the surefire.xml output.

All paths used are relative to script's path, not to the running user's CWD.
Run the script with -h flag to learn about script's running options.
"""

import argparse
import glob
import os
import platform
import sys
import time
import shutil

sys.path.insert(0, 'bamboos/docker')
from environment import docker

parser = argparse.ArgumentParser(
    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    description='Run Common Tests.')

parser.add_argument(
    '--image', '-i',
    action='store',
    default='onedata/worker',
    help='docker image to use as a test master',
    dest='image')

parser.add_argument(
    '--suite', '-s',
    action='append',
    help='name of the test suite',
    dest='suites')

parser.add_argument(
    '--case', '-c',
    action='append',
    help='name of the test case',
    dest='cases')

parser.add_argument(
    '--performance', '-p',
    action='store_true',
    default=False,
    help='run performance tests',
    dest='performance')

args = parser.parse_args()
script_dir = os.path.dirname(os.path.abspath(__file__))
uid = str(int(time.time()))

excluded_modules = glob.glob(os.path.join(script_dir, 'test_distributed', '*.erl'))
for i, item in enumerate(excluded_modules):
    excluded_modules[i] = os.path.basename(item)[:-4]

new_cover = os.path.join(script_dir, 'test_distributed', 'cover_tmp.spec')
shutil.copyfile(os.path.join(script_dir, 'test_distributed', 'cover.spec'),
                new_cover)

with open(new_cover, 'a') as cover:
    cover.write('\n{excl_mods, [performance,  ' + ', '.join(excluded_modules) + ']}.')

ct_command = ['ct_run',
              '-no_auto_compile',
              '-dir', '.',
              '-logdir', './logs/',
              '-cover', 'cover_tmp.spec',
              '-ct_hooks', 'cth_surefire', '[{path, "surefire.xml"}]',
              '-noshell',
              '-name', 'testmaster@testmaster.{0}.dev.docker'.format(uid),
              '-include', '../include', '../deps']

code_paths = ['-pa', os.path.join(script_dir, 'ebin')]
code_paths.extend(glob.glob(os.path.join(script_dir, 'deps', '*', 'ebin')))
ct_command.extend(code_paths)

if args.suites:
    ct_command.append('-suite')
    ct_command.extend(args.suites)

if args.cases:
    ct_command.append('-case')
    ct_command.extend(args.cases)

if args.performance:
    ct_command.extend(['-env', 'performance', 'true'])

command = '''
import os, subprocess, sys, stat

if {shed_privileges}:
    os.environ['HOME'] = '/tmp'
    docker_gid = os.stat('/var/run/docker.sock').st_gid
    os.chmod('/etc/resolv.conf', 0o666)
    os.setgroups([docker_gid])
    os.setregid({gid}, {gid})
    os.setreuid({uid}, {uid})

command = {cmd}
ret = subprocess.call(command)

import xml.etree.ElementTree as ElementTree, glob, re
for file in glob.glob('logs/*/surefire.xml'):
    tree = ElementTree.parse(file)
    for suite in tree.findall('.//testsuite'):
        for test in suite.findall('testcase'):
            match = re.match('(init|end)_per_suite', test.attrib['name'])
            if match is not None:
                suite.remove(test)
    tree.write(file)

sys.exit(ret)
'''
command = command.format(
    uid=os.geteuid(),
    gid=os.getegid(),
    cmd=ct_command,
    shed_privileges=(platform.system() == 'Linux'))

ret = docker.run(tty=True,
                 rm=True,
                 interactive=True,
                 workdir=os.path.join(script_dir, 'test_distributed'),
                 reflect=[(script_dir, 'rw'),
                          ('/var/run/docker.sock', 'rw')],
                 name='testmaster_{0}'.format(uid),
                 hostname='testmaster.{0}.dev.docker'.format(uid),
                 image=args.image,
                 command=['python', '-c', command])

os.remove(new_cover)
sys.exit(ret)
<commit_msg>Cover analysis extended (ccm added to cover)<commit_after>#!/usr/bin/env python

"""Author: Konrad Zemek
Copyright (C) 2015 ACK CYFRONET AGH
This software is released under the MIT license cited in 'LICENSE.txt'

Runs oneprovider integration tests, providing Erlang's ct_run with every
environmental argument it needs for successful run. The output is put into
'test_distributed/logs'. The (init|end)_per_suite "testcases" are removed from
the surefire.xml output.

All paths used are relative to script's path, not to the running user's CWD.
Run the script with -h flag to learn about script's running options.
"""

import argparse
import glob
import os
import platform
import sys
import time
import shutil

sys.path.insert(0, 'bamboos/docker')
from environment import docker

parser = argparse.ArgumentParser(
    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    description='Run Common Tests.')

parser.add_argument(
    '--image', '-i',
    action='store',
    default='onedata/worker',
    help='docker image to use as a test master',
    dest='image')

parser.add_argument(
    '--suite', '-s',
    action='append',
    help='name of the test suite',
    dest='suites')

parser.add_argument(
    '--case', '-c',
    action='append',
    help='name of the test case',
    dest='cases')

parser.add_argument(
    '--performance', '-p',
    action='store_true',
    default=False,
    help='run performance tests',
    dest='performance')

args = parser.parse_args()
script_dir = os.path.dirname(os.path.abspath(__file__))
uid = str(int(time.time()))

excluded_modules = glob.glob(os.path.join(script_dir, 'test_distributed', '*.erl'))
for i, item in enumerate(excluded_modules):
    excluded_modules[i] = os.path.basename(item)[:-4]

cover_template = os.path.join(script_dir, 'test_distributed', 'cover.spec')
new_cover = os.path.join(script_dir, 'test_distributed', 'cover_tmp.spec')

dirs = []
with open(cover_template) as f, open(new_cover, 'a') as cover:
    lines = f.readlines()
    for line in lines:
        if line.find('incl_dirs_r') != -1:
            start = line.find('[')
            stop = line.find(']')
            dirs_string = line[start+1:stop]
            dirs = dirs_string.split(', ')
        else:
            cover.write(line)
    cover.write('\n{excl_mods, [performance,  ' + ', '.join(excluded_modules) + ']}.')
    for i, item in enumerate(dirs):
        dirs[i] = os.path.join(script_dir, dirs[i][1:])
    cover.write('\n{incl_dirs_r, ["' + ', "'.join(dirs) + ']}.')

ct_command = ['ct_run',
              '-no_auto_compile',
              '-dir', '.',
              '-logdir', './logs/',
              '-cover', 'cover_tmp.spec',
              '-ct_hooks', 'cth_surefire', '[{path, "surefire.xml"}]',
              '-noshell',
              '-name', 'testmaster@testmaster.{0}.dev.docker'.format(uid),
              '-include', '../include', '../deps']

code_paths = ['-pa']
if dirs == []:
    code_paths.extend([os.path.join(script_dir, 'ebin')])
else:
    for i, item in enumerate(dirs):
        dirs[i] = os.path.join(script_dir, dirs[i][0:-1])
    code_paths.extend(dirs)
code_paths.extend(glob.glob(os.path.join(script_dir, 'deps', '*', 'ebin')))
ct_command.extend(code_paths)

if args.suites:
    ct_command.append('-suite')
    ct_command.extend(args.suites)

if args.cases:
    ct_command.append('-case')
    ct_command.extend(args.cases)

if args.performance:
    ct_command.extend(['-env', 'performance', 'true'])

command = '''
import os, subprocess, sys, stat

if {shed_privileges}:
    os.environ['HOME'] = '/tmp'
    docker_gid = os.stat('/var/run/docker.sock').st_gid
    os.chmod('/etc/resolv.conf', 0o666)
    os.setgroups([docker_gid])
    os.setregid({gid}, {gid})
    os.setreuid({uid}, {uid})

command = {cmd}
ret = subprocess.call(command)

import xml.etree.ElementTree as ElementTree, glob, re
for file in glob.glob('logs/*/surefire.xml'):
    tree = ElementTree.parse(file)
    for suite in tree.findall('.//testsuite'):
        for test in suite.findall('testcase'):
            match = re.match('(init|end)_per_suite', test.attrib['name'])
            if match is not None:
                suite.remove(test)
    tree.write(file)

sys.exit(ret)
'''
command = command.format(
    uid=os.geteuid(),
    gid=os.getegid(),
    cmd=ct_command,
    shed_privileges=(platform.system() == 'Linux'))

ret = docker.run(tty=True,
                 rm=True,
                 interactive=True,
                 workdir=os.path.join(script_dir, 'test_distributed'),
                 reflect=[(script_dir, 'rw'),
                          ('/var/run/docker.sock', 'rw')],
                 name='testmaster_{0}'.format(uid),
                 hostname='testmaster.{0}.dev.docker'.format(uid),
                 image=args.image,
                 command=['python', '-c', command])

os.remove(new_cover)
sys.exit(ret)
<|endoftext|>


<commit_before>import os

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    ('Charlie Leifer', 'coleifer@gmail.com'),
    ('Eric Holscher', 'eric@ericholscher.com'),
)

MANAGERS = ADMINS

SITE_ROOT = os.path.dirname(os.path.realpath(__file__))

MEDIA_ROOT = '%s/media/' % (SITE_ROOT)
MEDIA_URL = '/media/'
ADMIN_MEDIA_PREFIX = '/media/admin/'

CACHE_BACKEND = 'memcached://localhost:11211/'
CACHE_KEY_PREFIX = 'docs'
CACHE_MIDDLEWARE_SECONDS = 60

FORCE_WWW = True
LOGIN_REDIRECT_URL = '/'

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': 'docs',
        'USER': 'postgres',                      # Not used with sqlite3.
        'PASSWORD': '',
        'HOST': 'golem',
        'PORT': '',
    }
}

if DEBUG:
    CACHE_BACKEND = 'dummy://'
    DATABASES['default']['HOST'] = 'localhost'

TIME_ZONE = 'America/Chicago'
LANGUAGE_CODE = 'en-us'
USE_I18N = True
SITE_ID = 1
SECRET_KEY = 'asciidick'

TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.load_template_source',
    'django.template.loaders.app_directories.load_template_source',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
)

ROOT_URLCONF = 'urls'

TEMPLATE_DIRS = (
    '%s/templates/' % (SITE_ROOT),
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.admin',
    'django.contrib.comments',
    'django.contrib.contenttypes',
    'django.contrib.flatpages',
    'django.contrib.markup',
    'django.contrib.sessions',
    'django.contrib.sites',
    'pagination',
    'south',
    'projects',
)
<commit_msg>Adding profiles & registration to our settings<commit_after>import os

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    ('Charlie Leifer', 'coleifer@gmail.com'),
    ('Eric Holscher', 'eric@ericholscher.com'),
)

MANAGERS = ADMINS

SITE_ROOT = os.path.dirname(os.path.realpath(__file__))

MEDIA_ROOT = '%s/media/' % (SITE_ROOT)
MEDIA_URL = '/media/'
ADMIN_MEDIA_PREFIX = '/media/admin/'

CACHE_BACKEND = 'memcached://localhost:11211/'
CACHE_KEY_PREFIX = 'docs'
CACHE_MIDDLEWARE_SECONDS = 60

FORCE_WWW = True
LOGIN_REDIRECT_URL = '/'

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': 'docs',
        'USER': 'postgres',                      # Not used with sqlite3.
        'PASSWORD': '',
        'HOST': 'golem',
        'PORT': '',
    }
}

if DEBUG:
    CACHE_BACKEND = 'dummy://'
    DATABASES['default']['HOST'] = 'localhost'

TIME_ZONE = 'America/Chicago'
LANGUAGE_CODE = 'en-us'
USE_I18N = True
SITE_ID = 1
SECRET_KEY = 'asciidick'

TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.load_template_source',
    'django.template.loaders.app_directories.load_template_source',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
)

ROOT_URLCONF = 'urls'

TEMPLATE_DIRS = (
    '%s/templates/' % (SITE_ROOT),
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.admin',
    'django.contrib.comments',
    'django.contrib.contenttypes',
    'django.contrib.flatpages',
    'django.contrib.markup',
    'django.contrib.sessions',
    'django.contrib.sites',

    # third party apps
    'pagination',
    'profiles',
    'registration',
    'south',

    # our apps
    'projects',
)
<|endoftext|>


<commit_before>#!/usr/bin/python
# Pythonic abstraction layer for Nonolith Connect's RESTful API.
# Released under the terms of the GNU GPLv3+
# (C) 2012 Nonolith Labs, LLC
# Authors:
#   Ian Daniher

import httplib, atexit, urllib, json

class CEE:

	def __init__(self, devID = "com.nonolithlabs.cee*"):
		""" Starts an HTTP connection, determines the target device ID, and starts capture."""
		self.connection = httplib.HTTPConnection("localhost:9003")
		atexit.register(self.onClose)

		if devID == "com.nonolithlabs.cee*":
			self.devID = devID

		elif devID:
			self.connection.request("GET", "/rest/v1/devices/")
			response = json.loads(self.connection.getresponse().read())
			if response.keys()[0] == devID:
				self.devID = devID
			elif response.keys()[0] == '':
				raise Exception('No devices found.')
			elif type(devID) == int:
				try:
					self.devID = response.keys()[devID]
				except:
					raise Exception('Device index invalid.')

		options = {"capture":"on"}
		options = urllib.urlencode(options)
		headers = {"Content-Type": "application/x-www-form-urlencoded"}
		self.connection.request("POST", "/rest/v1/devices/%s" % self.devID, options, headers)
		self.connection.getresponse()

	def onClose(self):
		""" Stop capturing and close the HTTP connection."""
		options = {"capture":"off"}
		options = urllib.urlencode(options)
		headers = {"Content-Type": "application/x-www-form-urlencoded"}
		self.connection.request("POST", "/rest/v1/devices/%s" % self.devID, options, headers)
		self.connection.getresponse()
		self.connection.close()

	def getOutput(self, channel = "a"):
		""" Return a dictionary containing the output state of a given channel."""
		self.connection.request("GET", "/rest/v1/devices/%s/%s/output" % (self.devID, channel))
		return dict(json.loads(self.connection.getresponse().read()))

	def setOutput(self, channel = "a", stream = "d", value = 0, wave="constant", amplitude = None, frequency = 0, relPhase = 1, phase = 0):
		""" Set the output state for a given channel.
			'stream' can be 'v' to set voltage, 'i' to set current, or 'd' for high impedance mode.
			'value' is an number either in volts or milliamps determining the target value in DC mode or the center value in AC mode.
			'wave' can be either 'constant,' 'triangle,' 'square,' or 'sine.'
			The following parameters only have meaning when 'wave' is not constant:
				'amplitude' determines the maximum offset from center.
				'frequency' is the cycles per second.
				'relPhase' determines whether the starting value is based off of the previous output setting to provide seamless change in frequency.
				'phase' is the phase offset in seconds from the beginning of the stream (relPhase=0) or from the previous source (relPhase=1)."""
		if wave == "constant":
			options = {"mode": stream, "value": value}
		elif wave in ["square", "triangle", "sine"]:
			options = {"mode": stream, "value": value, "wave": wave, "amplitude": amplitude, "frequency": frequency, "relPhase": relPhase, "phase": phase}
		else:
			raise Exception('Invalid option for "wave"')
		options = urllib.urlencode(options)
		headers = {"Content-Type": "application/x-www-form-urlencoded"}
		self.connection.request("POST", "/rest/v1/devices/%s/%s/output" % (self.devID, channel),  options, headers)
		return json.loads(self.connection.getresponse().read())

	def getInput(self, channel = "a", resample = .01, count = 1):
		""" Returns a pair of list indicating the measured state of the specified channel.
			The first list is measured voltage, the second list is measured current.
			Each list contains 'count' samples, averaged over 'resample' seconds, separated by 'resample' seconds."""
		options = {"resample":resample, "count":count, "header":0}
		options = "?" + urllib.urlencode(options)
		self.connection.request("GET", "/rest/v1/devices/%s/%s/input" % (self.devID, channel) + options)
		values = [[float(item) for item in item.split(',')]
			for item in self.connection.getresponse().read().split('\n')
			if item != '']
		return map(list, zip(*values))

if __name__ == "__main__":
	from numpy import arange
	CEE = CEE()
	print CEE.setOutput(stream = "v", wave = "sine", amplitude = 1, value = 1, frequency = 100)
	print CEE.getInput(resample = 0, count = 100)
<commit_msg>fixed syntax error<commit_after>#"!/usr/bin/python
# Pythonic abstraction layer for Nonolith Connect's RESTful API.
# Released under the terms of the GNU GPLv3+
# (C) 2012 Nonolith Labs, LLC
# Authors:
#   Ian Daniher

import httplib, atexit, urllib, json

class CEE:

	def __init__(self, devID = "com.nonolithlabs.cee*"):
		""" Starts an HTTP connection, determines the target device ID, and starts capture."""
		self.connection = httplib.HTTPConnection("localhost:9003")
		atexit.register(self.onClose)

		if devID == "com.nonolithlabs.cee*":
			self.devID = devID

		elif devID:
			self.connection.request("GET", "/rest/v1/devices/")
			response = json.loads(self.connection.getresponse().read())
			if response.keys()[0] == devID:
				self.devID = devID
			elif response.keys()[0] == '':
				raise Exception('No devices found.')
			elif type(devID) == int:
				try:
					self.devID = response.keys()[devID]
				except:
					raise Exception('Device index invalid.')

		options = {"capture":"on"}
		options = urllib.urlencode(options)
		headers = {"Content-Type": "application/x-www-form-urlencoded"}
		self.connection.request("POST", "/rest/v1/devices/%s" % self.devID, options, headers)
		self.connection.getresponse()

	def onClose(self):
		""" Stop capturing and close the HTTP connection."""
		options = {"capture":"off"}
		options = urllib.urlencode(options)
		headers = {"Content-Type": "application/x-www-form-urlencoded"}
		self.connection.request("POST", "/rest/v1/devices/%s" % self.devID, options, headers)
		self.connection.getresponse()
		self.connection.close()

	def getOutput(self, channel = "a"):
		""" Return a dictionary containing the output state of a given channel."""
		self.connection.request("GET", "/rest/v1/devices/%s/%s/output" % (self.devID, channel))
		return dict(json.loads(self.connection.getresponse().read()))

	def setOutput(self, channel = "a", stream = "d", value = 0, wave="constant", amplitude = None, frequency = 0, relPhase = 1, phase = 0):
		""" Set the output state for a given channel.
			'stream' can be 'v' to set voltage, 'i' to set current, or 'd' for high impedance mode.
			'value' is an number either in volts or milliamps determining the target value in DC mode or the center value in AC mode.
			'wave' can be either 'constant,' 'triangle,' 'square,' or 'sine.'
			The following parameters only have meaning when 'wave' is not constant:
				'amplitude' determines the maximum offset from center.
				'frequency' is the cycles per second.
				'relPhase' determines whether the starting value is based off of the previous output setting to provide seamless change in frequency.
				'phase' is the phase offset in seconds from the beginning of the stream (relPhase=0) or from the previous source (relPhase=1)."""
		if wave == "constant":
			options = {"mode": stream, "value": value}
		elif wave in ["square", "triangle", "sine"]:
			options = {"mode": stream, "value": value, "wave": wave, "amplitude": amplitude, "frequency": frequency, "relPhase": relPhase, "phase": phase}
		else:
			raise Exception('Invalid option for "wave"')
		options = urllib.urlencode(options)
		headers = {"Content-Type": "application/x-www-form-urlencoded"}
		self.connection.request("POST", "/rest/v1/devices/%s/%s/output" % (self.devID, channel),  options, headers)
		return json.loads(self.connection.getresponse().read())

	def getInput(self, channel = "a", resample = .01, count = 1):
		""" Returns a pair of list indicating the measured state of the specified channel.
			The first list is measured voltage, the second list is measured current.
			Each list contains 'count' samples, averaged over 'resample' seconds, separated by 'resample' seconds."""
		options = {"resample":resample, "count":count, "header":0}
		options = "?" + urllib.urlencode(options)
		self.connection.request("GET", "/rest/v1/devices/%s/%s/input" % (self.devID, channel) + options)
		values = [[float(item) for item in item.split(',')]
			for item in self.connection.getresponse().read().split('\n')
			if item != '']
		return map(list, zip(*values))

if __name__ == "__main__":
	from numpy import arange
	CEE = CEE()
	print(CEE.setOutput(stream = "v", wave = "sine", amplitude = 1, value = 1, frequency = 100))
	print(CEE.getInput(resample = 0, count = 100))
<|endoftext|>