<commit_before>#
# This file is part of pySMT.
#
#   Copyright 2014 Andrea Micheli and Marco Gario
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
from pysmt.shortcuts import (And, Iff, Or, Symbol, Implies, Not,
                             Exists, ForAll,
                             Times, Plus, Minus, Equals, Real,
                             is_valid, is_sat, Function)
from pysmt.test import TestCase, skipIfNoSolverForLogic, main
from pysmt.rewritings import prenex_normal_form, nnf, conjunctive_partition, aig
from pysmt.rewritings import disjunctive_partition
from pysmt.rewritings import TimesDistributor, Ackermannizer
from pysmt.test.examples import get_example_formulae
from pysmt.exceptions import SolverReturnedUnknownResultError
from pysmt.logics import BOOL, QF_NRA, QF_LRA, QF_AUFLIA
from pysmt.typing import REAL, INT, FunctionType

class TestRewritings(TestCase):

    def test_prenex_basic(self):
        a,b,c = (Symbol(x) for x in "abc")
        f = Not(And(a, Exists([b], And(a, b)), ForAll([c], Or(a, c))))
        prenex = prenex_normal_form(f)
        # Two prenex normal forms are possible
        my_prenex_1 = Exists([c], ForAll([b], Not(And(a, And(a, b), Or(a, c)))))
        my_prenex_2 = ForAll([b], Exists([c], Not(And(a, And(a, b), Or(a, c)))))
        self.assertTrue(prenex == my_prenex_1 or prenex == my_prenex_2)

    @skipIfNoSolverForLogic(BOOL)
    def test_prenex_simple_exists(self):
        a,b = (Symbol(x) for x in "ab")
        f = And(b, Exists([b], Implies(a, b)))
        prenex = prenex_normal_form(f)
        self.assertTrue(prenex.is_exists())
        self.assertValid(Iff(f, prenex), logic=BOOL)

    @skipIfNoSolverForLogic(BOOL)
    def test_prenex_simple_forall(self):
        a,b = (Symbol(x) for x in "ab")
        f = Or(b, ForAll([b], Implies(a, b)))
        prenex = prenex_normal_form(f)
        self.assertTrue(prenex.is_forall())
        self.assertValid(Iff(f, prenex), logic=BOOL)

    @skipIfNoSolverForLogic(BOOL)
    def test_prenex_negated_exists(self):
        a,b = (Symbol(x) for x in "ab")
        f = Implies(Exists([b], Implies(a, b)), b)
        prenex = prenex_normal_form(f)
        self.assertTrue(prenex.is_forall())
        self.assertValid(Iff(f, prenex), logic=BOOL)

    @skipIfNoSolverForLogic(BOOL)
    def test_prenex_negated_forall(self):
        a,b = (Symbol(x) for x in "ab")
        f = Implies(ForAll([b], Implies(a, b)), b)
        prenex = prenex_normal_form(f)
        self.assertTrue(prenex.is_exists())
        self.assertValid(Iff(f, prenex), logic=BOOL)

    def test_prenex_examples(self):
        for (f, _, _, logic) in get_example_formulae():
            if self.env.factory.has_solvers(logic=logic):
                prenex = prenex_normal_form(f)
                if ( prenex is not None):
                    try:
                        ok = is_valid(Iff(f, prenex), logic=logic)
                    except SolverReturnedUnknownResultError:
                        ok = not logic.quantifier_free
                    self.assertTrue(ok)

    @skipIfNoSolverForLogic(QF_AUFLIA)
    def test_ackermannization_unary(self):
        self.env.enable_infix_notation = True
        a, b = (Symbol(x, INT) for x in "ab")
        f, g, h = (Symbol(x, FunctionType(INT, [INT])) for x in "fgh")

        formula1 = Not(Equals(f(g(h(a))),
                              f(g(h(b)))))
        formula2 = Equals(a, b)
        formula = And(formula1, formula2)
        self._verify_ackermannization(formula)

    @skipIfNoSolverForLogic(QF_AUFLIA)
    def test_ackermannization_pairwise(self):
        self.env.enable_infix_notation = True
        a, b, c, d = (Symbol(x, INT) for x in "abcd")
        f = Symbol("f", FunctionType(INT, [INT]))
        formula = And(Not(Equals(f(b), f(c))),
                      Equals(f(a), f(b)),
                      Equals(f(c), f(d)),
                      Equals(a, d))
        self.assertUnsat(formula)
        formula_ack = Ackermannizer().do_ackermannization(formula)
        self.assertUnsat(formula_ack)

    @skipIfNoSolverForLogic(QF_AUFLIA)
    def test_ackermannization_explicit(self):
        self.env.enable_infix_notation = True
        a,b = (Symbol(x, INT) for x in "ab")
        f,g = (Symbol(x, FunctionType(INT, [INT, INT])) for x in "fg")
        h = Symbol("h", FunctionType(INT, [INT]))

        formula1 = Not(Equals(f(a, g(a, h(a))),
                              f(b, g(b, h(b)))))

        # Explicit the Ackermanization of this expression We end up
        # with a conjunction of implications that is then conjoined
        # with the original formula.
        ackermannization = Ackermannizer()
        actual_ack = ackermannization.do_ackermannization(formula1)

        terms_to_consts = ackermannization.get_term_to_const_dict()
        ack_h_a = terms_to_consts[h(a)]
        ack_h_b = terms_to_consts[h(b)]
        ack_g_a_h_a = terms_to_consts[g(a, h(a))]
        ack_g_b_h_b = terms_to_consts[g(b, h(b))]
        ack_f_a_g_a_h_a = terms_to_consts[f(a, g(a, h(a)))]
        ack_f_b_g_b_h_b = terms_to_consts[f(b, g(b, h(b)))]

        target_ack = And(
            Equals(a, b).Implies(Equals(ack_h_a, ack_h_b)),
            And(Equals(a, b),
                Equals(ack_h_a, ack_h_b)).Implies(
                    Equals(ack_g_a_h_a, ack_g_b_h_b)),
            And(Equals(a, b),
                Equals(ack_h_a, ack_h_b),
                Equals(ack_g_a_h_a, ack_g_b_h_b)).Implies(
                    Equals(ack_f_a_g_a_h_a, ack_f_b_g_b_h_b)))
        target_ack = And(target_ack,
                         Not(Equals(ack_f_a_g_a_h_a, ack_f_b_g_b_h_b)))
        self.assertValid(target_ack.Iff(actual_ack))

    @skipIfNoSolverForLogic(QF_AUFLIA)
    def test_ackermannization_binary(self):
        self.env.enable_infix_notation = True
        a,b = (Symbol(x, INT) for x in "ab")
        f,g = (Symbol(x, FunctionType(INT, [INT, INT])) for x in "fg")
        h = Symbol("h", FunctionType(INT, [INT]))

        formula1 = Not(Equals(f(a, g(a, h(a))),
                              f(b, g(b, h(b)))))

        formula2 = Equals(a, b)
        formula = And(formula1, formula2)
        self._verify_ackermannization(formula)

    def test_ackermannization_for_examples(self):
        for (f, _, _, logic) in get_example_formulae():
            if not logic.is_quantified() and logic.theory.uninterpreted:
                if self.env.factory.has_solvers(logic=logic):
                    self._verify_ackermannization(f)

    def test_ackermannization_dictionaries(self):
        self.env.enable_infix_notation = True
        a,b = (Symbol(x, INT) for x in "ab")
        f,g = (Symbol(x, FunctionType(INT, [INT, INT])) for x in "fg")
        h = Symbol("h", FunctionType(INT, [INT]))

        formula1 = Not(Equals(f(a, g(a, h(a))),
                              f(b, g(b, h(b)))))
        formula2 = Equals(a, b)
        formula = And(formula1, formula2)
        ackermannization = Ackermannizer()
        _ = ackermannization.do_ackermannization(formula)
        terms_to_consts = ackermannization.get_term_to_const_dict()
        consts_to_terms = ackermannization.get_const_to_term_dict()
        # The maps have the same length
        self.assertEqual(len(terms_to_consts), len(consts_to_terms))
        # The maps are the inverse of each other
        for t in terms_to_consts:
            self.assertEqual(t, consts_to_terms[terms_to_consts[t]])
        # Check that the the functions are there
        for atom in formula.get_atoms():
            if atom.is_function_application():
                self.assertIsNotNone(terms_to_consts[atom])

    def _verify_ackermannization(self, formula):
        ackermannization = Ackermannizer()
        ack = ackermannization.do_ackermannization(formula)
        #verify that there are no functions in ack
        atoms = ack.get_atoms()
        for atom in atoms:
            for arg in atom.args():
                self.assertFalse(arg.is_function_application())
        #verify that ack and formula are equisat
        formula_sat = is_sat(formula)
        ack_sat = is_sat(ack)
        self.assertTrue(formula_sat == ack_sat)

    def test_nnf_examples(self):
        for (f, _, _, logic) in get_example_formulae():
            if self.env.factory.has_solvers(logic=logic):
                rf = nnf(f)
                try:
                    ok = is_valid(Iff(f, rf), logic=logic)
                except SolverReturnedUnknownResultError:
                    ok = not logic.quantifier_free
                self.assertTrue(ok)

    def test_conj_partitioning(self):
        for (f, _, _, logic) in get_example_formulae():
            if self.env.factory.has_solvers(logic=logic):
                conjuncts = list(conjunctive_partition(f))
                try:
                    ok = is_valid(Iff(f, And(conjuncts)), logic=logic)
                except SolverReturnedUnknownResultError:
                    ok = not logic.quantifier_free
                self.assertTrue(ok)

    def test_disj_partitioning(self):
        for (f, _, _, logic) in get_example_formulae():
            if self.env.factory.has_solvers(logic=logic):
                disjuncts = list(disjunctive_partition(f))
                try:
                    ok = is_valid(Iff(f, Or(disjuncts)), logic=logic)
                except SolverReturnedUnknownResultError:
                    ok = not logic.quantifier_free
                self.assertTrue(ok)

    def test_aig_examples(self):
        for (f, _, _, logic) in get_example_formulae():
            if self.env.factory.has_solvers(logic=logic):
                f_aig = aig(f)
                try:
                    ok = is_valid(Iff(f, f_aig), logic=logic)
                except SolverReturnedUnknownResultError:
                    ok = not logic.quantifier_free
                self.assertTrue(ok, "Was: %s\n Got:%s" % (f, f_aig))

    @skipIfNoSolverForLogic(QF_NRA)
    def test_times_distributivity(self):
        r = Symbol("r", REAL)
        s = Symbol("s", REAL)
        td = TimesDistributor()

        f = Times(Plus(r, Real(1)), Real(3))
        fp = td.walk(f)
        self.assertValid(Equals(f, fp), (f, fp))

        f = Times(Plus(r, Real(1)), s)
        fp = td.walk(f)
        self.assertValid(Equals(f, fp), (f, fp))

        f = Times(Plus(r, Real(1), s), Real(3))
        fp = td.walk(f)
        self.assertValid(Equals(f, fp), (f, fp))

        f = Times(Minus(r, Real(1)), Real(3))
        fp = td.walk(f)
        self.assertValid(Equals(f, fp), (f, fp))

        f = Times(Minus(r, Real(1)), s)
        fp = td.walk(f)
        self.assertValid(Equals(f, fp), (f, fp))

        f = Times(Minus(Real(1), s), Real(3))
        fp = td.walk(f)
        self.assertValid(Equals(f, fp), (f, fp))

        f = Times(Minus(r, Real(1)), Plus(r, s))
        fp = td.walk(f)
        self.assertValid(Equals(f, fp), (f, fp))

        # (r + 1) * (s-1) = r*s + (-r) + s - 1
        f = Times(Plus(r, Real(1)), Minus(s, Real(1)))
        fp = td.walk(f).simplify()
        target = Plus(Times(r, s),
                      Times(r, Real(-1)),
                      s,
                      Real(-1))
        self.assertValid(Equals(fp, target), fp)
        self.assertTrue(fp.is_plus(), fp)

    @skipIfNoSolverForLogic(QF_NRA)
    def test_times_distributivity_smtlib_nra(self):
        from pysmt.test.smtlib.parser_utils import formulas_from_smtlib_test_set
        test_set = formulas_from_smtlib_test_set(logics=[QF_LRA, QF_NRA])
        for (_, fname, f, _) in test_set:
            td = TimesDistributor()
            _ = td.walk(f)
            for (old, new) in td.memoization.items():
                if not old.is_times(): continue
                if old is new: continue # Nothing changed
                self.assertValid(Equals(old, new),
                                 (old, new), solver_name="z3")

if __name__ == "__main__":
    main()
<commit_msg>added test for toplevel-propagator<commit_after>#
# This file is part of pySMT.
#
#   Copyright 2014 Andrea Micheli and Marco Gario
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
from pysmt.shortcuts import (And, Iff, Or, Symbol, Implies, Not,
                             Exists, ForAll,
                             Times, Plus, Minus, Equals, Real,
                             is_valid, is_sat, Function)
from pysmt.test import TestCase, skipIfNoSolverForLogic, main
from pysmt.rewritings import prenex_normal_form, nnf, conjunctive_partition, aig
from pysmt.rewritings import disjunctive_partition, propagate_toplevel
from pysmt.rewritings import TimesDistributor, Ackermannizer
from pysmt.test.examples import get_example_formulae
from pysmt.exceptions import SolverReturnedUnknownResultError
from pysmt.logics import BOOL, QF_NRA, QF_LRA, QF_AUFLIA
from pysmt.typing import REAL, INT, FunctionType

class TestRewritings(TestCase):

    def test_prenex_basic(self):
        a,b,c = (Symbol(x) for x in "abc")
        f = Not(And(a, Exists([b], And(a, b)), ForAll([c], Or(a, c))))
        prenex = prenex_normal_form(f)
        # Two prenex normal forms are possible
        my_prenex_1 = Exists([c], ForAll([b], Not(And(a, And(a, b), Or(a, c)))))
        my_prenex_2 = ForAll([b], Exists([c], Not(And(a, And(a, b), Or(a, c)))))
        self.assertTrue(prenex == my_prenex_1 or prenex == my_prenex_2)

    @skipIfNoSolverForLogic(BOOL)
    def test_prenex_simple_exists(self):
        a,b = (Symbol(x) for x in "ab")
        f = And(b, Exists([b], Implies(a, b)))
        prenex = prenex_normal_form(f)
        self.assertTrue(prenex.is_exists())
        self.assertValid(Iff(f, prenex), logic=BOOL)

    @skipIfNoSolverForLogic(BOOL)
    def test_prenex_simple_forall(self):
        a,b = (Symbol(x) for x in "ab")
        f = Or(b, ForAll([b], Implies(a, b)))
        prenex = prenex_normal_form(f)
        self.assertTrue(prenex.is_forall())
        self.assertValid(Iff(f, prenex), logic=BOOL)

    @skipIfNoSolverForLogic(BOOL)
    def test_prenex_negated_exists(self):
        a,b = (Symbol(x) for x in "ab")
        f = Implies(Exists([b], Implies(a, b)), b)
        prenex = prenex_normal_form(f)
        self.assertTrue(prenex.is_forall())
        self.assertValid(Iff(f, prenex), logic=BOOL)

    @skipIfNoSolverForLogic(BOOL)
    def test_prenex_negated_forall(self):
        a,b = (Symbol(x) for x in "ab")
        f = Implies(ForAll([b], Implies(a, b)), b)
        prenex = prenex_normal_form(f)
        self.assertTrue(prenex.is_exists())
        self.assertValid(Iff(f, prenex), logic=BOOL)

    def test_prenex_examples(self):
        for (f, _, _, logic) in get_example_formulae():
            if self.env.factory.has_solvers(logic=logic):
                prenex = prenex_normal_form(f)
                if ( prenex is not None):
                    try:
                        ok = is_valid(Iff(f, prenex), logic=logic)
                    except SolverReturnedUnknownResultError:
                        ok = not logic.quantifier_free
                    self.assertTrue(ok)

    @skipIfNoSolverForLogic(QF_AUFLIA)
    def test_ackermannization_unary(self):
        self.env.enable_infix_notation = True
        a, b = (Symbol(x, INT) for x in "ab")
        f, g, h = (Symbol(x, FunctionType(INT, [INT])) for x in "fgh")

        formula1 = Not(Equals(f(g(h(a))),
                              f(g(h(b)))))
        formula2 = Equals(a, b)
        formula = And(formula1, formula2)
        self._verify_ackermannization(formula)

    @skipIfNoSolverForLogic(QF_AUFLIA)
    def test_ackermannization_pairwise(self):
        self.env.enable_infix_notation = True
        a, b, c, d = (Symbol(x, INT) for x in "abcd")
        f = Symbol("f", FunctionType(INT, [INT]))
        formula = And(Not(Equals(f(b), f(c))),
                      Equals(f(a), f(b)),
                      Equals(f(c), f(d)),
                      Equals(a, d))
        self.assertUnsat(formula)
        formula_ack = Ackermannizer().do_ackermannization(formula)
        self.assertUnsat(formula_ack)

    @skipIfNoSolverForLogic(QF_AUFLIA)
    def test_ackermannization_explicit(self):
        self.env.enable_infix_notation = True
        a,b = (Symbol(x, INT) for x in "ab")
        f,g = (Symbol(x, FunctionType(INT, [INT, INT])) for x in "fg")
        h = Symbol("h", FunctionType(INT, [INT]))

        formula1 = Not(Equals(f(a, g(a, h(a))),
                              f(b, g(b, h(b)))))

        # Explicit the Ackermanization of this expression We end up
        # with a conjunction of implications that is then conjoined
        # with the original formula.
        ackermannization = Ackermannizer()
        actual_ack = ackermannization.do_ackermannization(formula1)

        terms_to_consts = ackermannization.get_term_to_const_dict()
        ack_h_a = terms_to_consts[h(a)]
        ack_h_b = terms_to_consts[h(b)]
        ack_g_a_h_a = terms_to_consts[g(a, h(a))]
        ack_g_b_h_b = terms_to_consts[g(b, h(b))]
        ack_f_a_g_a_h_a = terms_to_consts[f(a, g(a, h(a)))]
        ack_f_b_g_b_h_b = terms_to_consts[f(b, g(b, h(b)))]

        target_ack = And(
            Equals(a, b).Implies(Equals(ack_h_a, ack_h_b)),
            And(Equals(a, b),
                Equals(ack_h_a, ack_h_b)).Implies(
                    Equals(ack_g_a_h_a, ack_g_b_h_b)),
            And(Equals(a, b),
                Equals(ack_h_a, ack_h_b),
                Equals(ack_g_a_h_a, ack_g_b_h_b)).Implies(
                    Equals(ack_f_a_g_a_h_a, ack_f_b_g_b_h_b)))
        target_ack = And(target_ack,
                         Not(Equals(ack_f_a_g_a_h_a, ack_f_b_g_b_h_b)))
        self.assertValid(target_ack.Iff(actual_ack))

    @skipIfNoSolverForLogic(QF_AUFLIA)
    def test_ackermannization_binary(self):
        self.env.enable_infix_notation = True
        a,b = (Symbol(x, INT) for x in "ab")
        f,g = (Symbol(x, FunctionType(INT, [INT, INT])) for x in "fg")
        h = Symbol("h", FunctionType(INT, [INT]))

        formula1 = Not(Equals(f(a, g(a, h(a))),
                              f(b, g(b, h(b)))))

        formula2 = Equals(a, b)
        formula = And(formula1, formula2)
        self._verify_ackermannization(formula)

    def test_ackermannization_for_examples(self):
        for (f, _, _, logic) in get_example_formulae():
            if not logic.is_quantified() and logic.theory.uninterpreted:
                if self.env.factory.has_solvers(logic=logic):
                    self._verify_ackermannization(f)

    def test_ackermannization_dictionaries(self):
        self.env.enable_infix_notation = True
        a,b = (Symbol(x, INT) for x in "ab")
        f,g = (Symbol(x, FunctionType(INT, [INT, INT])) for x in "fg")
        h = Symbol("h", FunctionType(INT, [INT]))

        formula1 = Not(Equals(f(a, g(a, h(a))),
                              f(b, g(b, h(b)))))
        formula2 = Equals(a, b)
        formula = And(formula1, formula2)
        ackermannization = Ackermannizer()
        _ = ackermannization.do_ackermannization(formula)
        terms_to_consts = ackermannization.get_term_to_const_dict()
        consts_to_terms = ackermannization.get_const_to_term_dict()
        # The maps have the same length
        self.assertEqual(len(terms_to_consts), len(consts_to_terms))
        # The maps are the inverse of each other
        for t in terms_to_consts:
            self.assertEqual(t, consts_to_terms[terms_to_consts[t]])
        # Check that the the functions are there
        for atom in formula.get_atoms():
            if atom.is_function_application():
                self.assertIsNotNone(terms_to_consts[atom])

    def _verify_ackermannization(self, formula):
        ackermannization = Ackermannizer()
        ack = ackermannization.do_ackermannization(formula)
        #verify that there are no functions in ack
        atoms = ack.get_atoms()
        for atom in atoms:
            for arg in atom.args():
                self.assertFalse(arg.is_function_application())
        #verify that ack and formula are equisat
        formula_sat = is_sat(formula)
        ack_sat = is_sat(ack)
        self.assertTrue(formula_sat == ack_sat)

    def test_nnf_examples(self):
        for (f, _, _, logic) in get_example_formulae():
            if self.env.factory.has_solvers(logic=logic):
                rf = nnf(f)
                try:
                    ok = is_valid(Iff(f, rf), logic=logic)
                except SolverReturnedUnknownResultError:
                    ok = not logic.quantifier_free
                self.assertTrue(ok)

    def test_conj_partitioning(self):
        for (f, _, _, logic) in get_example_formulae():
            if self.env.factory.has_solvers(logic=logic):
                conjuncts = list(conjunctive_partition(f))
                try:
                    ok = is_valid(Iff(f, And(conjuncts)), logic=logic)
                except SolverReturnedUnknownResultError:
                    ok = not logic.quantifier_free
                self.assertTrue(ok)

    def test_disj_partitioning(self):
        for (f, _, _, logic) in get_example_formulae():
            if self.env.factory.has_solvers(logic=logic):
                disjuncts = list(disjunctive_partition(f))
                try:
                    ok = is_valid(Iff(f, Or(disjuncts)), logic=logic)
                except SolverReturnedUnknownResultError:
                    ok = not logic.quantifier_free
                self.assertTrue(ok)

    def test_propagate_toplevel(self):
       for (f, _, _, logic) in get_example_formulae():
            if self.env.factory.has_solvers(logic=logic):
                rwf = propagate_toplevel(f)
                try:
                    ok = is_valid(Iff(f, rwf), logic=logic)
                except SolverReturnedUnknownResultError:
                    ok = not logic.quantifier_free
                self.assertTrue(ok)

    def test_aig_examples(self):
        for (f, _, _, logic) in get_example_formulae():
            if self.env.factory.has_solvers(logic=logic):
                f_aig = aig(f)
                try:
                    ok = is_valid(Iff(f, f_aig), logic=logic)
                except SolverReturnedUnknownResultError:
                    ok = not logic.quantifier_free
                self.assertTrue(ok, "Was: %s\n Got:%s" % (f, f_aig))

    @skipIfNoSolverForLogic(QF_NRA)
    def test_times_distributivity(self):
        r = Symbol("r", REAL)
        s = Symbol("s", REAL)
        td = TimesDistributor()

        f = Times(Plus(r, Real(1)), Real(3))
        fp = td.walk(f)
        self.assertValid(Equals(f, fp), (f, fp))

        f = Times(Plus(r, Real(1)), s)
        fp = td.walk(f)
        self.assertValid(Equals(f, fp), (f, fp))

        f = Times(Plus(r, Real(1), s), Real(3))
        fp = td.walk(f)
        self.assertValid(Equals(f, fp), (f, fp))

        f = Times(Minus(r, Real(1)), Real(3))
        fp = td.walk(f)
        self.assertValid(Equals(f, fp), (f, fp))

        f = Times(Minus(r, Real(1)), s)
        fp = td.walk(f)
        self.assertValid(Equals(f, fp), (f, fp))

        f = Times(Minus(Real(1), s), Real(3))
        fp = td.walk(f)
        self.assertValid(Equals(f, fp), (f, fp))

        f = Times(Minus(r, Real(1)), Plus(r, s))
        fp = td.walk(f)
        self.assertValid(Equals(f, fp), (f, fp))

        # (r + 1) * (s-1) = r*s + (-r) + s - 1
        f = Times(Plus(r, Real(1)), Minus(s, Real(1)))
        fp = td.walk(f).simplify()
        target = Plus(Times(r, s),
                      Times(r, Real(-1)),
                      s,
                      Real(-1))
        self.assertValid(Equals(fp, target), fp)
        self.assertTrue(fp.is_plus(), fp)

    @skipIfNoSolverForLogic(QF_NRA)
    def test_times_distributivity_smtlib_nra(self):
        from pysmt.test.smtlib.parser_utils import formulas_from_smtlib_test_set
        test_set = formulas_from_smtlib_test_set(logics=[QF_LRA, QF_NRA])
        for (_, fname, f, _) in test_set:
            td = TimesDistributor()
            _ = td.walk(f)
            for (old, new) in td.memoization.items():
                if not old.is_times(): continue
                if old is new: continue # Nothing changed
                self.assertValid(Equals(old, new),
                                 (old, new), solver_name="z3")

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import pytest

from pmxbot import util

def test_MongoDBKarma(mongodb_uri):
	k = util.Karma.from_URI(mongodb_uri)
	k.db = k.db.database.connection[k.db.database.name+'_test'][k.db.name]
	k.db.drop()
	try:
		k.change('foo', 1)
		k.change('bar', 1)
		k.set('baz', 3)
		k.set('baz', 2)
		k.link('foo', 'bar')
		assert k.lookup('foo') == 2
		k.link('foo', 'baz')
		assert k.lookup('baz') == k.lookup('foo') == 4
	finally:
		k.db.drop()

def test_MongoDBQuotes(mongodb_uri):
	q = util.Quotes.from_URI(mongodb_uri)
	q.lib = 'test'
	clean = lambda: q.db.remove({'library': 'test'})
	clean()
	try:
		q.quoteAdd('who would ever say such a thing')
		q.quoteAdd('go ahead, take my pay')
		q.quoteAdd("let's do the Time Warp again")
		q.quoteLookup('time warp')
		q.quoteLookup('nonexistent')
	finally:
		clean()

@pytest.has_internet
def test_lookup():
	assert util.lookup('dachshund') is not None
<commit_msg>Adding test revealing error in karma accounting<commit_after>import pytest

from pmxbot import util

def test_MongoDBKarma(mongodb_uri):
	k = util.Karma.from_URI(mongodb_uri)
	k.db = k.db.database.connection[k.db.database.name+'_test'][k.db.name]
	k.db.drop()
	try:
		k.change('foo', 1)
		k.change('bar', 1)
		k.set('baz', 3)
		k.set('baz', 2)
		k.link('foo', 'bar')
		assert k.lookup('foo') == 2
		k.link('foo', 'baz')
		assert k.lookup('baz') == k.lookup('foo') == 4
		k.change('foo', 1)
		assert k.lookup('foo') == k.lookup('bar') == 5
	finally:
		k.db.drop()

def test_MongoDBQuotes(mongodb_uri):
	q = util.Quotes.from_URI(mongodb_uri)
	q.lib = 'test'
	clean = lambda: q.db.remove({'library': 'test'})
	clean()
	try:
		q.quoteAdd('who would ever say such a thing')
		q.quoteAdd('go ahead, take my pay')
		q.quoteAdd("let's do the Time Warp again")
		q.quoteLookup('time warp')
		q.quoteLookup('nonexistent')
	finally:
		clean()

@pytest.has_internet
def test_lookup():
	assert util.lookup('dachshund') is not None
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (c) 2014 Spotify AB
from __future__ import absolute_import, division, print_function

import os

from ramlfications.validate import RAMLValidationError, validate

from .base import BaseTestCase

class TestValidateRAML(BaseTestCase):
    def setUp(self):
        self.here = os.path.abspath(os.path.dirname(__file__))

    def fail_validate(self, raml_file, expected_msg):
        e = self.assert_raises(RAMLValidationError, validate,
                               ramlfile=raml_file)

        self.assertEqual(expected_msg, str(e))

    def test_validate_raml_header(self):
        invalid_header = "examples/incorrect-raml-header.raml"
        raml_file = os.path.join(self.here, invalid_header)
        expected_msg = 'Not a valid RAML header: #%FOO.'

        self.fail_validate(raml_file, expected_msg)

    def test_validate_raml_version(self):
        invalid_header = "examples/invalid-version-raml-header.raml"
        raml_file = os.path.join(self.here, invalid_header)
        expected_msg = 'Not a valid version of RAML: 0.9.'

        self.fail_validate(raml_file, expected_msg)

    def test_validate_title(self):
        no_title = "examples/validate/no-title.raml"
        raml_file = os.path.join(self.here, no_title)
        expected_msg = 'RAML File does not define an API title.'

        self.fail_validate(raml_file, expected_msg)

    def test_validate_version(self):
        no_version = "examples/validate/no-version.raml"
        raml_file = os.path.join(self.here, no_version)
        expected_msg = 'RAML File does not define an API version.'

        self.fail_validate(raml_file, expected_msg)

    def test_validate_base_uri(self):
        no_base_uri = "examples/validate/no-base-uri.raml"
        raml_file = os.path.join(self.here, no_base_uri)
        expected_msg = 'RAML File does not define the baseUri.'

        self.fail_validate(raml_file, expected_msg)

    def test_validate_base_params(self):
        no_default_param = "examples/validate/no-default-base-uri-params.raml"
        raml_file = os.path.join(self.here, no_default_param)
        expected_msg = "'domainName' needs a default parameter."

        self.fail_validate(raml_file, expected_msg)

    def test_validate_resource_responses(self):
        invalid_responses = "examples/validate/responses.raml"
        raml_file = os.path.join(self.here, invalid_responses)

        expected_msg = "'anInvalidKey' not a valid Response parameter."

        self.fail_validate(raml_file, expected_msg)

    def test_validate_docs_title(self):
        docs_no_title = "examples/validate/docs-no-title.raml"
        raml_file = os.path.join(self.here, docs_no_title)

        expected_msg = "API Documentation requires a title."

        self.fail_validate(raml_file, expected_msg)

    def test_validate_docs_content(self):
        no_content = "examples/validate/docs-no-content.raml"
        raml_file = os.path.join(self.here, no_content)
        expected_msg = "API Documentation requires content defined."

        self.fail_validate(raml_file, expected_msg)

    def test_validate_security_scheme(self):
        invalid_sec_scheme = "examples/validate/invalid-security-scheme.raml"
        raml_file = os.path.join(self.here, invalid_sec_scheme)
        expected_msg = "'Invalid Scheme' is not a valid Security Scheme."

        self.fail_validate(raml_file, expected_msg)
<commit_msg>tests for load within validate<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (c) 2014 Spotify AB
from __future__ import absolute_import, division, print_function

import os

from ramlfications.loader import load
from ramlfications.validate import RAMLValidationError, validate

from .base import BaseTestCase

class TestValidateRAML(BaseTestCase):
    def setUp(self):
        self.here = os.path.abspath(os.path.dirname(__file__))

    def fail_validate(self, raml_file, expected_msg):
        self.load_object = load(raml_file)
        e = self.assert_raises(RAMLValidationError, validate,
                               load_object=self.load_object)

        self.assertEqual(expected_msg, str(e))

    def test_validate_raml_header(self):
        invalid_header = "examples/incorrect-raml-header.raml"
        raml_file = os.path.join(self.here, invalid_header)
        expected_msg = 'Not a valid RAML header: #%FOO.'

        self.fail_validate(raml_file, expected_msg)

    def test_validate_raml_version(self):
        invalid_header = "examples/invalid-version-raml-header.raml"
        raml_file = os.path.join(self.here, invalid_header)
        expected_msg = 'Not a valid version of RAML: 0.9.'

        self.fail_validate(raml_file, expected_msg)

    def test_validate_title(self):
        no_title = "examples/validate/no-title.raml"
        raml_file = os.path.join(self.here, no_title)
        expected_msg = 'RAML File does not define an API title.'

        self.fail_validate(raml_file, expected_msg)

    def test_validate_version(self):
        no_version = "examples/validate/no-version.raml"
        raml_file = os.path.join(self.here, no_version)
        expected_msg = 'RAML File does not define an API version.'

        self.fail_validate(raml_file, expected_msg)

    def test_validate_base_uri(self):
        no_base_uri = "examples/validate/no-base-uri.raml"
        raml_file = os.path.join(self.here, no_base_uri)
        expected_msg = 'RAML File does not define the baseUri.'

        self.fail_validate(raml_file, expected_msg)

    def test_validate_base_params(self):
        no_default_param = "examples/validate/no-default-base-uri-params.raml"
        raml_file = os.path.join(self.here, no_default_param)
        expected_msg = "'domainName' needs a default parameter."

        self.fail_validate(raml_file, expected_msg)

    def test_validate_resource_responses(self):
        invalid_responses = "examples/validate/responses.raml"
        raml_file = os.path.join(self.here, invalid_responses)

        expected_msg = "'anInvalidKey' not a valid Response parameter."

        self.fail_validate(raml_file, expected_msg)

    def test_validate_docs_title(self):
        docs_no_title = "examples/validate/docs-no-title.raml"
        raml_file = os.path.join(self.here, docs_no_title)

        expected_msg = "API Documentation requires a title."

        self.fail_validate(raml_file, expected_msg)

    def test_validate_docs_content(self):
        no_content = "examples/validate/docs-no-content.raml"
        raml_file = os.path.join(self.here, no_content)
        expected_msg = "API Documentation requires content defined."

        self.fail_validate(raml_file, expected_msg)

    def test_validate_security_scheme(self):
        invalid_sec_scheme = "examples/validate/invalid-security-scheme.raml"
        raml_file = os.path.join(self.here, invalid_sec_scheme)
        expected_msg = "'Invalid Scheme' is not a valid Security Scheme."

        self.fail_validate(raml_file, expected_msg)
<|endoftext|>


<commit_before>#!/usr/bin/python
#
#  Copyright 2002-2018 Barcelona Supercomputing Center (www.bsc.es)
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

# -*- coding: utf-8 -*-

"""
PyCOMPSs Utils - Location
===================
    This file contains the methods to detect the origin of the call stack.
    Useful to detect if we are in the master or in the worker.
"""

import inspect
from pycompss.runtime.commons import IS_PYTHON3
from pycompss.runtime.commons import IS_INTERACTIVE

DECORATORS_TO_CHECK = ['pycompss/api/task.py',
                       'pycompss/api/constraint.py',
                       'pycompss/api/implement.py']

def i_am_at_master():
    """
    Determine if the execution is being performed in the master node
    # if 'pycompss/runtime/launch.py' in inspect.stack()[-1][1]: --> I am at master
    # if inspect.stack()[-2][3] == 'compss_main' --> I am at master
    # if 'pycompss/interactive.py' in inspect.stack()[3][1] --> I am at master in interactive mode
    # if 'pycompss/api/task.py' in inspect.stack()[2][1]    --> I am at master in interactive task decorator
    # if 'pycompss/api/task.py' in inspect.stack()[1][1]    --> I am at master in interactive task decorator

    :return: <Boolean> - True if we are in the master node.
    """

    stack = inspect.stack()
    if IS_INTERACTIVE:
        if IS_PYTHON3:
            return 'pycompss/interactive.py' in stack[9][1] \
                   or any(decorator in stack[2][1] for decorator in DECORATORS_TO_CHECK) \
                   or any(decorator in stack[1][1] for decorator in DECORATORS_TO_CHECK)
        else:
            return 'pycompss/interactive.py' in stack[3][1] \
                   or any(decorator in stack[2][1] for decorator in DECORATORS_TO_CHECK) \
                   or any(decorator in stack[1][1] for decorator in DECORATORS_TO_CHECK)
    else:
        return 'pycompss/runtime/launch.py' in stack[-1][1]

def i_am_at_worker():
    """
    Determine if the execution is being performed in a worker node.
    # if (inspect.stack()[-2][3] == 'compss_worker' or
    #     inspect.stack()[-2][3] == 'compss_persistent_worker'): --> I am at worker

    :return: <Boolean> - True if we are in a worker node.
    """

    return inspect.stack()[-2][3] in ['compss_worker', 'compss_persistent_worker']

def i_am_within_scope():
    """
    Determine if the execution is being performed within the PyCOMPSs scope.

    :return:  <Boolean> - True if under scope. False on the contrary.
    """

    return i_am_at_master() or i_am_at_worker()
    # Old way: - Conflicts with dataClay
    # import sys
    # return sys.path[0].endswith('Bindings/python/2/pycompss/runtime') or \
    #        sys.path[0].endswith('Bindings/python/2/pycompss/worker') or \
    #        sys.path[0].endswith('Bindings/python/3/pycompss/runtime') or \
    #        sys.path[0].endswith('Bindings/python/3/pycompss/worker') or \
    #        sys.path[0] == ''
<commit_msg>Fixed location when using other decorators in jupyter-notebooks.<commit_after>#!/usr/bin/python
#
#  Copyright 2002-2018 Barcelona Supercomputing Center (www.bsc.es)
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

# -*- coding: utf-8 -*-

"""
PyCOMPSs Utils - Location
===================
    This file contains the methods to detect the origin of the call stack.
    Useful to detect if we are in the master or in the worker.
"""

import inspect
from pycompss.runtime.commons import IS_PYTHON3
from pycompss.runtime.commons import IS_INTERACTIVE

DECORATORS_TO_CHECK = ['pycompss/api/binary.py',
                       'pycompss/api/constraint.py',
                       'pycompss/api/decaf.py',
                       'pycompss/api/implement.py',
                       'pycompss/api/mpi.py',
                       'pycompss/api/ompss.py',
                       'pycompss/api/opencl.py',
                       'pycompss/api/parallel.py',
                       'pycompss/api/task.py']

def i_am_at_master():
    """
    Determine if the execution is being performed in the master node
    # if 'pycompss/runtime/launch.py' in inspect.stack()[-1][1]: --> I am at master
    # if inspect.stack()[-2][3] == 'compss_main' --> I am at master
    # if 'pycompss/interactive.py' in inspect.stack()[3][1] --> I am at master in interactive mode
    # if 'pycompss/api/task.py' in inspect.stack()[2][1]    --> I am at master in interactive task decorator
    # if 'pycompss/api/task.py' in inspect.stack()[1][1]    --> I am at master in interactive task decorator

    :return: <Boolean> - True if we are in the master node.
    """

    stack = inspect.stack()
    if IS_INTERACTIVE:
        if IS_PYTHON3:
            return 'pycompss/interactive.py' in stack[9][1] \
                   or any(decorator in stack[2][1] for decorator in DECORATORS_TO_CHECK) \
                   or any(decorator in stack[1][1] for decorator in DECORATORS_TO_CHECK)
        else:
            return 'pycompss/interactive.py' in stack[3][1] \
                   or any(decorator in stack[2][1] for decorator in DECORATORS_TO_CHECK) \
                   or any(decorator in stack[1][1] for decorator in DECORATORS_TO_CHECK)
    else:
        return 'pycompss/runtime/launch.py' in stack[-1][1]

def i_am_at_worker():
    """
    Determine if the execution is being performed in a worker node.
    # if (inspect.stack()[-2][3] == 'compss_worker' or
    #     inspect.stack()[-2][3] == 'compss_persistent_worker'): --> I am at worker

    :return: <Boolean> - True if we are in a worker node.
    """

    return inspect.stack()[-2][3] in ['compss_worker', 'compss_persistent_worker']

def i_am_within_scope():
    """
    Determine if the execution is being performed within the PyCOMPSs scope.

    :return:  <Boolean> - True if under scope. False on the contrary.
    """

    return i_am_at_master() or i_am_at_worker()
    # Old way: - Conflicts with dataClay
    # import sys
    # return sys.path[0].endswith('Bindings/python/2/pycompss/runtime') or \
    #        sys.path[0].endswith('Bindings/python/2/pycompss/worker') or \
    #        sys.path[0].endswith('Bindings/python/3/pycompss/runtime') or \
    #        sys.path[0].endswith('Bindings/python/3/pycompss/worker') or \
    #        sys.path[0] == ''
<|endoftext|>


<commit_before>from celery.utils import uuid
from celery.events import Event

from HTMLParser import HTMLParser
import xml.etree.ElementTree as ET

class HtmlTableParser(HTMLParser):
    def __init__(self, *args, **kwargs):
        HTMLParser.__init__(self, *args, **kwargs)
        self.table = ''
        self.inTable = False

    def handle_starttag(self, tag, attrs):
        if tag == 'table':
            self.inTable = True
        if self.inTable:
            self.table += '<%s' % tag
            for attr in attrs:
                self.table += ' %s="%s"' % attr
            self.table += '>'

    def handle_endtag(self, tag):
        if self.inTable:
            self.table += '</%s>' % tag
            if tag == 'table':
                self.inTable = False

    def handle_data(self, data):
        if self.inTable:
            self.table += data

    def parse(self, source):
        self.feed(source)

    def query(self, pattern):
        root = ET.fromstring(self.table)
        return root.findall(pattern)

    def rows(self):
        return self.query('tbody/tr')

    def get_row(self, row_id):
        row = self.query('tbody/tr[@id="%s"]' % row_id)
        if not row:
            return None
        cells = row[0].findall('td')
        return map(lambda x: getattr(x, 'text'), cells)

def task_succeeded_events(worker, id=None, name=None):
    id = id or uuid()
    name = name or 'sometask'
    return [Event('task-received', uuid=id, name=name,
                    args='(2, 2)', kwargs="{'foo': 'bar'}",
                    retries=0, eta=None, hostname=worker),
            Event('task-started', uuid=id, hostname=worker),
            Event('task-succeeded', uuid=id, result='4',
                    runtime=0.1234, hostname=worker)]

def task_failed_events(worker, id=None, name=None):
    id = id or uuid()
    name = name or 'sometask'
    return [Event('task-received', uuid=id, name=name,
                    args='(2, 2)', kwargs="{'foo': 'bar'}",
                    retries=0, eta=None, hostname=worker),
            Event('task-started', uuid=id, hostname=worker),
            Event('task-failed', uuid=id, exception="KeyError('foo')",
                    traceback='line 1 at main', hostname=worker)]
<commit_msg>Fix HTMLParser import for python3<commit_after>from celery.utils import uuid
from celery.events import Event

try:
    from HTMLParser import HTMLParser
except ImportError:
    from html.parser import HTMLParser
import xml.etree.ElementTree as ET

class HtmlTableParser(HTMLParser):
    def __init__(self, *args, **kwargs):
        HTMLParser.__init__(self, *args, **kwargs)
        self.table = ''
        self.inTable = False

    def handle_starttag(self, tag, attrs):
        if tag == 'table':
            self.inTable = True
        if self.inTable:
            self.table += '<%s' % tag
            for attr in attrs:
                self.table += ' %s="%s"' % attr
            self.table += '>'

    def handle_endtag(self, tag):
        if self.inTable:
            self.table += '</%s>' % tag
            if tag == 'table':
                self.inTable = False

    def handle_data(self, data):
        if self.inTable:
            self.table += data

    def parse(self, source):
        self.feed(source)

    def query(self, pattern):
        root = ET.fromstring(self.table)
        return root.findall(pattern)

    def rows(self):
        return self.query('tbody/tr')

    def get_row(self, row_id):
        row = self.query('tbody/tr[@id="%s"]' % row_id)
        if not row:
            return None
        cells = row[0].findall('td')
        return map(lambda x: getattr(x, 'text'), cells)

def task_succeeded_events(worker, id=None, name=None):
    id = id or uuid()
    name = name or 'sometask'
    return [Event('task-received', uuid=id, name=name,
                    args='(2, 2)', kwargs="{'foo': 'bar'}",
                    retries=0, eta=None, hostname=worker),
            Event('task-started', uuid=id, hostname=worker),
            Event('task-succeeded', uuid=id, result='4',
                    runtime=0.1234, hostname=worker)]

def task_failed_events(worker, id=None, name=None):
    id = id or uuid()
    name = name or 'sometask'
    return [Event('task-received', uuid=id, name=name,
                    args='(2, 2)', kwargs="{'foo': 'bar'}",
                    retries=0, eta=None, hostname=worker),
            Event('task-started', uuid=id, hostname=worker),
            Event('task-failed', uuid=id, exception="KeyError('foo')",
                    traceback='line 1 at main', hostname=worker)]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import logging
from time import sleep

LOG = logging.getLogger(__name__)

def build_uncomment_skip_slave_script():

    return """
        sed -e 's/^#skip_slave_start/skip_slave_start/' -i /etc/my.cnf
        """

def build_comment_skip_slave_script():

    return """
        sed -e 's/^skip_slave_start/#skip_slave_start/' -i /etc/my.cnf
        """

def build_permission_script():

    return """
        chown mysql:mysql /data
        die_if_error "Error changing datadir permission"
        chown -R mysql:mysql /data/*
        die_if_error "Error changing datadir permission"
        """

def build_server_id_conf_script():
    return """
    echo ""; echo $(date "+%Y-%m-%d %T") "- Creating the server id db file"
    \n(cat <<EOF_DBAAS
    \n[mysqld]
    \nserver_id={{SERVERID}}
    \nEOF_DBAAS
    \n) >  /etc/server_id.cnf
    """

def build_mount_snapshot_volume_script():
    return """
        mkdir /data2

        mount -t nfs -o bg,intr,nolock {{EXPORT_PATH}} /data2
        die_if_error "Error mounting volume"

        cp -rp /data2/.snapshot/{{SNAPSHOPT_NAME}}/* /data/
        die_if_error "Error coping datafiles"

        umount /data2
        rm -rf /data2

    """

def build_remove_deprecated_files_script():
    return """
        rm -f /data/data/auto.cnf
        rm -f /data/data/*.err
        rm -f /data/data/*.pid
    """

def get_client(instance):
    databaseinfra = instance.databaseinfra
    driver = databaseinfra.get_driver()

    return driver.get_client(instance)

def get_client_for_infra(databaseinfra):
    driver = databaseinfra.get_driver()

    return driver.get_client(None)

def get_replication_info(instance):
    client = get_client(instance)

    client.query('show master status')
    r = client.store_result()
    row = r.fetch_row(maxrows=0, how=1)
    return row[0]['File'], row[0]['Position']

def check_seconds_behind(instance, retries=50):
    client = get_client(instance)

    for attempt in range(retries):
        LOG.info("Checking replication %i on %s " % (attempt + 1, instance))

        client.query("show slave status")
        r = client.store_result()
        row = r.fetch_row(maxrows=0, how=1)
        seconds_behind = row[0]['Seconds_Behind_Master']

        if seconds_behind is None:
            raise Exception("Replication is not running")

        if seconds_behind == '0':
            return True

        if attempt == retries - 1:
            LOG.warning("Seconds behind: {}".format(seconds_behind))
            return False

        sleep(10)

def change_master_to(instance, master_host, bin_log_file, bin_log_position):
    client = get_client(instance)
    client.query('stop slave')

    sql_command = "change master to master_host='{}', master_log_file='{}', master_log_pos={}"
    sql_command = sql_command.format(
        master_host, bin_log_file, bin_log_position)
    client.query(sql_command)

    client.query('start slave')

def get_replication_information_from_file(host,):
    command = 'cat /data/data/mysql_binlog_master_file_pos'
    # output = {}
    # return_code = exec_remote_command_host(host, command, output)
    # if return_code != 0:
    #     raise Exception("Could not read file: {}".format(output))
    output = host.ssh.run_script(command)

    replication_file, replication_position = parse_replication_info(
        output['stdout'][0])

    return replication_file, replication_position

def parse_replication_info(replication_info):
    split_vars = replication_info.split(';')
    replication_file = split_var_and_value(split_vars[0])
    replication_position = split_vars[1]

    position_end_with_new_line = replication_position.find('\n')
    if position_end_with_new_line:
        replication_position = replication_position[
            :position_end_with_new_line]

    replication_position = split_var_and_value(replication_position)

    return replication_file, replication_position

def split_var_and_value(info):
    return info.split('=')[1]

def start_slave(instance):
    client = get_client(instance)
    client.query("start slave")
<commit_msg>Remove exec_remote_command coment<commit_after># -*- coding: utf-8 -*-
import logging
from time import sleep

LOG = logging.getLogger(__name__)

def build_uncomment_skip_slave_script():

    return """
        sed -e 's/^#skip_slave_start/skip_slave_start/' -i /etc/my.cnf
        """

def build_comment_skip_slave_script():

    return """
        sed -e 's/^skip_slave_start/#skip_slave_start/' -i /etc/my.cnf
        """

def build_permission_script():

    return """
        chown mysql:mysql /data
        die_if_error "Error changing datadir permission"
        chown -R mysql:mysql /data/*
        die_if_error "Error changing datadir permission"
        """

def build_server_id_conf_script():
    return """
    echo ""; echo $(date "+%Y-%m-%d %T") "- Creating the server id db file"
    \n(cat <<EOF_DBAAS
    \n[mysqld]
    \nserver_id={{SERVERID}}
    \nEOF_DBAAS
    \n) >  /etc/server_id.cnf
    """

def build_mount_snapshot_volume_script():
    return """
        mkdir /data2

        mount -t nfs -o bg,intr,nolock {{EXPORT_PATH}} /data2
        die_if_error "Error mounting volume"

        cp -rp /data2/.snapshot/{{SNAPSHOPT_NAME}}/* /data/
        die_if_error "Error coping datafiles"

        umount /data2
        rm -rf /data2

    """

def build_remove_deprecated_files_script():
    return """
        rm -f /data/data/auto.cnf
        rm -f /data/data/*.err
        rm -f /data/data/*.pid
    """

def get_client(instance):
    databaseinfra = instance.databaseinfra
    driver = databaseinfra.get_driver()

    return driver.get_client(instance)

def get_client_for_infra(databaseinfra):
    driver = databaseinfra.get_driver()

    return driver.get_client(None)

def get_replication_info(instance):
    client = get_client(instance)

    client.query('show master status')
    r = client.store_result()
    row = r.fetch_row(maxrows=0, how=1)
    return row[0]['File'], row[0]['Position']

def check_seconds_behind(instance, retries=50):
    client = get_client(instance)

    for attempt in range(retries):
        LOG.info("Checking replication %i on %s " % (attempt + 1, instance))

        client.query("show slave status")
        r = client.store_result()
        row = r.fetch_row(maxrows=0, how=1)
        seconds_behind = row[0]['Seconds_Behind_Master']

        if seconds_behind is None:
            raise Exception("Replication is not running")

        if seconds_behind == '0':
            return True

        if attempt == retries - 1:
            LOG.warning("Seconds behind: {}".format(seconds_behind))
            return False

        sleep(10)

def change_master_to(instance, master_host, bin_log_file, bin_log_position):
    client = get_client(instance)
    client.query('stop slave')

    sql_command = "change master to master_host='{}', master_log_file='{}', master_log_pos={}"
    sql_command = sql_command.format(
        master_host, bin_log_file, bin_log_position)
    client.query(sql_command)

    client.query('start slave')

def get_replication_information_from_file(host,):
    command = 'cat /data/data/mysql_binlog_master_file_pos'
    output = host.ssh.run_script(command)

    replication_file, replication_position = parse_replication_info(
        output['stdout'][0])

    return replication_file, replication_position

def parse_replication_info(replication_info):
    split_vars = replication_info.split(';')
    replication_file = split_var_and_value(split_vars[0])
    replication_position = split_vars[1]

    position_end_with_new_line = replication_position.find('\n')
    if position_end_with_new_line:
        replication_position = replication_position[
            :position_end_with_new_line]

    replication_position = split_var_and_value(replication_position)

    return replication_file, replication_position

def split_var_and_value(info):
    return info.split('=')[1]

def start_slave(instance):
    client = get_client(instance)
    client.query("start slave")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
pysteps.motion.lucaskanade
==========================

The Lucas-Kanade (LK) local feature tracking module.

This module implements the interface to the local `Lucas-Kanade`_ routine
available in OpenCV_.

For its dense method, it additionally interpolates the sparse vectors over a
regular grid to return a motion field.

.. _OpenCV: https://opencv.org/

.. _`Lucas-Kanade`:\
    https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga473e4b886d0bcc6b65831eb88ed93323

.. autosummary::
    :toctree: ../generated/

    dense_lucaskanade
    track_features
"""

import numpy as np
from numpy.ma.core import MaskedArray

from pysteps.decorators import check_input_frames
from pysteps.exceptions import MissingOptionalDependency

from pysteps import utils
from pysteps.utils.cleansing import decluster, detect_outliers
from pysteps.utils.images import morph_opening

try:
    import cv2

    CV2_IMPORTED = True
except ImportError:
    CV2_IMPORTED = False

import time
import warnings

@check_input_frames(2)
def dense_lucaskanade(input_images,
                      lk_kwargs=None,
                      fd_method="ShiTomasi",
                      fd_kwargs=None,
                      interp_method="rbfinterp2d",
                      interp_kwargs=None,
                      dense=True,
                      nr_std_outlier=3,
                      k_outlier=30,
                      size_opening=3,
                      decl_scale=10,
                      verbose=False):
    """Run the Lucas-Kanade optical flow routine and interpolate the motion
    vectors.

    .. _OpenCV: https://opencv.org/

    .. _`Lucas-Kanade`:\
        https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga473e4b886d0bcc6b65831eb88ed93323

    .. _MaskedArray:\
        https://docs.scipy.org/doc/numpy/reference/maskedarray.baseclass.html#numpy.ma.MaskedArray

    Interface to the OpenCV_ implementation of the local `Lucas-Kanade`_ optical
    flow method applied in combination to a feature detection routine.

    The sparse motion vectors are finally interpolated to return the whole
    motion field.

    Parameters
    ----------

    input_images : array_like or MaskedArray_
        Array of shape (T, m, n) containing a sequence of *T* two-dimensional
        input images of shape (m, n). The indexing order in **input_images** is
        assumed to be (time, latitude, longitude).

        *T* = 2 is the minimum required number of images.
        With *T* > 2, all the resulting sparse vectors are pooled together for
        the final interpolation on a regular grid.

        In case of array_like, invalid values (Nans or infs) are masked,
        otherwise the mask of the MaskedArray_ is used. Such mask defines a
        region where features are not detected for the tracking algorithm.

    lk_kwargs : dict, optional
        Optional dictionary containing keyword arguments for the `Lucas-Kanade`_
        features tracking algorithm. See the documentation of
        :py:func:`pysteps.motion.lucaskanade.track_features`.

    fd_method : {"ShiTomasi"}, optional
      Name of the feature detection routine. See feature detection methods in
      :py:mod:`pysteps.utils.images`.

    fd_kwargs : dict, optional
        Optional dictionary containing keyword arguments for the features
        detection algorithm.
        See the documentation of :py:mod:`pysteps.utils.images`.

    interp_method : {"rbfinterp2d"}, optional
      Name of the interpolation method to use. See interpolation methods in
      :py:mod:`pysteps.utils.interpolate`.

    interp_kwargs : dict, optional
        Optional dictionary containing keyword arguments for the interpolation
        algorithm. See the documentation of :py:mod:`pysteps.utils.interpolate`.

    dense : bool, optional
        If True, return the three-dimensional array (2, m, n) containing
        the dense x- and y-components of the motion field.

        If False, return the sparse motion vectors as 2-D **xy** and **uv**
        arrays, where **xy** defines the vector positions, **uv** defines the
        x and y direction components of the vectors.

    nr_std_outlier : int, optional
        Maximum acceptable deviation from the mean in terms of number of
        standard deviations. Any sparse vector with a deviation larger than
        this threshold is flagged as outlier and excluded from the
        interpolation.
        See the documentation of
        :py:func:`pysteps.utils.cleansing.detect_outliers`.

    k_outlier : int or None, optional
        The number of nearest neighbours used to localize the outlier detection.
        If set to None, it employs all the data points (global detection).
        See the documentation of
        :py:func:`pysteps.utils.cleansing.detect_outliers`.

    size_opening : int, optional
        The size of the structuring element kernel in pixels. This is used to
        perform a binary morphological opening on the input fields in order to
        filter isolated echoes due to clutter. If set to zero, the filtering
        is not perfomed.
        See the documentation of
        :py:func:`pysteps.utils.images.morph_opening`.

    decl_scale : int, optional
        The scale declustering parameter in pixels used to reduce the number of
        redundant sparse vectors before the interpolation.
        Sparse vectors within this declustering scale are averaged together.
        If set to less than 2 pixels, the declustering is not perfomed.
        See the documentation of
        :py:func:`pysteps.utils.cleansing.decluster`.

    verbose : bool, optional
        If set to True, print some information about the program.

    Returns
    -------

    out : array_like or tuple
        If **dense=True** (the default), return the advection field having shape
        (2, m, n), where out[0, :, :] contains the x-components of the motion
        vectors and out[1, :, :] contains the y-components.
        The velocities are in units of pixels / timestep, where timestep is the
        time difference between the two input images.
        Return a zero motion field of shape (2, m, n) when no motion is
        detected.

        If **dense=False**, it returns a tuple containing the 2-dimensional
        arrays **xy** and **uv**, where x, y define the vector locations,
        u, v define the x and y direction components of the vectors.
        Return two empty arrays when no motion is detected.

    See also
    --------

    pysteps.motion.lucaskanade.track_features

    References
    ----------

    Bouguet,  J.-Y.:  Pyramidal  implementation  of  the  affine  Lucas Kanade
    feature tracker description of the algorithm, Intel Corp., 5, 4,
    https://doi.org/10.1109/HPDC.2004.1323531, 2001

    Lucas, B. D. and Kanade, T.: An iterative image registration technique with
    an application to stereo vision, in: Proceedings of the 1981 DARPA Imaging
    Understanding Workshop, pp. 121–130, 1981.
    """

    input_images = input_images.copy()

    if verbose:
        print("Computing the motion field with the Lucas-Kanade method.")
        t0 = time.time()

    nr_fields = input_images.shape[0]
    domain_size = (input_images.shape[1], input_images.shape[2])

    feature_detection_method = utils.get_method(fd_method)
    interpolation_method = utils.get_method(interp_method)

    if fd_kwargs is None:
        fd_kwargs = dict()

    if lk_kwargs is None:
        lk_kwargs = dict()

    if interp_kwargs is None:
        interp_kwargs = dict()

    xy = np.empty(shape=(0, 2))
    uv = np.empty(shape=(0, 2))
    for n in range(nr_fields - 1):

        # extract consecutive images
        prvs_img = input_images[n, :, :].copy()
        next_img = input_images[n + 1, :, :].copy()

        if ~isinstance(prvs_img, MaskedArray):
            prvs_img = np.ma.masked_invalid(prvs_img)
        np.ma.set_fill_value(prvs_img, prvs_img.min())

        if ~isinstance(next_img, MaskedArray):
            next_img = np.ma.masked_invalid(next_img)
        np.ma.set_fill_value(next_img, next_img.min())

        # remove small noise with a morphological operator (opening)
        if size_opening > 0:
            prvs_img = morph_opening(prvs_img, prvs_img.min(), size_opening)
            next_img = morph_opening(next_img, next_img.min(), size_opening)

        # features detection
        points = feature_detection_method(prvs_img, **fd_kwargs)

        # skip loop if no features to track
        if points.shape[0] == 0:
            continue

        # get sparse u, v vectors with Lucas-Kanade tracking
        xy_, uv_ = track_features(prvs_img, next_img, points, **lk_kwargs)

        # skip loop if no vectors
        if xy_.shape[0] == 0:
            continue

        # stack vectors
        xy = np.append(xy, xy_, axis=0)
        uv = np.append(uv, uv_, axis=0)

    # return zero motion field is no sparse vectors are found
    if xy.shape[0] == 0:
        if dense:
            return np.zeros((2, domain_size[0], domain_size[1]))
        else:
            return xy, uv

    # detect and remove outliers
    outliers = detect_outliers(uv, nr_std_outlier, xy, k_outlier, verbose)
    xy = xy[~outliers, :]
    uv = uv[~outliers, :]

    if verbose:
        print("--- LK found %i sparse vectors ---" % xy.shape[0])

    # return sparse vectors if required
    if not dense:
        return xy, uv

    # decluster sparse motion vectors
    if decl_scale > 1:
        xy, uv = decluster(xy, uv, decl_scale, 1, verbose)

    # return zero motion field if no sparse vectors are left for interpolation
    if xy.shape[0] == 0:
        return np.zeros((2, domain_size[0], domain_size[1]))

    # interpolation
    xgrid = np.arange(domain_size[1])
    ygrid = np.arange(domain_size[0])
    UV = interpolation_method(xy, uv, xgrid, ygrid, **interp_kwargs)

    if verbose:
        print("--- total time: %.2f seconds ---" % (time.time() - t0))

    return UV

def track_features(
        prvs_image,
        next_image,
        points,
        winsize=(50, 50),
        nr_levels=3,
        criteria=(3, 10, 0),
        flags=0,
        min_eig_thr=1e-4,
        verbose=False,
):
    """
    Interface to the OpenCV `Lucas-Kanade`_ features tracking algorithm
    (cv.calcOpticalFlowPyrLK).

    .. _`Lucas-Kanade`:\
       https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga473e4b886d0bcc6b65831eb88ed93323

    .. _calcOpticalFlowPyrLK:\
       https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga473e4b886d0bcc6b65831eb88ed93323

    .. _MaskedArray:\
        https://docs.scipy.org/doc/numpy/reference/maskedarray.baseclass.html#numpy.ma.MaskedArray

    Parameters
    ----------

    prvs_image : array_like or MaskedArray_
        Array of shape (m, n) containing the first image.
        Invalid values (Nans or infs) are filled using the min value.

    next_image : array_like or MaskedArray_
        Array of shape (m, n) containing the successive image.
        Invalid values (Nans or infs) are filled using the min value.

    points : array_like
        Array of shape (p, 2) indicating the pixel coordinates of the
        tracking points (corners).

    winsize : tuple of int, optional
        The **winSize** parameter in calcOpticalFlowPyrLK_.
        It represents the size of the search window that it is used at each
        pyramid level.

    nr_levels : int, optional
        The **maxLevel** parameter in calcOpticalFlowPyrLK_.
        It represents the 0-based maximal pyramid level number.

    criteria : tuple of int, optional
        The **TermCriteria** parameter in calcOpticalFlowPyrLK_ ,
        which specifies the termination criteria of the iterative search
        algorithm.

    flags : int, optional
        Operation flags, see documentation calcOpticalFlowPyrLK_.

    min_eig_thr : float, optional
        The **minEigThreshold** parameter in calcOpticalFlowPyrLK_.

    verbose : bool, optional
        Print the number of vectors that have been found.

    Returns
    -------

    xy : array_like
        Array of shape (d, 2) with the x- and y-coordinates of *d* <= *p*
        detected sparse motion vectors.

    uv : array_like
        Array of shape (d, 2) with the u- and v-components of *d* <= *p*
        detected sparse motion vectors.

    Notes
    -----

    The tracking points can be obtained with the
    :py:func:`pysteps.utils.images.ShiTomasi_detection` routine.

    See also
    --------

    pysteps.motion.lucaskanade.dense_lucaskanade

    References
    ----------

    Bouguet,  J.-Y.:  Pyramidal  implementation  of  the  affine  Lucas Kanade
    feature tracker description of the algorithm, Intel Corp., 5, 4,
    https://doi.org/10.1109/HPDC.2004.1323531, 2001

    Lucas, B. D. and Kanade, T.: An iterative image registration technique with
    an application to stereo vision, in: Proceedings of the 1981 DARPA Imaging
    Understanding Workshop, pp. 121–130, 1981.
    """

    if not CV2_IMPORTED:
        raise MissingOptionalDependency(
            "opencv package is required for the calcOpticalFlowPyrLK() "
            "routine but it is not installed"
        )

    prvs_img = np.copy(prvs_image)
    next_img = np.copy(next_image)
    p0 = np.copy(points)

    if ~isinstance(prvs_img, MaskedArray):
        prvs_img = np.ma.masked_invalid(prvs_img)
    np.ma.set_fill_value(prvs_img, prvs_img.min())

    if ~isinstance(next_img, MaskedArray):
        next_img = np.ma.masked_invalid(next_img)
    np.ma.set_fill_value(next_img, next_img.min())

    # scale between 0 and 255
    prvs_img = ((prvs_img.filled() - prvs_img.min()) /
                (prvs_img.max() - prvs_img.min()) * 255)

    next_img = ((next_img.filled() - next_img.min()) /
                (next_img.max() - next_img.min()) * 255)

    # convert to 8-bit
    prvs_img = np.ndarray.astype(prvs_img, "uint8")
    next_img = np.ndarray.astype(next_img, "uint8")

    # Lucas-Kanade
    # TODO: use the error returned by the OpenCV routine
    params = dict(
        winSize=winsize,
        maxLevel=nr_levels,
        criteria=criteria,
        flags=flags,
        minEigThreshold=min_eig_thr,
    )
    p1, st, __ = cv2.calcOpticalFlowPyrLK(prvs_img, next_img,
                                          p0, None, **params)

    # keep only features that have been found
    st = st.squeeze() == 1
    if np.any(st):
        p1 = p1[st, :]
        p0 = p0[st, :]

        # extract vectors
        xy = p0
        uv = p1 - p0

    else:
        xy = uv = np.empty(shape=(0, 2))

    if verbose:
        print("--- %i sparse vectors found ---" % xy.shape[0])

    return xy, uv
<commit_msg>Avoid zero division<commit_after># -*- coding: utf-8 -*-
"""
pysteps.motion.lucaskanade
==========================

The Lucas-Kanade (LK) local feature tracking module.

This module implements the interface to the local `Lucas-Kanade`_ routine
available in OpenCV_.

For its dense method, it additionally interpolates the sparse vectors over a
regular grid to return a motion field.

.. _OpenCV: https://opencv.org/

.. _`Lucas-Kanade`:\
    https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga473e4b886d0bcc6b65831eb88ed93323

.. autosummary::
    :toctree: ../generated/

    dense_lucaskanade
    track_features
"""

import numpy as np
from numpy.ma.core import MaskedArray

from pysteps.decorators import check_input_frames
from pysteps.exceptions import MissingOptionalDependency

from pysteps import utils
from pysteps.utils.cleansing import decluster, detect_outliers
from pysteps.utils.images import morph_opening

try:
    import cv2

    CV2_IMPORTED = True
except ImportError:
    CV2_IMPORTED = False

import time
import warnings

@check_input_frames(2)
def dense_lucaskanade(input_images,
                      lk_kwargs=None,
                      fd_method="ShiTomasi",
                      fd_kwargs=None,
                      interp_method="rbfinterp2d",
                      interp_kwargs=None,
                      dense=True,
                      nr_std_outlier=3,
                      k_outlier=30,
                      size_opening=3,
                      decl_scale=10,
                      verbose=False):
    """Run the Lucas-Kanade optical flow routine and interpolate the motion
    vectors.

    .. _OpenCV: https://opencv.org/

    .. _`Lucas-Kanade`:\
        https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga473e4b886d0bcc6b65831eb88ed93323

    .. _MaskedArray:\
        https://docs.scipy.org/doc/numpy/reference/maskedarray.baseclass.html#numpy.ma.MaskedArray

    Interface to the OpenCV_ implementation of the local `Lucas-Kanade`_ optical
    flow method applied in combination to a feature detection routine.

    The sparse motion vectors are finally interpolated to return the whole
    motion field.

    Parameters
    ----------

    input_images : array_like or MaskedArray_
        Array of shape (T, m, n) containing a sequence of *T* two-dimensional
        input images of shape (m, n). The indexing order in **input_images** is
        assumed to be (time, latitude, longitude).

        *T* = 2 is the minimum required number of images.
        With *T* > 2, all the resulting sparse vectors are pooled together for
        the final interpolation on a regular grid.

        In case of array_like, invalid values (Nans or infs) are masked,
        otherwise the mask of the MaskedArray_ is used. Such mask defines a
        region where features are not detected for the tracking algorithm.

    lk_kwargs : dict, optional
        Optional dictionary containing keyword arguments for the `Lucas-Kanade`_
        features tracking algorithm. See the documentation of
        :py:func:`pysteps.motion.lucaskanade.track_features`.

    fd_method : {"ShiTomasi"}, optional
      Name of the feature detection routine. See feature detection methods in
      :py:mod:`pysteps.utils.images`.

    fd_kwargs : dict, optional
        Optional dictionary containing keyword arguments for the features
        detection algorithm.
        See the documentation of :py:mod:`pysteps.utils.images`.

    interp_method : {"rbfinterp2d"}, optional
      Name of the interpolation method to use. See interpolation methods in
      :py:mod:`pysteps.utils.interpolate`.

    interp_kwargs : dict, optional
        Optional dictionary containing keyword arguments for the interpolation
        algorithm. See the documentation of :py:mod:`pysteps.utils.interpolate`.

    dense : bool, optional
        If True, return the three-dimensional array (2, m, n) containing
        the dense x- and y-components of the motion field.

        If False, return the sparse motion vectors as 2-D **xy** and **uv**
        arrays, where **xy** defines the vector positions, **uv** defines the
        x and y direction components of the vectors.

    nr_std_outlier : int, optional
        Maximum acceptable deviation from the mean in terms of number of
        standard deviations. Any sparse vector with a deviation larger than
        this threshold is flagged as outlier and excluded from the
        interpolation.
        See the documentation of
        :py:func:`pysteps.utils.cleansing.detect_outliers`.

    k_outlier : int or None, optional
        The number of nearest neighbours used to localize the outlier detection.
        If set to None, it employs all the data points (global detection).
        See the documentation of
        :py:func:`pysteps.utils.cleansing.detect_outliers`.

    size_opening : int, optional
        The size of the structuring element kernel in pixels. This is used to
        perform a binary morphological opening on the input fields in order to
        filter isolated echoes due to clutter. If set to zero, the filtering
        is not perfomed.
        See the documentation of
        :py:func:`pysteps.utils.images.morph_opening`.

    decl_scale : int, optional
        The scale declustering parameter in pixels used to reduce the number of
        redundant sparse vectors before the interpolation.
        Sparse vectors within this declustering scale are averaged together.
        If set to less than 2 pixels, the declustering is not perfomed.
        See the documentation of
        :py:func:`pysteps.utils.cleansing.decluster`.

    verbose : bool, optional
        If set to True, print some information about the program.

    Returns
    -------

    out : array_like or tuple
        If **dense=True** (the default), return the advection field having shape
        (2, m, n), where out[0, :, :] contains the x-components of the motion
        vectors and out[1, :, :] contains the y-components.
        The velocities are in units of pixels / timestep, where timestep is the
        time difference between the two input images.
        Return a zero motion field of shape (2, m, n) when no motion is
        detected.

        If **dense=False**, it returns a tuple containing the 2-dimensional
        arrays **xy** and **uv**, where x, y define the vector locations,
        u, v define the x and y direction components of the vectors.
        Return two empty arrays when no motion is detected.

    See also
    --------

    pysteps.motion.lucaskanade.track_features

    References
    ----------

    Bouguet,  J.-Y.:  Pyramidal  implementation  of  the  affine  Lucas Kanade
    feature tracker description of the algorithm, Intel Corp., 5, 4,
    https://doi.org/10.1109/HPDC.2004.1323531, 2001

    Lucas, B. D. and Kanade, T.: An iterative image registration technique with
    an application to stereo vision, in: Proceedings of the 1981 DARPA Imaging
    Understanding Workshop, pp. 121–130, 1981.
    """

    input_images = input_images.copy()

    if verbose:
        print("Computing the motion field with the Lucas-Kanade method.")
        t0 = time.time()

    nr_fields = input_images.shape[0]
    domain_size = (input_images.shape[1], input_images.shape[2])

    feature_detection_method = utils.get_method(fd_method)
    interpolation_method = utils.get_method(interp_method)

    if fd_kwargs is None:
        fd_kwargs = dict()

    if lk_kwargs is None:
        lk_kwargs = dict()

    if interp_kwargs is None:
        interp_kwargs = dict()

    xy = np.empty(shape=(0, 2))
    uv = np.empty(shape=(0, 2))
    for n in range(nr_fields - 1):

        # extract consecutive images
        prvs_img = input_images[n, :, :].copy()
        next_img = input_images[n + 1, :, :].copy()

        if ~isinstance(prvs_img, MaskedArray):
            prvs_img = np.ma.masked_invalid(prvs_img)
        np.ma.set_fill_value(prvs_img, prvs_img.min())

        if ~isinstance(next_img, MaskedArray):
            next_img = np.ma.masked_invalid(next_img)
        np.ma.set_fill_value(next_img, next_img.min())

        # remove small noise with a morphological operator (opening)
        if size_opening > 0:
            prvs_img = morph_opening(prvs_img, prvs_img.min(), size_opening)
            next_img = morph_opening(next_img, next_img.min(), size_opening)

        # features detection
        points = feature_detection_method(prvs_img, **fd_kwargs)

        # skip loop if no features to track
        if points.shape[0] == 0:
            continue

        # get sparse u, v vectors with Lucas-Kanade tracking
        xy_, uv_ = track_features(prvs_img, next_img, points, **lk_kwargs)

        # skip loop if no vectors
        if xy_.shape[0] == 0:
            continue

        # stack vectors
        xy = np.append(xy, xy_, axis=0)
        uv = np.append(uv, uv_, axis=0)

    # return zero motion field is no sparse vectors are found
    if xy.shape[0] == 0:
        if dense:
            return np.zeros((2, domain_size[0], domain_size[1]))
        else:
            return xy, uv

    # detect and remove outliers
    outliers = detect_outliers(uv, nr_std_outlier, xy, k_outlier, verbose)
    xy = xy[~outliers, :]
    uv = uv[~outliers, :]

    if verbose:
        print("--- LK found %i sparse vectors ---" % xy.shape[0])

    # return sparse vectors if required
    if not dense:
        return xy, uv

    # decluster sparse motion vectors
    if decl_scale > 1:
        xy, uv = decluster(xy, uv, decl_scale, 1, verbose)

    # return zero motion field if no sparse vectors are left for interpolation
    if xy.shape[0] == 0:
        return np.zeros((2, domain_size[0], domain_size[1]))

    # interpolation
    xgrid = np.arange(domain_size[1])
    ygrid = np.arange(domain_size[0])
    UV = interpolation_method(xy, uv, xgrid, ygrid, **interp_kwargs)

    if verbose:
        print("--- total time: %.2f seconds ---" % (time.time() - t0))

    return UV

def track_features(
        prvs_image,
        next_image,
        points,
        winsize=(50, 50),
        nr_levels=3,
        criteria=(3, 10, 0),
        flags=0,
        min_eig_thr=1e-4,
        verbose=False,
):
    """
    Interface to the OpenCV `Lucas-Kanade`_ features tracking algorithm
    (cv.calcOpticalFlowPyrLK).

    .. _`Lucas-Kanade`:\
       https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga473e4b886d0bcc6b65831eb88ed93323

    .. _calcOpticalFlowPyrLK:\
       https://docs.opencv.org/3.4/dc/d6b/group__video__track.html#ga473e4b886d0bcc6b65831eb88ed93323

    .. _MaskedArray:\
        https://docs.scipy.org/doc/numpy/reference/maskedarray.baseclass.html#numpy.ma.MaskedArray

    Parameters
    ----------

    prvs_image : array_like or MaskedArray_
        Array of shape (m, n) containing the first image.
        Invalid values (Nans or infs) are filled using the min value.

    next_image : array_like or MaskedArray_
        Array of shape (m, n) containing the successive image.
        Invalid values (Nans or infs) are filled using the min value.

    points : array_like
        Array of shape (p, 2) indicating the pixel coordinates of the
        tracking points (corners).

    winsize : tuple of int, optional
        The **winSize** parameter in calcOpticalFlowPyrLK_.
        It represents the size of the search window that it is used at each
        pyramid level.

    nr_levels : int, optional
        The **maxLevel** parameter in calcOpticalFlowPyrLK_.
        It represents the 0-based maximal pyramid level number.

    criteria : tuple of int, optional
        The **TermCriteria** parameter in calcOpticalFlowPyrLK_ ,
        which specifies the termination criteria of the iterative search
        algorithm.

    flags : int, optional
        Operation flags, see documentation calcOpticalFlowPyrLK_.

    min_eig_thr : float, optional
        The **minEigThreshold** parameter in calcOpticalFlowPyrLK_.

    verbose : bool, optional
        Print the number of vectors that have been found.

    Returns
    -------

    xy : array_like
        Array of shape (d, 2) with the x- and y-coordinates of *d* <= *p*
        detected sparse motion vectors.

    uv : array_like
        Array of shape (d, 2) with the u- and v-components of *d* <= *p*
        detected sparse motion vectors.

    Notes
    -----

    The tracking points can be obtained with the
    :py:func:`pysteps.utils.images.ShiTomasi_detection` routine.

    See also
    --------

    pysteps.motion.lucaskanade.dense_lucaskanade

    References
    ----------

    Bouguet,  J.-Y.:  Pyramidal  implementation  of  the  affine  Lucas Kanade
    feature tracker description of the algorithm, Intel Corp., 5, 4,
    https://doi.org/10.1109/HPDC.2004.1323531, 2001

    Lucas, B. D. and Kanade, T.: An iterative image registration technique with
    an application to stereo vision, in: Proceedings of the 1981 DARPA Imaging
    Understanding Workshop, pp. 121–130, 1981.
    """

    if not CV2_IMPORTED:
        raise MissingOptionalDependency(
            "opencv package is required for the calcOpticalFlowPyrLK() "
            "routine but it is not installed"
        )

    prvs_img = np.copy(prvs_image)
    next_img = np.copy(next_image)
    p0 = np.copy(points)

    if ~isinstance(prvs_img, MaskedArray):
        prvs_img = np.ma.masked_invalid(prvs_img)
    np.ma.set_fill_value(prvs_img, prvs_img.min())

    if ~isinstance(next_img, MaskedArray):
        next_img = np.ma.masked_invalid(next_img)
    np.ma.set_fill_value(next_img, next_img.min())

    # scale between 0 and 255
    im_min = prvs_img.min()
    im_max = prvs_img.max()
    if im_max - im_min > 1e-8:
        prvs_img = ((prvs_img.filled() - im_min) /
                    (im_max - im_min) * 255)
    else:
        prvs_img = ((prvs_img.filled() - im_min)

    im_min = next_img.min()
    im_max = next_img.max()
    if im_max - im_min > 1e-8:
        next_img = ((next_img.filled() - im_min) /
                    (im_max.max() - im_min) * 255)
    else:
        next_img = ((next_img.filled() - im_min)

    # convert to 8-bit
    prvs_img = np.ndarray.astype(prvs_img, "uint8")
    next_img = np.ndarray.astype(next_img, "uint8")

    # Lucas-Kanade
    # TODO: use the error returned by the OpenCV routine
    params = dict(
        winSize=winsize,
        maxLevel=nr_levels,
        criteria=criteria,
        flags=flags,
        minEigThreshold=min_eig_thr,
    )
    p1, st, __ = cv2.calcOpticalFlowPyrLK(prvs_img, next_img,
                                          p0, None, **params)

    # keep only features that have been found
    st = st.squeeze() == 1
    if np.any(st):
        p1 = p1[st, :]
        p0 = p0[st, :]

        # extract vectors
        xy = p0
        uv = p1 - p0

    else:
        xy = uv = np.empty(shape=(0, 2))

    if verbose:
        print("--- %i sparse vectors found ---" % xy.shape[0])

    return xy, uv
<|endoftext|>


<commit_before>"""
Test dbcollection core API: process.
"""

import pytest

from dbcollection.core.api.process import process, ProcessAPI

@pytest.fixture()
def mocks_init_class(mocker):
    mock_cache = mocker.patch.object(ProcessAPI, "get_cache_manager", return_value=True)
    mock_data_dir = mocker.patch.object(ProcessAPI, "get_dataset_data_dir_path", return_value='/some/path/cache/db')
    mock_cache_dir = mocker.patch.object(ProcessAPI, "get_dataset_cache_dir_path", return_value='/some/path/cache')
    mock_exists = mocker.patch.object(ProcessAPI, 'check_if_task_exists_in_database')
    return [mock_cache, mock_data_dir, mock_cache_dir, mock_exists]

def assert_mock_init_class(mocks):
    for mock in mocks:
        assert mock.called

class TestCallProcess:
    """Unit tests for the core api process method."""

    def test_call_with_dataset_name(self, mocker, mocks_init_class):
        mock_run = mocker.patch.object(ProcessAPI, "run")
        dataset = 'mnist'

        process(dataset)

        assert_mock_init_class(mocks_init_class)
        assert mock_run.called

    def test_call_with_all_inputs(self, mocker, mocks_init_class):
        mock_run = mocker.patch.object(ProcessAPI, "run")
        dataset = 'mnist'
        task = ''
        verbose = True

        process(dataset, task=task, verbose=verbose)

        assert_mock_init_class(mocks_init_class)
        assert mock_run.called

    def test_call__raises_error_no_inputs(self, mocker):
        with pytest.raises(TypeError):
            process()

    def test_call__raises_error_invalid_dataset_name(self, mocker):
        with pytest.raises(KeyError):
            process('invalid_dataset_name')

class TestClassDownloadAPI:
    """Unit tests for the DownloadAPI class."""

    def test_init_with_all_inputs(self, mocker, mocks_init_class):
        dataset = 'mnist'
        task = 'default'
        verbose = True

        process_api = ProcessAPI(dataset, task, verbose)

        assert_mock_init_class(mocks_init_class)
        assert process_api.name == dataset
        assert process_api.task == 'classification'  # default task name for mnist
        assert process_api.verbose == verbose

    def test_init__raises_error_missing_inputs(self, mocker):
        with pytest.raises(TypeError):
            ProcessAPI()

    @pytest.mark.parametrize("dataset, task, verbose", [
        ('mnist', 'classification', None),
        ('mnist', None, True),
        (None, '', False),
        ('', 'default', False),
        (None, None, None)
    ])
    def test_init__raises_error_missing_some_inputs(self, mocker, dataset, task, verbose):
        with pytest.raises(AssertionError):
            ProcessAPI(dataset, task, verbose)

    @pytest.mark.parametrize("test_task", ['classification', 'some_task', 'another_task'])
    def test_parse_task_name(self, mocker, mocks_init_class, test_task):
        process_api = ProcessAPI('mnist', test_task, True)

        assert_mock_init_class(mocks_init_class)
        assert process_api.parse_task_name(test_task) == test_task

    @pytest.mark.parametrize("test_task", ['', 'default'])
    def test_parse_task_name_to_default(self, mocker, mocks_init_class, test_task):
        process_api = ProcessAPI('mnist', '', True)

        assert_mock_init_class(mocks_init_class)
        assert process_api.parse_task_name(test_task) == "classification"

    def test_run(self, mocker, mocks_init_class):
        mock_create = mocker.patch.object(ProcessAPI, "create_dir")
        mock_process = mocker.patch.object(ProcessAPI, "process_dataset", return_value="some_info")
        mock_update = mocker.patch.object(ProcessAPI, "update_cache")
        dataset = 'mnist'
        task = 'some_task'
        verbose = True

        process_api = ProcessAPI(dataset, task, verbose)
        process_api.run()

        assert_mock_init_class(mocks_init_class)
        assert mock_create.called
        assert mock_process.return_value == "some_info"
        assert mock_update.called

    def test_process_dataset(self, mocker, mocks_init_class):
        mock_constructor = mocker.patch.object(ProcessAPI, "get_dataset_constructor", return_value=mocker.MagicMock())
        dataset = 'mnist'
        task = 'some_task'
        verbose = True

        process_api = ProcessAPI(dataset, task, verbose)
        task_info = process_api.process_dataset()

        assert_mock_init_class(mocks_init_class)
        assert mock_constructor.called
        assert task_info is not None
<commit_msg>Fix test name<commit_after>"""
Test dbcollection core API: process.
"""

import pytest

from dbcollection.core.api.process import process, ProcessAPI

@pytest.fixture()
def mocks_init_class(mocker):
    mock_cache = mocker.patch.object(ProcessAPI, "get_cache_manager", return_value=True)
    mock_data_dir = mocker.patch.object(ProcessAPI, "get_dataset_data_dir_path", return_value='/some/path/cache/db')
    mock_cache_dir = mocker.patch.object(ProcessAPI, "get_dataset_cache_dir_path", return_value='/some/path/cache')
    mock_exists = mocker.patch.object(ProcessAPI, 'check_if_task_exists_in_database')
    return [mock_cache, mock_data_dir, mock_cache_dir, mock_exists]

def assert_mock_init_class(mocks):
    for mock in mocks:
        assert mock.called

class TestCallProcess:
    """Unit tests for the core api process method."""

    def test_call_with_dataset_name(self, mocker, mocks_init_class):
        mock_run = mocker.patch.object(ProcessAPI, "run")
        dataset = 'mnist'

        process(dataset)

        assert_mock_init_class(mocks_init_class)
        assert mock_run.called

    def test_call_with_all_inputs(self, mocker, mocks_init_class):
        mock_run = mocker.patch.object(ProcessAPI, "run")
        dataset = 'mnist'
        task = ''
        verbose = True

        process(dataset, task=task, verbose=verbose)

        assert_mock_init_class(mocks_init_class)
        assert mock_run.called

    def test_call__raises_error_no_inputs(self, mocker):
        with pytest.raises(TypeError):
            process()

    def test_call__raises_error_invalid_dataset_name(self, mocker):
        with pytest.raises(KeyError):
            process('invalid_dataset_name')

class TestClassProcessAPI:
    """Unit tests for the ProcessAPI class."""

    def test_init_with_all_inputs(self, mocker, mocks_init_class):
        dataset = 'mnist'
        task = 'default'
        verbose = True

        process_api = ProcessAPI(dataset, task, verbose)

        assert_mock_init_class(mocks_init_class)
        assert process_api.name == dataset
        assert process_api.task == 'classification'  # default task name for mnist
        assert process_api.verbose == verbose

    def test_init__raises_error_missing_inputs(self, mocker):
        with pytest.raises(TypeError):
            ProcessAPI()

    @pytest.mark.parametrize("dataset, task, verbose", [
        ('mnist', 'classification', None),
        ('mnist', None, True),
        (None, '', False),
        ('', 'default', False),
        (None, None, None)
    ])
    def test_init__raises_error_missing_some_inputs(self, mocker, dataset, task, verbose):
        with pytest.raises(AssertionError):
            ProcessAPI(dataset, task, verbose)

    @pytest.mark.parametrize("test_task", ['classification', 'some_task', 'another_task'])
    def test_parse_task_name(self, mocker, mocks_init_class, test_task):
        process_api = ProcessAPI('mnist', test_task, True)

        assert_mock_init_class(mocks_init_class)
        assert process_api.parse_task_name(test_task) == test_task

    @pytest.mark.parametrize("test_task", ['', 'default'])
    def test_parse_task_name_to_default(self, mocker, mocks_init_class, test_task):
        process_api = ProcessAPI('mnist', '', True)

        assert_mock_init_class(mocks_init_class)
        assert process_api.parse_task_name(test_task) == "classification"

    def test_run(self, mocker, mocks_init_class):
        mock_create = mocker.patch.object(ProcessAPI, "create_dir")
        mock_process = mocker.patch.object(ProcessAPI, "process_dataset", return_value="some_info")
        mock_update = mocker.patch.object(ProcessAPI, "update_cache")
        dataset = 'mnist'
        task = 'some_task'
        verbose = True

        process_api = ProcessAPI(dataset, task, verbose)
        process_api.run()

        assert_mock_init_class(mocks_init_class)
        assert mock_create.called
        assert mock_process.return_value == "some_info"
        assert mock_update.called

    def test_process_dataset(self, mocker, mocks_init_class):
        mock_constructor = mocker.patch.object(ProcessAPI, "get_dataset_constructor", return_value=mocker.MagicMock())
        dataset = 'mnist'
        task = 'some_task'
        verbose = True

        process_api = ProcessAPI(dataset, task, verbose)
        task_info = process_api.process_dataset()

        assert_mock_init_class(mocks_init_class)
        assert mock_constructor.called
        assert task_info is not None
<|endoftext|>


<commit_before>"""
Tests for pika.adapters.tornado_connection

"""
from tornado import ioloop
import mock
try:
    import unittest2 as unittest
except ImportError:
    import unittest

from pika.adapters import tornado_connection

class TornadoConnectionTests(unittest.TestCase):

    def test_tornado_connection_ioloop_override(self):
        ioloop_value = mock.Mock()
        obj = tornado_connection.TornadoConnection(custom_ioloop=ioloop_value)
        self.assertEqual(obj.ioloop, ioloop_value)

    def test_tornado_connection_ioloop_singleton(self):
        ioloop_instance = ioloop.IOLoop.instance()
        obj = tornado_connection.TornadoConnection()
        self.assertEqual(obj.ioloop, ioloop_instance)

    @mock.patch('pika.adapters.base_connection.BaseConnection.__init__')
    def test_tornado_connection_call_parent(self, mock_init):
        obj = tornado_connection.TornadoConnection()
        mock_init.called_once_with(None, None, False)

    @mock.patch('pika.adapters.base_connection.BaseConnection._adapter_disconnect')
    def test_tornado_adapter_disconnect_call_parent(self, adapter_disconnect):
        obj = tornado_connection.TornadoConnection()
        obj._adapter_disconnect()
        adapter_disconnect.assert_called_once_with()

    @mock.patch('time.time')
    def test_add_timeout(self, time):
        time.return_value = 20
        ioloop_value = mock.Mock()
        callback = mock.Mock()
        obj = tornado_connection.TornadoConnection(custom_ioloop=ioloop_value)
        obj.add_timeout(10, callback)
        ioloop_value.add_timeout.called_once_with(30, callback)

    def test_remove_timeout(self):
        ioloop_value = mock.Mock()
        callback = mock.Mock()
        obj = tornado_connection.TornadoConnection(custom_ioloop=ioloop_value)
        timeout_id = obj.add_timeout(10, callback)
        obj.remove_timeout(timeout_id)
        ioloop_value.remove_timeout.called_once_with(timeout_id)
<commit_msg>Remove broken tests<commit_after>"""
Tests for pika.adapters.tornado_connection

"""
from tornado import ioloop
import mock
try:
    import unittest2 as unittest
except ImportError:
    import unittest

from pika.adapters import tornado_connection

class TornadoConnectionTests(unittest.TestCase):

    @mock.patch('pika.adapters.base_connection.BaseConnection.__init__')
    def test_tornado_connection_call_parent(self, mock_init):
        obj = tornado_connection.TornadoConnection()
        mock_init.called_once_with(None, None, False)
<|endoftext|>


<commit_before>def pull(_git, branch, origin):
    out = f'{origin}/' if origin else ''
    return f'git pull {out}{branch}'

<commit_msg>remove redundant file<commit_after><|endoftext|>