<commit_before>import os
import random
from django.test import TestCase
from avocado.stats import cluster, kmeans
from scipy.cluster import vq
import numpy as np
from itertools import chain

__all__ = ('KmeansTestCase',)

random_points_file = open(os.path.join(os.path.dirname(__file__), '../fixtures/random_points.txt'))
random_points_3d_file = open(os.path.join(os.path.dirname(__file__), '../fixtures/random_points_3d.txt'))
random_points = [float(x.strip()) for x in random_points_file.xreadlines()]
random_points_3d = [[float(x) for x in l.strip().split(",")] for l in random_points_3d_file.xreadlines()]
PLACES = 10

class KmeansTestCase(TestCase):
    maxDiff = None

    def assertSequenceAlmostEqual(self, seq1, seq2):
        """
        Helper method for checking that 2 sequences are almost equal.

        Two sequences are considered almost equal if they have the same order
        and each pair of elements of the sequences passes the
        assertAlmostEqual to the number of decimal places specified in PLACES.
        This method will also work for nested lists of numbers. For example,
        let's say we wanted to see if two collections of 3D points were almost
        equal, we could use the following:

            pts1 = [[0.206331960142751, 0.612540082088810, 0.843236918599283],
                    [0.269705965446683, 0.218132746363829, 0.277332011689122],
                    [0.728684538148946, 0.734953792412333, 0.722476119561547]]
            pts2 = [[0.206331960142700, 0.612540082088819, 0.843236918599288],
                    [0.269705965446609, 0.218132746363899, 0.277332011689182],
                    [0.728684538148046, 0.734953792412933, 0.722476119561847]]
            assertSequenceAlmostEqual(pts1, pts2)

        Arguments:
            seq1, seq2: (nested)list of numbers to check for near equality

        NOTE: This method assumes 'seq1' 'seq2' have equal, non-zero length. If
        you are not certain the lengths match, use the following assert before
        calling this method or this method might have unpredictable results.
        For nested lists, it is not only assumed that overall list length is
        the same, but also that each nested list pair is of equal length.

            assertEqual(len(seq1), len(seq2))
        """
        if isinstance(seq1[0], list):
            for list1, list2 in zip(seq1, seq2):
                self.assertSequenceAlmostEqual(list1, list2)
        else:
            for num1, num2 in zip(seq1, seq2):
                self.assertAlmostEqual(num1, num2, PLACES)

    def test_std_dev(self):
        numpy_std_dev = np.std(np.array(random_points))
        our_std_dev = kmeans.std_dev(random_points)

        self.assertEqual(numpy_std_dev, our_std_dev)

    def test_whiten(self):
        scipy_whiten = vq.whiten(np.array(random_points))
        our_whiten = kmeans.whiten(random_points)

        self.assertEqual(len(scipy_whiten), len(our_whiten))

        comp_whiten = zip(scipy_whiten, our_whiten)

        [self.assertEqual(*comp) for comp in comp_whiten]

        scipy_whiten = vq.whiten(np.array(random_points_3d))
        our_whiten = kmeans.whiten(random_points_3d)

        self.assertEqual(len(scipy_whiten), len(our_whiten))

        comp_whiten = zip(scipy_whiten, our_whiten)

        [self.assertSequenceEqual(scipy_list.tolist(), our_list) for \
                scipy_list, our_list in comp_whiten]

    def test_scipy_vq_versions_1d(self):
        one_dim_points = [1.9, 2.3, 1.5, 2.5, 0.8, 0.6, 0.4, 1.8, 1.0, 1.0]
        vq_points = np.array([[p] for p in one_dim_points])
        book = np.array([vq_points[3], vq_points[7]])

        c_code, c_dist = vq.vq(vq_points, book)
        p_code, p_dist = vq.py_vq(vq_points, book)

        self.assertSequenceEqual(c_code.tolist(), p_code.tolist())
        self.assertSequenceEqual(c_dist.tolist(), p_dist.tolist())

    def test_scipy_vq_versions(self):
        vq_points = np.array([[ 1.9,2.3],
                              [ 1.5,2.5],
                              [ 0.8,0.6],
                              [ 0.4,1.8],
                              [ 1.0,1.0]])
        book = np.array((vq_points[0], vq_points[2]))

        c_code, c_dist = vq.vq(vq_points, book)
        p_code, p_dist = vq.py_vq(vq_points, book)

        self.assertSequenceEqual(c_code.tolist(), p_code.tolist())
        self.assertSequenceAlmostEqual(c_dist.tolist(), p_dist.tolist())

    def test_vq_1d(self):
        book = [p for p in random.sample(random_points, 8)]

        # SciPy doesn't work with 1d arrays yet so the 1d test data needs to
        # be transformed to a multidimensional representation
        one_d_array = np.array([[p] for p in random_points])
        one_d_book_array = np.array([[b] for b in book])

        s_code, s_dist = vq.vq(one_d_array, one_d_book_array)
        m_code, m_dist = kmeans.vq(random_points, book)

        self.assertSequenceEqual(s_code.tolist(), m_code)
        self.assertSequenceAlmostEqual(s_dist.tolist(), m_dist)

    def test_vq(self):
        book = [p for p in random.sample(random_points_3d, 8)]

        s_code, s_dist = vq.vq(np.array(random_points_3d), np.array(book))
        m_code, m_dist = kmeans.vq(random_points_3d, book)

        self.assertSequenceEqual(s_code.tolist(), m_code)
        self.assertSequenceAlmostEqual(s_dist.tolist(), m_dist)

    def test_kmeans(self):
        centroids = [p for p in random.sample(random_points_3d, 3)]
        s_centroids, s_distance = \
                vq.kmeans(np.array(random_points_3d), np.array(centroids))
        m_centroids, m_distance = kmeans.kmeans(random_points_3d, centroids)
        self.assertEqual(s_distance, m_distance)

        self.assertEqual(len(s_centroids.tolist()), len(m_centroids))
        self.assertSequenceAlmostEqual(s_centroids.tolist(), m_centroids)

    def test_no_outliers(self):
        points = [[i,i] for i in range(300)]
        c_outliers = cluster.find_outliers(points, whitened=False)
        m_outliers = kmeans.find_outliers(points, whitened=False)

        self.assertEqual(c_outliers, [])
        self.assertEqual(m_outliers, [])

    def test_find_outliers(self):
        c_outliers = cluster.find_outliers(random_points_3d, whitened=False)
        m_outliers = kmeans.find_outliers(random_points_3d, whitened=False)

        self.assertEqual(len(c_outliers), len(m_outliers))
        self.assertSequenceEqual(c_outliers, m_outliers)
<commit_msg>Add test for kmeans_optm<commit_after>import os
import random
from django.test import TestCase
from avocado.stats import cluster, kmeans
from scipy.cluster import vq
import numpy as np
from itertools import chain

__all__ = ('KmeansTestCase',)

random_points_file = open(os.path.join(os.path.dirname(__file__), '../fixtures/random_points.txt'))
random_points_3d_file = open(os.path.join(os.path.dirname(__file__), '../fixtures/random_points_3d.txt'))

random_points = [float(x.strip()) for x in random_points_file.xreadlines()]
random_points_3d = [[float(x) for x in l.strip().split(",")] for l in random_points_3d_file.xreadlines()]

PLACES = 10

class KmeansTestCase(TestCase):
    def assertSequenceAlmostEqual(self, seq1, seq2):
        """
        Helper method for checking that 2 sequences are almost equal.

        Two sequences are considered almost equal if they have the same order
        and each pair of elements of the sequences passes the
        assertAlmostEqual to the number of decimal places specified in PLACES.
        This method will also work for nested lists of numbers. For example,
        let's say we wanted to see if two collections of 3D points were almost
        equal, we could use the following:

            pts1 = [[0.206331960142751, 0.612540082088810, 0.843236918599283],
                    [0.269705965446683, 0.218132746363829, 0.277332011689122],
                    [0.728684538148946, 0.734953792412333, 0.722476119561547]]
            pts2 = [[0.206331960142700, 0.612540082088819, 0.843236918599288],
                    [0.269705965446609, 0.218132746363899, 0.277332011689182],
                    [0.728684538148046, 0.734953792412933, 0.722476119561847]]
            assertSequenceAlmostEqual(pts1, pts2)

        Arguments:
            seq1, seq2: (nested)list of numbers to check for near equality

        NOTE: This method assumes 'seq1' 'seq2' have equal, non-zero length. If
        you are not certain the lengths match, use the following assert before
        calling this method or this method might have unpredictable results.
        For nested lists, it is not only assumed that overall list length is
        the same, but also that each nested list pair is of equal length.

            assertEqual(len(seq1), len(seq2))
        """
        if isinstance(seq1[0], list):
            for list1, list2 in zip(seq1, seq2):
                self.assertSequenceAlmostEqual(list1, list2)
        else:
            for num1, num2 in zip(seq1, seq2):
                self.assertAlmostEqual(num1, num2, PLACES)

    def test_std_dev(self):
        numpy_std_dev = np.std(np.array(random_points))
        our_std_dev = kmeans.std_dev(random_points)

        self.assertEqual(numpy_std_dev, our_std_dev)

    def test_whiten(self):
        scipy_whiten = vq.whiten(np.array(random_points))
        our_whiten = kmeans.whiten(random_points)

        self.assertEqual(len(scipy_whiten), len(our_whiten))

        comp_whiten = zip(scipy_whiten, our_whiten)

        [self.assertEqual(*comp) for comp in comp_whiten]

        scipy_whiten = vq.whiten(np.array(random_points_3d))
        our_whiten = kmeans.whiten(random_points_3d)

        self.assertEqual(len(scipy_whiten), len(our_whiten))

        comp_whiten = zip(scipy_whiten, our_whiten)

        [self.assertSequenceEqual(scipy_list.tolist(), our_list) for \
                scipy_list, our_list in comp_whiten]

    def test_scipy_vq_versions_1d(self):
        one_dim_points = [1.9, 2.3, 1.5, 2.5, 0.8, 0.6, 0.4, 1.8, 1.0, 1.0]
        vq_points = np.array([[p] for p in one_dim_points])
        book = np.array([vq_points[3], vq_points[7]])

        c_code, c_dist = vq.vq(vq_points, book)
        p_code, p_dist = vq.py_vq(vq_points, book)

        self.assertSequenceEqual(c_code.tolist(), p_code.tolist())
        self.assertSequenceEqual(c_dist.tolist(), p_dist.tolist())

    def test_scipy_vq_versions(self):
        vq_points = np.array([[ 1.9,2.3],
                              [ 1.5,2.5],
                              [ 0.8,0.6],
                              [ 0.4,1.8],
                              [ 1.0,1.0]])
        book = np.array((vq_points[0], vq_points[2]))

        c_code, c_dist = vq.vq(vq_points, book)
        p_code, p_dist = vq.py_vq(vq_points, book)

        self.assertSequenceEqual(c_code.tolist(), p_code.tolist())
        self.assertSequenceAlmostEqual(c_dist.tolist(), p_dist.tolist())

    def test_vq_1d(self):
        book = [p for p in random.sample(random_points, 8)]

        # SciPy doesn't work with 1d arrays yet so the 1d test data needs to
        # be transformed to a multidimensional representation
        one_d_array = np.array([[p] for p in random_points])
        one_d_book_array = np.array([[b] for b in book])

        s_code, s_dist = vq.vq(one_d_array, one_d_book_array)
        m_code, m_dist = kmeans.vq(random_points, book)

        self.assertSequenceEqual(s_code.tolist(), m_code)
        self.assertSequenceAlmostEqual(s_dist.tolist(), m_dist)

    def test_vq(self):
        book = [p for p in random.sample(random_points_3d, 8)]

        s_code, s_dist = vq.vq(np.array(random_points_3d), np.array(book))
        m_code, m_dist = kmeans.vq(random_points_3d, book)

        self.assertSequenceEqual(s_code.tolist(), m_code)
        self.assertSequenceAlmostEqual(s_dist.tolist(), m_dist)

    def test_kmeans(self):
        centroids = [p for p in random.sample(random_points_3d, 3)]
        s_centroids, s_distance = \
                vq.kmeans(np.array(random_points_3d), np.array(centroids))
        m_centroids, m_distance = kmeans.kmeans(random_points_3d, centroids)
        self.assertEqual(s_distance, m_distance)

        self.assertEqual(len(s_centroids.tolist()), len(m_centroids))
        self.assertSequenceAlmostEqual(s_centroids.tolist(), m_centroids)

    def test_no_outliers(self):
        points = [[i,i] for i in range(300)]
        c_outliers = cluster.find_outliers(points, whitened=False)
        m_outliers = kmeans.find_outliers(points, whitened=False)

        self.assertEqual(c_outliers, [])
        self.assertEqual(m_outliers, [])

    def test_find_outliers(self):
        c_outliers = cluster.find_outliers(random_points_3d, whitened=False)
        m_outliers = kmeans.find_outliers(random_points_3d, whitened=False)

        self.assertEqual(len(c_outliers), len(m_outliers))
        self.assertSequenceEqual(c_outliers, m_outliers)

    def test_kmeans_optm(self):
        c_kmeans = cluster.kmeans_optm(random_points_3d)
        m_kmeans = kmeans.kmeans_optm(random_points_3d)

        # Make sure all the right keys are present in both dictionaries
        for key in ['centroids', 'indexes', 'distances', 'outliers']:
            self.assertTrue(c_kmeans.has_key(key) and m_kmeans.has_key(key))

        c_indexes = c_kmeans['indexes'].tolist()
        m_indexes = m_kmeans['indexes']
        self.assertSequenceEqual(c_indexes, m_indexes)

        c_distances = c_kmeans['distances']
        m_distances = m_kmeans['distances']
        self.assertSequenceAlmostEqual(c_distances, m_distances)

        c_centroids = c_kmeans['centroids']
        m_centroids = m_kmeans['centroids']
        self.assertSequenceAlmostEqual(c_centroids.tolist(), m_centroids)

        c_outliers = c_kmeans['outliers']
        m_outliers = m_kmeans['outliers']
        self.assertSequenceEqual(c_outliers, m_outliers)
<|endoftext|>


<commit_before>from math import pi

import pytest

from compas.geometry import Transformation
from compas.geometry import Translation
from compas.robots import Axis
from compas.robots import Joint
from compas.robots import Limit
from compas.robots import Mimic

def test_constructor_joint_type_str():
    assert Joint("joint", 'revolute', "parent_joint", "child_joint")
    assert Joint("joint", 'continuous', "parent_joint", "child_joint")
    assert Joint("joint", 'prismatic', "parent_joint", "child_joint")
    assert Joint("joint", 'fixed', "parent_joint", "child_joint")

def test_constructor_joint_type_int():
    assert Joint("joint", Joint.REVOLUTE, "parent_joint", "child_joint")
    assert Joint("joint", Joint.CONTINUOUS, "parent_joint", "child_joint")
    assert Joint("joint", Joint.PRISMATIC, "parent_joint", "child_joint")
    assert Joint("joint", Joint.FIXED, "parent_joint", "child_joint")

def test_constructor_joint_type_guard_str():
    with pytest.raises(ValueError):
        Joint("joint", "synovial", "femur", "tibia")

def test_constructor_joint_type_guard_int():
    joint_types = [-1, len(Joint.SUPPORTED_TYPES)]  # just outside acceptable range
    for type_ in joint_types:
        with pytest.raises(ValueError):
            Joint("joint", type_, "femur", "tibia")

def test_revolute_calculate_transformation():
    limit = Limit(lower=-2 * pi, upper=2 * pi)
    j1 = Joint('j1', 'revolute', None, None, limit=limit)
    transformation = j1.calculate_transformation(2*pi)
    assert transformation == Transformation()

def test_prismatic_calculate_transformation():
    limit = Limit(lower=0, upper=1000)
    j1 = Joint('j1', 'prismatic', None, None, axis=Axis('1 0 0'), limit=limit)
    t = j1.calculate_transformation(550)
    assert t == Translation.from_vector([550, 0, 0])

def test_mimic_calculate_position():
    multiplier = 5.0
    offset = 100.
    j1 = Joint('j1', 'prismatic', None, None, axis=Axis('1 0 0'))
    j1.position = 200
    mimic = Mimic(j1, multiplier=multiplier, offset=offset)
    result = mimic.calculate_position(j1.position)
    assert result == multiplier * j1.position + offset
<commit_msg>Make test simpler.<commit_after>from math import pi

import pytest

from compas.geometry import Transformation
from compas.geometry import Translation
from compas.robots import Axis
from compas.robots import Joint
from compas.robots import Limit
from compas.robots import Mimic

def test_constructor_joint_type_str():
    assert Joint("joint", 'revolute', "parent_joint", "child_joint")
    assert Joint("joint", 'continuous', "parent_joint", "child_joint")
    assert Joint("joint", 'prismatic', "parent_joint", "child_joint")
    assert Joint("joint", 'fixed', "parent_joint", "child_joint")

def test_constructor_joint_type_int():
    assert Joint("joint", Joint.REVOLUTE, "parent_joint", "child_joint")
    assert Joint("joint", Joint.CONTINUOUS, "parent_joint", "child_joint")
    assert Joint("joint", Joint.PRISMATIC, "parent_joint", "child_joint")
    assert Joint("joint", Joint.FIXED, "parent_joint", "child_joint")

def test_constructor_joint_type_guard_str():
    with pytest.raises(ValueError):
        Joint("joint", "synovial", "femur", "tibia")

def test_constructor_joint_type_guard_int():
    with pytest.raises(ValueError):
        Joint("weld", 620, "base_metal1", "base_metal2")

def test_revolute_calculate_transformation():
    limit = Limit(lower=-2 * pi, upper=2 * pi)
    j1 = Joint('j1', 'revolute', None, None, limit=limit)
    transformation = j1.calculate_transformation(2*pi)
    assert transformation == Transformation()

def test_prismatic_calculate_transformation():
    limit = Limit(lower=0, upper=1000)
    j1 = Joint('j1', 'prismatic', None, None, axis=Axis('1 0 0'), limit=limit)
    t = j1.calculate_transformation(550)
    assert t == Translation.from_vector([550, 0, 0])

def test_mimic_calculate_position():
    multiplier = 5.0
    offset = 100.
    j1 = Joint('j1', 'prismatic', None, None, axis=Axis('1 0 0'))
    j1.position = 200
    mimic = Mimic(j1, multiplier=multiplier, offset=offset)
    result = mimic.calculate_position(j1.position)
    assert result == multiplier * j1.position + offset
<|endoftext|>


<commit_before>"""Tests for parallel client.py

Authors:

* Min RK
"""

#-------------------------------------------------------------------------------
#  Copyright (C) 2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
# Imports
#-------------------------------------------------------------------------------

from __future__ import division

import time
from datetime import datetime
from tempfile import mktemp

import zmq

from IPython.parallel.client import client as clientmod
from IPython.parallel import error
from IPython.parallel import AsyncResult, AsyncHubResult
from IPython.parallel import LoadBalancedView, DirectView

from clienttest import ClusterTestCase, segfault, wait, add_engines

def setup():
    add_engines(4, total=True)

class TestClient(ClusterTestCase):

    def test_ids(self):
        n = len(self.client.ids)
        self.add_engines(2)
        self.assertEquals(len(self.client.ids), n+2)

    def test_view_indexing(self):
        """test index access for views"""
        self.minimum_engines(4)
        targets = self.client._build_targets('all')[-1]
        v = self.client[:]
        self.assertEquals(v.targets, targets)
        t = self.client.ids[2]
        v = self.client[t]
        self.assert_(isinstance(v, DirectView))
        self.assertEquals(v.targets, t)
        t = self.client.ids[2:4]
        v = self.client[t]
        self.assert_(isinstance(v, DirectView))
        self.assertEquals(v.targets, t)
        v = self.client[::2]
        self.assert_(isinstance(v, DirectView))
        self.assertEquals(v.targets, targets[::2])
        v = self.client[1::3]
        self.assert_(isinstance(v, DirectView))
        self.assertEquals(v.targets, targets[1::3])
        v = self.client[:-3]
        self.assert_(isinstance(v, DirectView))
        self.assertEquals(v.targets, targets[:-3])
        v = self.client[-1]
        self.assert_(isinstance(v, DirectView))
        self.assertEquals(v.targets, targets[-1])
        self.assertRaises(TypeError, lambda : self.client[None])

    def test_lbview_targets(self):
        """test load_balanced_view targets"""
        v = self.client.load_balanced_view()
        self.assertEquals(v.targets, None)
        v = self.client.load_balanced_view(-1)
        self.assertEquals(v.targets, [self.client.ids[-1]])
        v = self.client.load_balanced_view('all')
        self.assertEquals(v.targets, None)

    def test_dview_targets(self):
        """test direct_view targets"""
        v = self.client.direct_view()
        self.assertEquals(v.targets, 'all')
        v = self.client.direct_view('all')
        self.assertEquals(v.targets, 'all')
        v = self.client.direct_view(-1)
        self.assertEquals(v.targets, self.client.ids[-1])

    def test_lazy_all_targets(self):
        """test lazy evaluation of rc.direct_view('all')"""
        v = self.client.direct_view()
        self.assertEquals(v.targets, 'all')

        def double(x):
            return x*2
        seq = range(100)
        ref = [ double(x) for x in seq ]

        # add some engines, which should be used
        self.add_engines(1)
        n1 = len(self.client.ids)

        # simple apply
        r = v.apply_sync(lambda : 1)
        self.assertEquals(r, [1] * n1)

        # map goes through remotefunction
        r = v.map_sync(double, seq)
        self.assertEquals(r, ref)

        # add a couple more engines, and try again
        self.add_engines(2)
        n2 = len(self.client.ids)
        self.assertNotEquals(n2, n1)

        # apply
        r = v.apply_sync(lambda : 1)
        self.assertEquals(r, [1] * n2)

        # map
        r = v.map_sync(double, seq)
        self.assertEquals(r, ref)

    def test_targets(self):
        """test various valid targets arguments"""
        build = self.client._build_targets
        ids = self.client.ids
        idents,targets = build(None)
        self.assertEquals(ids, targets)

    def test_clear(self):
        """test clear behavior"""
        self.minimum_engines(2)
        v = self.client[:]
        v.block=True
        v.push(dict(a=5))
        v.pull('a')
        id0 = self.client.ids[-1]
        self.client.clear(targets=id0, block=True)
        a = self.client[:-1].get('a')
        self.assertRaisesRemote(NameError, self.client[id0].get, 'a')
        self.client.clear(block=True)
        for i in self.client.ids:
            self.assertRaisesRemote(NameError, self.client[i].get, 'a')

    def test_get_result(self):
        """test getting results from the Hub."""
        c = clientmod.Client(profile='iptest')
        t = c.ids[-1]
        ar = c[t].apply_async(wait, 1)
        # give the monitor time to notice the message
        time.sleep(.25)
        ahr = self.client.get_result(ar.msg_ids)
        self.assertTrue(isinstance(ahr, AsyncHubResult))
        self.assertEquals(ahr.get(), ar.get())
        ar2 = self.client.get_result(ar.msg_ids)
        self.assertFalse(isinstance(ar2, AsyncHubResult))
        c.close()

    def test_ids_list(self):
        """test client.ids"""
        ids = self.client.ids
        self.assertEquals(ids, self.client._ids)
        self.assertFalse(ids is self.client._ids)
        ids.remove(ids[-1])
        self.assertNotEquals(ids, self.client._ids)

    def test_queue_status(self):
        ids = self.client.ids
        id0 = ids[0]
        qs = self.client.queue_status(targets=id0)
        self.assertTrue(isinstance(qs, dict))
        self.assertEquals(sorted(qs.keys()), ['completed', 'queue', 'tasks'])
        allqs = self.client.queue_status()
        self.assertTrue(isinstance(allqs, dict))
        intkeys = list(allqs.keys())
        intkeys.remove('unassigned')
        self.assertEquals(sorted(intkeys), sorted(self.client.ids))
        unassigned = allqs.pop('unassigned')
        for eid,qs in allqs.items():
            self.assertTrue(isinstance(qs, dict))
            self.assertEquals(sorted(qs.keys()), ['completed', 'queue', 'tasks'])

    def test_shutdown(self):
        ids = self.client.ids
        id0 = ids[0]
        self.client.shutdown(id0, block=True)
        while id0 in self.client.ids:
            time.sleep(0.1)
            self.client.spin()

        self.assertRaises(IndexError, lambda : self.client[id0])

    def test_result_status(self):
        pass
        # to be written

    def test_db_query_dt(self):
        """test db query by date"""
        hist = self.client.hub_history()
        middle = self.client.db_query({'msg_id' : hist[len(hist)//2]})[0]
        tic = middle['submitted']
        before = self.client.db_query({'submitted' : {'$lt' : tic}})
        after = self.client.db_query({'submitted' : {'$gte' : tic}})
        self.assertEquals(len(before)+len(after),len(hist))
        for b in before:
            self.assertTrue(b['submitted'] < tic)
        for a in after:
            self.assertTrue(a['submitted'] >= tic)
        same = self.client.db_query({'submitted' : tic})
        for s in same:
            self.assertTrue(s['submitted'] == tic)

    def test_db_query_keys(self):
        """test extracting subset of record keys"""
        found = self.client.db_query({'msg_id': {'$ne' : ''}},keys=['submitted', 'completed'])
        for rec in found:
            self.assertEquals(set(rec.keys()), set(['msg_id', 'submitted', 'completed']))

    def test_db_query_default_keys(self):
        """default db_query excludes buffers"""
        found = self.client.db_query({'msg_id': {'$ne' : ''}})
        for rec in found:
            keys = set(rec.keys())
            self.assertFalse('buffers' in keys, "'buffers' should not be in: %s" % keys)
            self.assertFalse('result_buffers' in keys, "'result_buffers' should not be in: %s" % keys)

    def test_db_query_msg_id(self):
        """ensure msg_id is always in db queries"""
        found = self.client.db_query({'msg_id': {'$ne' : ''}},keys=['submitted', 'completed'])
        for rec in found:
            self.assertTrue('msg_id' in rec.keys())
        found = self.client.db_query({'msg_id': {'$ne' : ''}},keys=['submitted'])
        for rec in found:
            self.assertTrue('msg_id' in rec.keys())
        found = self.client.db_query({'msg_id': {'$ne' : ''}},keys=['msg_id'])
        for rec in found:
            self.assertTrue('msg_id' in rec.keys())

    def test_db_query_get_result(self):
        """pop in db_query shouldn't pop from result itself"""
        self.client[:].apply_sync(lambda : 1)
        found = self.client.db_query({'msg_id': {'$ne' : ''}})
        rc2 = clientmod.Client(profile='iptest')
        # If this bug is not fixed, this call will hang:
        ar = rc2.get_result(self.client.history[-1])
        ar.wait(2)
        self.assertTrue(ar.ready())
        ar.get()
        rc2.close()

    def test_db_query_in(self):
        """test db query with '$in','$nin' operators"""
        hist = self.client.hub_history()
        even = hist[::2]
        odd = hist[1::2]
        recs = self.client.db_query({ 'msg_id' : {'$in' : even}})
        found = [ r['msg_id'] for r in recs ]
        self.assertEquals(set(even), set(found))
        recs = self.client.db_query({ 'msg_id' : {'$nin' : even}})
        found = [ r['msg_id'] for r in recs ]
        self.assertEquals(set(odd), set(found))

    def test_hub_history(self):
        hist = self.client.hub_history()
        recs = self.client.db_query({ 'msg_id' : {"$ne":''}})
        recdict = {}
        for rec in recs:
            recdict[rec['msg_id']] = rec

        latest = datetime(1984,1,1)
        for msg_id in hist:
            rec = recdict[msg_id]
            newt = rec['submitted']
            self.assertTrue(newt >= latest)
            latest = newt
        ar = self.client[-1].apply_async(lambda : 1)
        ar.get()
        time.sleep(0.25)
        self.assertEquals(self.client.hub_history()[-1:],ar.msg_ids)

    def test_resubmit(self):
        def f():
            import random
            return random.random()
        v = self.client.load_balanced_view()
        ar = v.apply_async(f)
        r1 = ar.get(1)
        # give the Hub a chance to notice:
        time.sleep(0.5)
        ahr = self.client.resubmit(ar.msg_ids)
        r2 = ahr.get(1)
        self.assertFalse(r1 == r2)

    def test_resubmit_inflight(self):
        """ensure ValueError on resubmit of inflight task"""
        v = self.client.load_balanced_view()
        ar = v.apply_async(time.sleep,1)
        # give the message a chance to arrive
        time.sleep(0.2)
        self.assertRaisesRemote(ValueError, self.client.resubmit, ar.msg_ids)
        ar.get(2)

    def test_resubmit_badkey(self):
        """ensure KeyError on resubmit of nonexistant task"""
        self.assertRaisesRemote(KeyError, self.client.resubmit, ['invalid'])

    def test_purge_results(self):
        # ensure there are some tasks
        for i in range(5):
            self.client[:].apply_sync(lambda : 1)
        # Wait for the Hub to realise the result is done:
        # This prevents a race condition, where we
        # might purge a result the Hub still thinks is pending.
        time.sleep(0.1)
        rc2 = clientmod.Client(profile='iptest')
        hist = self.client.hub_history()
        ahr = rc2.get_result([hist[-1]])
        ahr.wait(10)
        self.client.purge_results(hist[-1])
        newhist = self.client.hub_history()
        self.assertEquals(len(newhist)+1,len(hist))
        rc2.spin()
        rc2.close()

    def test_purge_all_results(self):
        self.client.purge_results('all')
        hist = self.client.hub_history()
        self.assertEquals(len(hist), 0)

    def test_spin_thread(self):
        self.client.spin_thread(0.01)
        ar = self.client[-1].apply_async(lambda : 1)
        time.sleep(0.1)
        self.assertTrue(ar.wall_time < 0.1,
            "spin should have kept wall_time < 0.1, but got %f" % ar.wall_time
        )

    def test_stop_spin_thread(self):
        self.client.spin_thread(0.01)
        self.client.stop_spin_thread()
        ar = self.client[-1].apply_async(lambda : 1)
        time.sleep(0.15)
        self.assertTrue(ar.wall_time > 0.1,
            "Shouldn't be spinning, but got wall_time=%f" % ar.wall_time
        )

<commit_msg>test resubmit of aborted tasks (#1647)<commit_after>"""Tests for parallel client.py

Authors:

* Min RK
"""

#-------------------------------------------------------------------------------
#  Copyright (C) 2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
# Imports
#-------------------------------------------------------------------------------

from __future__ import division

import time
from datetime import datetime
from tempfile import mktemp

import zmq

from IPython.parallel.client import client as clientmod
from IPython.parallel import error
from IPython.parallel import AsyncResult, AsyncHubResult
from IPython.parallel import LoadBalancedView, DirectView

from clienttest import ClusterTestCase, segfault, wait, add_engines

def setup():
    add_engines(4, total=True)

class TestClient(ClusterTestCase):

    def test_ids(self):
        n = len(self.client.ids)
        self.add_engines(2)
        self.assertEquals(len(self.client.ids), n+2)

    def test_view_indexing(self):
        """test index access for views"""
        self.minimum_engines(4)
        targets = self.client._build_targets('all')[-1]
        v = self.client[:]
        self.assertEquals(v.targets, targets)
        t = self.client.ids[2]
        v = self.client[t]
        self.assert_(isinstance(v, DirectView))
        self.assertEquals(v.targets, t)
        t = self.client.ids[2:4]
        v = self.client[t]
        self.assert_(isinstance(v, DirectView))
        self.assertEquals(v.targets, t)
        v = self.client[::2]
        self.assert_(isinstance(v, DirectView))
        self.assertEquals(v.targets, targets[::2])
        v = self.client[1::3]
        self.assert_(isinstance(v, DirectView))
        self.assertEquals(v.targets, targets[1::3])
        v = self.client[:-3]
        self.assert_(isinstance(v, DirectView))
        self.assertEquals(v.targets, targets[:-3])
        v = self.client[-1]
        self.assert_(isinstance(v, DirectView))
        self.assertEquals(v.targets, targets[-1])
        self.assertRaises(TypeError, lambda : self.client[None])

    def test_lbview_targets(self):
        """test load_balanced_view targets"""
        v = self.client.load_balanced_view()
        self.assertEquals(v.targets, None)
        v = self.client.load_balanced_view(-1)
        self.assertEquals(v.targets, [self.client.ids[-1]])
        v = self.client.load_balanced_view('all')
        self.assertEquals(v.targets, None)

    def test_dview_targets(self):
        """test direct_view targets"""
        v = self.client.direct_view()
        self.assertEquals(v.targets, 'all')
        v = self.client.direct_view('all')
        self.assertEquals(v.targets, 'all')
        v = self.client.direct_view(-1)
        self.assertEquals(v.targets, self.client.ids[-1])

    def test_lazy_all_targets(self):
        """test lazy evaluation of rc.direct_view('all')"""
        v = self.client.direct_view()
        self.assertEquals(v.targets, 'all')

        def double(x):
            return x*2
        seq = range(100)
        ref = [ double(x) for x in seq ]

        # add some engines, which should be used
        self.add_engines(1)
        n1 = len(self.client.ids)

        # simple apply
        r = v.apply_sync(lambda : 1)
        self.assertEquals(r, [1] * n1)

        # map goes through remotefunction
        r = v.map_sync(double, seq)
        self.assertEquals(r, ref)

        # add a couple more engines, and try again
        self.add_engines(2)
        n2 = len(self.client.ids)
        self.assertNotEquals(n2, n1)

        # apply
        r = v.apply_sync(lambda : 1)
        self.assertEquals(r, [1] * n2)

        # map
        r = v.map_sync(double, seq)
        self.assertEquals(r, ref)

    def test_targets(self):
        """test various valid targets arguments"""
        build = self.client._build_targets
        ids = self.client.ids
        idents,targets = build(None)
        self.assertEquals(ids, targets)

    def test_clear(self):
        """test clear behavior"""
        self.minimum_engines(2)
        v = self.client[:]
        v.block=True
        v.push(dict(a=5))
        v.pull('a')
        id0 = self.client.ids[-1]
        self.client.clear(targets=id0, block=True)
        a = self.client[:-1].get('a')
        self.assertRaisesRemote(NameError, self.client[id0].get, 'a')
        self.client.clear(block=True)
        for i in self.client.ids:
            self.assertRaisesRemote(NameError, self.client[i].get, 'a')

    def test_get_result(self):
        """test getting results from the Hub."""
        c = clientmod.Client(profile='iptest')
        t = c.ids[-1]
        ar = c[t].apply_async(wait, 1)
        # give the monitor time to notice the message
        time.sleep(.25)
        ahr = self.client.get_result(ar.msg_ids)
        self.assertTrue(isinstance(ahr, AsyncHubResult))
        self.assertEquals(ahr.get(), ar.get())
        ar2 = self.client.get_result(ar.msg_ids)
        self.assertFalse(isinstance(ar2, AsyncHubResult))
        c.close()

    def test_ids_list(self):
        """test client.ids"""
        ids = self.client.ids
        self.assertEquals(ids, self.client._ids)
        self.assertFalse(ids is self.client._ids)
        ids.remove(ids[-1])
        self.assertNotEquals(ids, self.client._ids)

    def test_queue_status(self):
        ids = self.client.ids
        id0 = ids[0]
        qs = self.client.queue_status(targets=id0)
        self.assertTrue(isinstance(qs, dict))
        self.assertEquals(sorted(qs.keys()), ['completed', 'queue', 'tasks'])
        allqs = self.client.queue_status()
        self.assertTrue(isinstance(allqs, dict))
        intkeys = list(allqs.keys())
        intkeys.remove('unassigned')
        self.assertEquals(sorted(intkeys), sorted(self.client.ids))
        unassigned = allqs.pop('unassigned')
        for eid,qs in allqs.items():
            self.assertTrue(isinstance(qs, dict))
            self.assertEquals(sorted(qs.keys()), ['completed', 'queue', 'tasks'])

    def test_shutdown(self):
        ids = self.client.ids
        id0 = ids[0]
        self.client.shutdown(id0, block=True)
        while id0 in self.client.ids:
            time.sleep(0.1)
            self.client.spin()

        self.assertRaises(IndexError, lambda : self.client[id0])

    def test_result_status(self):
        pass
        # to be written

    def test_db_query_dt(self):
        """test db query by date"""
        hist = self.client.hub_history()
        middle = self.client.db_query({'msg_id' : hist[len(hist)//2]})[0]
        tic = middle['submitted']
        before = self.client.db_query({'submitted' : {'$lt' : tic}})
        after = self.client.db_query({'submitted' : {'$gte' : tic}})
        self.assertEquals(len(before)+len(after),len(hist))
        for b in before:
            self.assertTrue(b['submitted'] < tic)
        for a in after:
            self.assertTrue(a['submitted'] >= tic)
        same = self.client.db_query({'submitted' : tic})
        for s in same:
            self.assertTrue(s['submitted'] == tic)

    def test_db_query_keys(self):
        """test extracting subset of record keys"""
        found = self.client.db_query({'msg_id': {'$ne' : ''}},keys=['submitted', 'completed'])
        for rec in found:
            self.assertEquals(set(rec.keys()), set(['msg_id', 'submitted', 'completed']))

    def test_db_query_default_keys(self):
        """default db_query excludes buffers"""
        found = self.client.db_query({'msg_id': {'$ne' : ''}})
        for rec in found:
            keys = set(rec.keys())
            self.assertFalse('buffers' in keys, "'buffers' should not be in: %s" % keys)
            self.assertFalse('result_buffers' in keys, "'result_buffers' should not be in: %s" % keys)

    def test_db_query_msg_id(self):
        """ensure msg_id is always in db queries"""
        found = self.client.db_query({'msg_id': {'$ne' : ''}},keys=['submitted', 'completed'])
        for rec in found:
            self.assertTrue('msg_id' in rec.keys())
        found = self.client.db_query({'msg_id': {'$ne' : ''}},keys=['submitted'])
        for rec in found:
            self.assertTrue('msg_id' in rec.keys())
        found = self.client.db_query({'msg_id': {'$ne' : ''}},keys=['msg_id'])
        for rec in found:
            self.assertTrue('msg_id' in rec.keys())

    def test_db_query_get_result(self):
        """pop in db_query shouldn't pop from result itself"""
        self.client[:].apply_sync(lambda : 1)
        found = self.client.db_query({'msg_id': {'$ne' : ''}})
        rc2 = clientmod.Client(profile='iptest')
        # If this bug is not fixed, this call will hang:
        ar = rc2.get_result(self.client.history[-1])
        ar.wait(2)
        self.assertTrue(ar.ready())
        ar.get()
        rc2.close()

    def test_db_query_in(self):
        """test db query with '$in','$nin' operators"""
        hist = self.client.hub_history()
        even = hist[::2]
        odd = hist[1::2]
        recs = self.client.db_query({ 'msg_id' : {'$in' : even}})
        found = [ r['msg_id'] for r in recs ]
        self.assertEquals(set(even), set(found))
        recs = self.client.db_query({ 'msg_id' : {'$nin' : even}})
        found = [ r['msg_id'] for r in recs ]
        self.assertEquals(set(odd), set(found))

    def test_hub_history(self):
        hist = self.client.hub_history()
        recs = self.client.db_query({ 'msg_id' : {"$ne":''}})
        recdict = {}
        for rec in recs:
            recdict[rec['msg_id']] = rec

        latest = datetime(1984,1,1)
        for msg_id in hist:
            rec = recdict[msg_id]
            newt = rec['submitted']
            self.assertTrue(newt >= latest)
            latest = newt
        ar = self.client[-1].apply_async(lambda : 1)
        ar.get()
        time.sleep(0.25)
        self.assertEquals(self.client.hub_history()[-1:],ar.msg_ids)

    def _wait_for_idle(self):
        """wait for an engine to become idle, according to the Hub"""
        rc = self.client

        # timeout 2s, polling every 100ms
        for i in range(20):
            qs = rc.queue_status()
            if qs['unassigned'] or any(qs[eid]['tasks'] for eid in rc.ids):
                time.sleep(0.1)
            else:
                break

        # ensure Hub up to date:
        qs = rc.queue_status()
        self.assertEquals(qs['unassigned'], 0)
        for eid in rc.ids:
            self.assertEquals(qs[eid]['tasks'], 0)

    def test_resubmit(self):
        def f():
            import random
            return random.random()
        v = self.client.load_balanced_view()
        ar = v.apply_async(f)
        r1 = ar.get(1)
        # give the Hub a chance to notice:
        self._wait_for_idle()
        ahr = self.client.resubmit(ar.msg_ids)
        r2 = ahr.get(1)
        self.assertFalse(r1 == r2)

    def test_resubmit_aborted(self):
        def f():
            import random
            return random.random()
        v = self.client.load_balanced_view()
        # restrict to one engine, so we can put a sleep
        # ahead of the task, so it will get aborted
        eid = self.client.ids[-1]
        v.targets = [eid]
        sleep = v.apply_async(time.sleep, 0.5)
        ar = v.apply_async(f)
        ar.abort()
        self.assertRaises(error.TaskAborted, ar.get)
        # Give the Hub a chance to get up to date:
        self._wait_for_idle()
        ahr = self.client.resubmit(ar.msg_ids)
        r2 = ahr.get(1)

    def test_resubmit_inflight(self):
        """ensure ValueError on resubmit of inflight task"""
        v = self.client.load_balanced_view()
        ar = v.apply_async(time.sleep,1)
        # give the message a chance to arrive
        time.sleep(0.2)
        self.assertRaisesRemote(ValueError, self.client.resubmit, ar.msg_ids)
        ar.get(2)

    def test_resubmit_badkey(self):
        """ensure KeyError on resubmit of nonexistant task"""
        self.assertRaisesRemote(KeyError, self.client.resubmit, ['invalid'])

    def test_purge_results(self):
        # ensure there are some tasks
        for i in range(5):
            self.client[:].apply_sync(lambda : 1)
        # Wait for the Hub to realise the result is done:
        # This prevents a race condition, where we
        # might purge a result the Hub still thinks is pending.
        time.sleep(0.1)
        rc2 = clientmod.Client(profile='iptest')
        hist = self.client.hub_history()
        ahr = rc2.get_result([hist[-1]])
        ahr.wait(10)
        self.client.purge_results(hist[-1])
        newhist = self.client.hub_history()
        self.assertEquals(len(newhist)+1,len(hist))
        rc2.spin()
        rc2.close()

    def test_purge_all_results(self):
        self.client.purge_results('all')
        hist = self.client.hub_history()
        self.assertEquals(len(hist), 0)

    def test_spin_thread(self):
        self.client.spin_thread(0.01)
        ar = self.client[-1].apply_async(lambda : 1)
        time.sleep(0.1)
        self.assertTrue(ar.wall_time < 0.1,
            "spin should have kept wall_time < 0.1, but got %f" % ar.wall_time
        )

    def test_stop_spin_thread(self):
        self.client.spin_thread(0.01)
        self.client.stop_spin_thread()
        ar = self.client[-1].apply_async(lambda : 1)
        time.sleep(0.15)
        self.assertTrue(ar.wall_time > 0.1,
            "Shouldn't be spinning, but got wall_time=%f" % ar.wall_time
        )

<|endoftext|>


<commit_before>from flask import url_for

def test_add_translation(app, client):
    data = dict(
        word='sheep',
        ipa='sziip',
        translation='owca',
        from_language='en',
        into_language='pl',
    )
    resp = app.test_client().post(
        url_for('ajax.add_translation'),
        data=data,
        headers={'Content-Type': 'application/x-www-form-urlencoded'})
    assert resp.status_code == 200
    assert resp.json == dict(translation=dict(id=1, ipa='sziip', translation='owca', word='sheep'))
<commit_msg>better test_add_translation<commit_after>from flask import url_for
from wordbook.domain.repo.translation import Repo
from wordbook.infra.models.translation import Translation

def test_add_translation(app, client):
    data = dict(
        word='sheep',
        ipa='sziip',
        translation='owca',
        from_language='en',
        into_language='pl',
    )
    resp = app.test_client().post(
        url_for('ajax.add_translation'),
        data=data,
        headers={'Content-Type': 'application/x-www-form-urlencoded'})
    assert resp.status_code == 200
    assert resp.json == dict(translation=dict(id=1, ipa='sziip', translation='owca', word='sheep'))
    repo = Repo()
    translation = repo.get(resp.json['translation']['id'])
    assert translation.translated == 'owca'
    assert translation.from_language == 'en'
    assert translation.word == 'sheep'
    assert translation.ipa == 'sziip'
    assert translation.into_language == 'pl'
<|endoftext|>


<commit_before>####################################################################
# This code is CNN based face recognition programme. It reads 5
# faces from ORL database and the rest 5 are used as test.
# PCA_Performance shows the recognition performance.
#
# Download the ORL database from internet.
# This code is written by Saurabh Puri in order to show the face
# recognition task
#######################################################################
import keras
from keras.optimizers import SGD
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import Flatten
from keras.layers import Activation
from keras.layers.convolutional import Convolution2D
from keras.layers.convolutional import MaxPooling2D
from keras.utils import np_utils
from sklearn.cross_validation import train_test_split
from keras import backend as K
K.set_image_dim_ordering('th')
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
import cv2

def baseline_model():
    #creat model
    model = Sequential()
    model.add(Convolution2D(16, 5, 5, border_mode='same', input_shape=(1,w,h)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2,2)))
    model.add(Activation('relu'))
    model.add(Convolution2D(32, 3, 3, border_mode='same'))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2,2)))
    model.add(Activation('relu'))
    model.add(Flatten())
    model.add(Dense(256))
    model.add(Activation('relu'))
    model.add(Dense(256))
    model.add(Activation('relu'))
    model.add(Dense(num_classes))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model

#fix the random seed for reproducibility
seed = 7
np.random.seed(seed)

zz=1;
noc=40;                 #no_of_classes
nots=5;                 #no_of_training_set

#width and height is hardcoded but could be derived from the image itself
#sometimes for a better performance, cropping of image is required as PCA is generally very sensitive to variations in the image (like light, shadow, etc.)
w = 112
h = 92

#Split the dataset into training and test set
#Folder location: ./att_faces/s*/*.pgm
#First half images in each class is considered as training set and other half are considered to be test set
X_train = np.empty(w*h, dtype=np.float32)
y_train = np.empty(1, dtype=np.int32)
X_test = np.empty(w*h, dtype=np.float32)
y_test = np.empty(1, dtype=np.int32)
for i in range(1,noc+1):
    for j in range(1,nots+1):
        #print(str(i) +' '+ str(j))
        file= "./att_faces/s" + str(i) + "/" +  str(j) + ".pgm"
        im = cv2.imread(file)
        im = im.transpose((2,0,1))
        im = np.expand_dims(im,axis=0)
        imgray = im[0][0]
        im1D = imgray.flatten('F')
        X_train = np.vstack((X_train,im1D))
        y_train = np.hstack((y_train,i-1))

for i in range(1,noc+1):
    for j in range(nots+1,nots+6):
        #print(str(i) +' '+ str(j))
        file= "./att_faces/s" + str(i) + "/" +  str(j) + ".pgm"
        im = cv2.imread(file)
        im = im.transpose((2,0,1))
        im = np.expand_dims(im,axis=0)
        imgray = im[0][0]
        im1D = imgray.flatten('F')
        X_test = np.vstack((X_test,im1D))
        y_test = np.hstack((y_test,i-1))

#delete first row as it was empty
X_train = np.delete(X_train,(0),axis=0)
y_train = np.delete(y_train,(0),axis=0)
X_test = np.delete(X_test,(0),axis=0)
y_test = np.delete(y_test,(0),axis=0)

print('loaded')

print(X_train.shape[0])
X_train = X_train.reshape(X_train.shape[0], 1, 112, 92).astype('float32')
X_test = X_test.reshape(X_test.shape[0], 1, 112, 92).astype('float32')

print(X_train.shape)

#normalize to 0-1
X_train = X_train/255
X_test = X_test/255

#one-hot encode outputs
y_train = np_utils.to_categorical(y_train)
y_test = np_utils.to_categorical(y_test)
num_classes = y_train.shape[1]

print(num_classes)

#build the model
model = baseline_model()

#Fit the model
history1 = model.fit(X_train,y_train, validation_data=(X_test,y_test), nb_epoch=50, batch_size=8, verbose=1)

print('Prediction on saved sample:')
#print(str(model.predict(X_test)))
out = model.predict(X_test[:1])
print(np.argmax(out))
#print(y_test[:200])

<commit_msg>This is to test some git commands. Ignore this.<commit_after>####################################################################
# This code is CNN based face recognition programme. It reads 5
# faces from ORL database and the rest 5 are used as test.
# PCA_Performance shows the recognition performance.
#
# Download the ORL database from internet.
# This code is written by Saurabh Puri in order to show the face
# recognition task
#######################################################################
import keras
from keras.optimizers import SGD
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import Flatten
from keras.layers import Activation
from keras.layers.convolutional import Convolution2D
from keras.layers.convolutional import MaxPooling2D
from keras.utils import np_utils
from sklearn.cross_validation import train_test_split
from keras import backend as K
K.set_image_dim_ordering('th')
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
import cv2

def baseline_model():
    #creat model
    model = Sequential()
    model.add(Convolution2D(16, 5, 5, border_mode='same', input_shape=(1,w,h)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2,2)))
    model.add(Activation('relu'))
    model.add(Convolution2D(32, 3, 3, border_mode='same'))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2,2)))
    model.add(Activation('relu'))
    model.add(Flatten())
    model.add(Dense(256))
    model.add(Activation('relu'))
    model.add(Dense(256))
    model.add(Activation('relu'))
    model.add(Dense(num_classes))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model

#fix the random seed for reproducibility
seed = 7
np.random.seed(seed)

zz=1;
noc=40;                 #no_of_classes
nots=5;                 #no_of_training_set

#width and height is hardcoded but could be derived from the image itself
#sometimes for a better performance, cropping of image is required as PCA is generally very sensitive to variations in the image (like light, shadow, etc.)
w = 112
h = 92

#Split the dataset into training and test set
#Folder location: ./att_faces/s*/*.pgm
#First half images in each class is considered as training set and other half are considered to be test set
X_train = np.empty(w*h, dtype=np.float32)
y_train = np.empty(1, dtype=np.int32)
X_test = np.empty(w*h, dtype=np.float32)
y_test = np.empty(1, dtype=np.int32)
for i in range(1,noc+1):
    for j in range(1,nots+1):
        #print(str(i) +' '+ str(j))
        file= "./att_faces/s" + str(i) + "/" +  str(j) + ".pgm"
        im = cv2.imread(file)
        im = im.transpose((2,0,1))
        im = np.expand_dims(im,axis=0)
        imgray = im[0][0]
        im1D = imgray.flatten('F')
        X_train = np.vstack((X_train,im1D))
        y_train = np.hstack((y_train,i-1))

for i in range(1,noc+1):
    for j in range(nots+1,nots+6):
        #print(str(i) +' '+ str(j))
        file= "./att_faces/s" + str(i) + "/" +  str(j) + ".pgm"
        im = cv2.imread(file)
        im = im.transpose((2,0,1))
        im = np.expand_dims(im,axis=0)
        imgray = im[0][0]
        im1D = imgray.flatten('F')
        X_test = np.vstack((X_test,im1D))
        y_test = np.hstack((y_test,i-1))

#delete first row as it was empty
X_train = np.delete(X_train,(0),axis=0)
y_train = np.delete(y_train,(0),axis=0)
X_test = np.delete(X_test,(0),axis=0)
y_test = np.delete(y_test,(0),axis=0)

print('loaded')

print(X_train.shape[0])
X_train = X_train.reshape(X_train.shape[0], 1, 112, 92).astype('float32')
X_test = X_test.reshape(X_test.shape[0], 1, 112, 92).astype('float32')

print(X_train.shape)

#normalize to 0-1
X_train = X_train/255
X_test = X_test/255

#one-hot encode outputs
y_train = np_utils.to_categorical(y_train)
y_test = np_utils.to_categorical(y_test)
num_classes = y_train.shape[1]

print(num_classes)

#build the model
model = baseline_model()

#Fit the model
history1 = model.fit(X_train,y_train, validation_data=(X_test,y_test), nb_epoch=50, batch_size=8, verbose=1)

print('Prediction on saved sample:')
#print(str(model.predict(X_test)))
out = model.predict(X_test[:1])
print(np.argmax(out))
#print(y_test[:200])

#Code ends here

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
import multiprocessing
import os
import time
import unittest

import pytest
from mock import patch

from airflow import AirflowException, models, settings
from airflow.executors.sequential_executor import SequentialExecutor
from airflow.jobs import LocalTaskJob
from airflow.models import DAG, TaskInstance as TI
from airflow.operators.dummy_operator import DummyOperator
from airflow.utils import timezone
from airflow.utils.net import get_hostname
from airflow.utils.session import create_session
from airflow.utils.state import State
from tests.test_utils.db import clear_db_runs
from tests.test_utils.mock_executor import MockExecutor

DEFAULT_DATE = timezone.datetime(2016, 1, 1)
TEST_DAG_FOLDER = os.environ['AIRFLOW__CORE__DAGS_FOLDER']

class TestLocalTaskJob(unittest.TestCase):
    def setUp(self):
        clear_db_runs()
        patcher = patch('airflow.jobs.base_job.sleep')
        self.addCleanup(patcher.stop)
        self.mock_base_job_sleep = patcher.start()

    def test_localtaskjob_essential_attr(self):
        """
        Check whether essential attributes
        of LocalTaskJob can be assigned with
        proper values without intervention
        """
        dag = DAG(
            'test_localtaskjob_essential_attr',
            start_date=DEFAULT_DATE,
            default_args={'owner': 'owner1'})

        with dag:
            op1 = DummyOperator(task_id='op1')

        dag.clear()
        dr = dag.create_dagrun(run_id="test",
                               state=State.SUCCESS,
                               execution_date=DEFAULT_DATE,
                               start_date=DEFAULT_DATE)
        ti = dr.get_task_instance(task_id=op1.task_id)

        job1 = LocalTaskJob(task_instance=ti,
                            ignore_ti_state=True,
                            executor=SequentialExecutor())

        essential_attr = ["dag_id", "job_type", "start_date", "hostname"]

        check_result_1 = [hasattr(job1, attr) for attr in essential_attr]
        self.assertTrue(all(check_result_1))

        check_result_2 = [getattr(job1, attr) is not None for attr in essential_attr]
        self.assertTrue(all(check_result_2))

    @patch('os.getpid')
    def test_localtaskjob_heartbeat(self, mock_pid):
        session = settings.Session()
        dag = DAG(
            'test_localtaskjob_heartbeat',
            start_date=DEFAULT_DATE,
            default_args={'owner': 'owner1'})

        with dag:
            op1 = DummyOperator(task_id='op1')

        dag.clear()
        dr = dag.create_dagrun(run_id="test",
                               state=State.SUCCESS,
                               execution_date=DEFAULT_DATE,
                               start_date=DEFAULT_DATE,
                               session=session)
        ti = dr.get_task_instance(task_id=op1.task_id, session=session)
        ti.state = State.RUNNING
        ti.hostname = "blablabla"
        session.commit()

        job1 = LocalTaskJob(task_instance=ti,
                            ignore_ti_state=True,
                            executor=SequentialExecutor())
        self.assertRaises(AirflowException, job1.heartbeat_callback)

        mock_pid.return_value = 1
        ti.state = State.RUNNING
        ti.hostname = get_hostname()
        ti.pid = 1
        session.merge(ti)
        session.commit()

        job1.heartbeat_callback(session=None)

        mock_pid.return_value = 2
        self.assertRaises(AirflowException, job1.heartbeat_callback)

    @patch('os.getpid')
    def test_heartbeat_failed_fast(self, mock_getpid):
        """
        Test that task heartbeat will sleep when it fails fast
        """
        mock_getpid.return_value = 1
        self.mock_base_job_sleep.side_effect = time.sleep

        with create_session() as session:
            dagbag = models.DagBag(
                dag_folder=TEST_DAG_FOLDER,
                include_examples=False,
            )
            dag_id = 'test_heartbeat_failed_fast'
            task_id = 'test_heartbeat_failed_fast_op'
            dag = dagbag.get_dag(dag_id)
            task = dag.get_task(task_id)

            dag.create_dagrun(run_id="test_heartbeat_failed_fast_run",
                              state=State.RUNNING,
                              execution_date=DEFAULT_DATE,
                              start_date=DEFAULT_DATE,
                              session=session)
            ti = TI(task=task, execution_date=DEFAULT_DATE)
            ti.refresh_from_db()
            ti.state = State.RUNNING
            ti.hostname = get_hostname()
            ti.pid = 1
            session.commit()

            job = LocalTaskJob(task_instance=ti, executor=MockExecutor(do_update=False))
            job.heartrate = 2
            heartbeat_records = []
            job.heartbeat_callback = lambda session: heartbeat_records.append(job.latest_heartbeat)
            job._execute()
            self.assertGreater(len(heartbeat_records), 2)
            for i in range(1, len(heartbeat_records)):
                time1 = heartbeat_records[i - 1]
                time2 = heartbeat_records[i]
                # Assert that difference small enough to avoid:
                # AssertionError: 1.996401 not greater than or equal to 2
                delta = (time2 - time1).total_seconds()
                self.assertAlmostEqual(delta, job.heartrate, delta=0.006)

    @pytest.mark.xfail(condition=True, reason="This test might be flaky in postgres/mysql")
    def test_mark_success_no_kill(self):
        """
        Test that ensures that mark_success in the UI doesn't cause
        the task to fail, and that the task exits
        """
        dagbag = models.DagBag(
            dag_folder=TEST_DAG_FOLDER,
            include_examples=False,
        )
        dag = dagbag.dags.get('test_mark_success')
        task = dag.get_task('task1')

        session = settings.Session()

        dag.clear()
        dag.create_dagrun(run_id="test",
                          state=State.RUNNING,
                          execution_date=DEFAULT_DATE,
                          start_date=DEFAULT_DATE,
                          session=session)
        ti = TI(task=task, execution_date=DEFAULT_DATE)
        ti.refresh_from_db()
        job1 = LocalTaskJob(task_instance=ti, ignore_ti_state=True)
        process = multiprocessing.Process(target=job1.run)
        process.start()
        ti.refresh_from_db()
        for _ in range(0, 50):
            if ti.state == State.RUNNING:
                break
            time.sleep(0.1)
            ti.refresh_from_db()
        self.assertEqual(State.RUNNING, ti.state)
        ti.state = State.SUCCESS
        session.merge(ti)
        session.commit()

        process.join(timeout=10)
        self.assertFalse(process.is_alive())
        ti.refresh_from_db()
        self.assertEqual(State.SUCCESS, ti.state)

    def test_localtaskjob_double_trigger(self):
        dagbag = models.DagBag(
            dag_folder=TEST_DAG_FOLDER,
            include_examples=False,
        )
        dag = dagbag.dags.get('test_localtaskjob_double_trigger')
        task = dag.get_task('test_localtaskjob_double_trigger_task')

        session = settings.Session()

        dag.clear()
        dr = dag.create_dagrun(run_id="test",
                               state=State.SUCCESS,
                               execution_date=DEFAULT_DATE,
                               start_date=DEFAULT_DATE,
                               session=session)
        ti = dr.get_task_instance(task_id=task.task_id, session=session)
        ti.state = State.RUNNING
        ti.hostname = get_hostname()
        ti.pid = 1
        session.merge(ti)
        session.commit()

        ti_run = TI(task=task, execution_date=DEFAULT_DATE)
        ti_run.refresh_from_db()
        job1 = LocalTaskJob(task_instance=ti_run,
                            executor=SequentialExecutor())
        from airflow.task.task_runner.standard_task_runner import StandardTaskRunner
        with patch.object(StandardTaskRunner, 'start', return_value=None) as mock_method:
            job1.run()
            mock_method.assert_not_called()

        ti = dr.get_task_instance(task_id=task.task_id, session=session)
        self.assertEqual(ti.pid, 1)
        self.assertEqual(ti.state, State.RUNNING)

        session.close()

    def test_localtaskjob_maintain_heart_rate(self):
        dagbag = models.DagBag(
            dag_folder=TEST_DAG_FOLDER,
            include_examples=False,
        )
        dag = dagbag.dags.get('test_localtaskjob_double_trigger')
        task = dag.get_task('test_localtaskjob_double_trigger_task')

        session = settings.Session()

        dag.clear()
        dag.create_dagrun(run_id="test",
                          state=State.SUCCESS,
                          execution_date=DEFAULT_DATE,
                          start_date=DEFAULT_DATE,
                          session=session)

        ti_run = TI(task=task, execution_date=DEFAULT_DATE)
        ti_run.refresh_from_db()
        job1 = LocalTaskJob(task_instance=ti_run,
                            executor=SequentialExecutor())

        # this should make sure we only heartbeat once and exit at the second
        # loop in _execute()
        return_codes = [None, 0]

        def multi_return_code():
            return return_codes.pop(0)

        time_start = time.time()
        from airflow.task.task_runner.standard_task_runner import StandardTaskRunner
        with patch.object(StandardTaskRunner, 'start', return_value=None) as mock_start:
            with patch.object(StandardTaskRunner, 'return_code') as mock_ret_code:
                mock_ret_code.side_effect = multi_return_code
                job1.run()
                self.assertEqual(mock_start.call_count, 1)
                self.assertEqual(mock_ret_code.call_count, 2)
        time_end = time.time()

        self.assertEqual(self.mock_base_job_sleep.call_count, 1)
        self.assertEqual(job1.state, State.SUCCESS)

        # Consider we have patched sleep call, it should not be sleeping to
        # keep up with the heart rate in other unpatched places
        #
        # We already make sure patched sleep call is only called once
        self.assertLess(time_end - time_start, job1.heartrate)
        session.close()

    def test_mark_failure_on_failure_callback(self):
        """
        Test that ensures that mark_failure in the UI fails
        the task, and executes on_failure_callback
        """
        data = {'called': False}

        def check_failure(context):
            self.assertEqual(context['dag_run'].dag_id,
                             'test_mark_failure')
            data['called'] = True

        dag = DAG(dag_id='test_mark_failure',
                  start_date=DEFAULT_DATE,
                  default_args={'owner': 'owner1'})

        task = DummyOperator(
            task_id='test_state_succeeded1',
            dag=dag,
            on_failure_callback=check_failure)

        session = settings.Session()

        dag.clear()
        dag.create_dagrun(run_id="test",
                          state=State.RUNNING,
                          execution_date=DEFAULT_DATE,
                          start_date=DEFAULT_DATE,
                          session=session)
        ti = TI(task=task, execution_date=DEFAULT_DATE)
        ti.refresh_from_db()
        job1 = LocalTaskJob(task_instance=ti,
                            ignore_ti_state=True,
                            executor=SequentialExecutor())
        from airflow.task.task_runner.standard_task_runner import StandardTaskRunner
        job1.task_runner = StandardTaskRunner(job1)
        process = multiprocessing.Process(target=job1.run)
        process.start()
        ti.refresh_from_db()
        for _ in range(0, 50):
            if ti.state == State.RUNNING:
                break
            time.sleep(0.1)
            ti.refresh_from_db()
        self.assertEqual(State.RUNNING, ti.state)
        ti.state = State.FAILED
        session.merge(ti)
        session.commit()

        job1.heartbeat_callback(session=None)
        self.assertTrue(data['called'])
        process.join(timeout=10)
        self.assertFalse(process.is_alive())
<commit_msg>[AIRFLOW-6642] Make local task job test less flaky (#7262)<commit_after># -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
import multiprocessing
import os
import time
import unittest

import pytest
from mock import patch

from airflow import AirflowException, models, settings
from airflow.executors.sequential_executor import SequentialExecutor
from airflow.jobs import LocalTaskJob
from airflow.models import DAG, TaskInstance as TI
from airflow.operators.dummy_operator import DummyOperator
from airflow.utils import timezone
from airflow.utils.net import get_hostname
from airflow.utils.session import create_session
from airflow.utils.state import State
from tests.test_utils.db import clear_db_runs
from tests.test_utils.mock_executor import MockExecutor

DEFAULT_DATE = timezone.datetime(2016, 1, 1)
TEST_DAG_FOLDER = os.environ['AIRFLOW__CORE__DAGS_FOLDER']

class TestLocalTaskJob(unittest.TestCase):
    def setUp(self):
        clear_db_runs()
        patcher = patch('airflow.jobs.base_job.sleep')
        self.addCleanup(patcher.stop)
        self.mock_base_job_sleep = patcher.start()

    def test_localtaskjob_essential_attr(self):
        """
        Check whether essential attributes
        of LocalTaskJob can be assigned with
        proper values without intervention
        """
        dag = DAG(
            'test_localtaskjob_essential_attr',
            start_date=DEFAULT_DATE,
            default_args={'owner': 'owner1'})

        with dag:
            op1 = DummyOperator(task_id='op1')

        dag.clear()
        dr = dag.create_dagrun(run_id="test",
                               state=State.SUCCESS,
                               execution_date=DEFAULT_DATE,
                               start_date=DEFAULT_DATE)
        ti = dr.get_task_instance(task_id=op1.task_id)

        job1 = LocalTaskJob(task_instance=ti,
                            ignore_ti_state=True,
                            executor=SequentialExecutor())

        essential_attr = ["dag_id", "job_type", "start_date", "hostname"]

        check_result_1 = [hasattr(job1, attr) for attr in essential_attr]
        self.assertTrue(all(check_result_1))

        check_result_2 = [getattr(job1, attr) is not None for attr in essential_attr]
        self.assertTrue(all(check_result_2))

    @patch('os.getpid')
    def test_localtaskjob_heartbeat(self, mock_pid):
        session = settings.Session()
        dag = DAG(
            'test_localtaskjob_heartbeat',
            start_date=DEFAULT_DATE,
            default_args={'owner': 'owner1'})

        with dag:
            op1 = DummyOperator(task_id='op1')

        dag.clear()
        dr = dag.create_dagrun(run_id="test",
                               state=State.SUCCESS,
                               execution_date=DEFAULT_DATE,
                               start_date=DEFAULT_DATE,
                               session=session)
        ti = dr.get_task_instance(task_id=op1.task_id, session=session)
        ti.state = State.RUNNING
        ti.hostname = "blablabla"
        session.commit()

        job1 = LocalTaskJob(task_instance=ti,
                            ignore_ti_state=True,
                            executor=SequentialExecutor())
        self.assertRaises(AirflowException, job1.heartbeat_callback)

        mock_pid.return_value = 1
        ti.state = State.RUNNING
        ti.hostname = get_hostname()
        ti.pid = 1
        session.merge(ti)
        session.commit()

        job1.heartbeat_callback(session=None)

        mock_pid.return_value = 2
        self.assertRaises(AirflowException, job1.heartbeat_callback)

    @patch('os.getpid')
    def test_heartbeat_failed_fast(self, mock_getpid):
        """
        Test that task heartbeat will sleep when it fails fast
        """
        mock_getpid.return_value = 1
        self.mock_base_job_sleep.side_effect = time.sleep

        with create_session() as session:
            dagbag = models.DagBag(
                dag_folder=TEST_DAG_FOLDER,
                include_examples=False,
            )
            dag_id = 'test_heartbeat_failed_fast'
            task_id = 'test_heartbeat_failed_fast_op'
            dag = dagbag.get_dag(dag_id)
            task = dag.get_task(task_id)

            dag.create_dagrun(run_id="test_heartbeat_failed_fast_run",
                              state=State.RUNNING,
                              execution_date=DEFAULT_DATE,
                              start_date=DEFAULT_DATE,
                              session=session)
            ti = TI(task=task, execution_date=DEFAULT_DATE)
            ti.refresh_from_db()
            ti.state = State.RUNNING
            ti.hostname = get_hostname()
            ti.pid = 1
            session.commit()

            job = LocalTaskJob(task_instance=ti, executor=MockExecutor(do_update=False))
            job.heartrate = 2
            heartbeat_records = []
            job.heartbeat_callback = lambda session: heartbeat_records.append(job.latest_heartbeat)
            job._execute()
            self.assertGreater(len(heartbeat_records), 2)
            for i in range(1, len(heartbeat_records)):
                time1 = heartbeat_records[i - 1]
                time2 = heartbeat_records[i]
                # Assert that difference small enough
                delta = (time2 - time1).total_seconds()
                self.assertAlmostEqual(delta, job.heartrate, delta=0.05)

    @pytest.mark.xfail(condition=True, reason="This test might be flaky in postgres/mysql")
    def test_mark_success_no_kill(self):
        """
        Test that ensures that mark_success in the UI doesn't cause
        the task to fail, and that the task exits
        """
        dagbag = models.DagBag(
            dag_folder=TEST_DAG_FOLDER,
            include_examples=False,
        )
        dag = dagbag.dags.get('test_mark_success')
        task = dag.get_task('task1')

        session = settings.Session()

        dag.clear()
        dag.create_dagrun(run_id="test",
                          state=State.RUNNING,
                          execution_date=DEFAULT_DATE,
                          start_date=DEFAULT_DATE,
                          session=session)
        ti = TI(task=task, execution_date=DEFAULT_DATE)
        ti.refresh_from_db()
        job1 = LocalTaskJob(task_instance=ti, ignore_ti_state=True)
        process = multiprocessing.Process(target=job1.run)
        process.start()
        ti.refresh_from_db()
        for _ in range(0, 50):
            if ti.state == State.RUNNING:
                break
            time.sleep(0.1)
            ti.refresh_from_db()
        self.assertEqual(State.RUNNING, ti.state)
        ti.state = State.SUCCESS
        session.merge(ti)
        session.commit()

        process.join(timeout=10)
        self.assertFalse(process.is_alive())
        ti.refresh_from_db()
        self.assertEqual(State.SUCCESS, ti.state)

    def test_localtaskjob_double_trigger(self):
        dagbag = models.DagBag(
            dag_folder=TEST_DAG_FOLDER,
            include_examples=False,
        )
        dag = dagbag.dags.get('test_localtaskjob_double_trigger')
        task = dag.get_task('test_localtaskjob_double_trigger_task')

        session = settings.Session()

        dag.clear()
        dr = dag.create_dagrun(run_id="test",
                               state=State.SUCCESS,
                               execution_date=DEFAULT_DATE,
                               start_date=DEFAULT_DATE,
                               session=session)
        ti = dr.get_task_instance(task_id=task.task_id, session=session)
        ti.state = State.RUNNING
        ti.hostname = get_hostname()
        ti.pid = 1
        session.merge(ti)
        session.commit()

        ti_run = TI(task=task, execution_date=DEFAULT_DATE)
        ti_run.refresh_from_db()
        job1 = LocalTaskJob(task_instance=ti_run,
                            executor=SequentialExecutor())
        from airflow.task.task_runner.standard_task_runner import StandardTaskRunner
        with patch.object(StandardTaskRunner, 'start', return_value=None) as mock_method:
            job1.run()
            mock_method.assert_not_called()

        ti = dr.get_task_instance(task_id=task.task_id, session=session)
        self.assertEqual(ti.pid, 1)
        self.assertEqual(ti.state, State.RUNNING)

        session.close()

    def test_localtaskjob_maintain_heart_rate(self):
        dagbag = models.DagBag(
            dag_folder=TEST_DAG_FOLDER,
            include_examples=False,
        )
        dag = dagbag.dags.get('test_localtaskjob_double_trigger')
        task = dag.get_task('test_localtaskjob_double_trigger_task')

        session = settings.Session()

        dag.clear()
        dag.create_dagrun(run_id="test",
                          state=State.SUCCESS,
                          execution_date=DEFAULT_DATE,
                          start_date=DEFAULT_DATE,
                          session=session)

        ti_run = TI(task=task, execution_date=DEFAULT_DATE)
        ti_run.refresh_from_db()
        job1 = LocalTaskJob(task_instance=ti_run,
                            executor=SequentialExecutor())

        # this should make sure we only heartbeat once and exit at the second
        # loop in _execute()
        return_codes = [None, 0]

        def multi_return_code():
            return return_codes.pop(0)

        time_start = time.time()
        from airflow.task.task_runner.standard_task_runner import StandardTaskRunner
        with patch.object(StandardTaskRunner, 'start', return_value=None) as mock_start:
            with patch.object(StandardTaskRunner, 'return_code') as mock_ret_code:
                mock_ret_code.side_effect = multi_return_code
                job1.run()
                self.assertEqual(mock_start.call_count, 1)
                self.assertEqual(mock_ret_code.call_count, 2)
        time_end = time.time()

        self.assertEqual(self.mock_base_job_sleep.call_count, 1)
        self.assertEqual(job1.state, State.SUCCESS)

        # Consider we have patched sleep call, it should not be sleeping to
        # keep up with the heart rate in other unpatched places
        #
        # We already make sure patched sleep call is only called once
        self.assertLess(time_end - time_start, job1.heartrate)
        session.close()

    def test_mark_failure_on_failure_callback(self):
        """
        Test that ensures that mark_failure in the UI fails
        the task, and executes on_failure_callback
        """
        data = {'called': False}

        def check_failure(context):
            self.assertEqual(context['dag_run'].dag_id,
                             'test_mark_failure')
            data['called'] = True

        dag = DAG(dag_id='test_mark_failure',
                  start_date=DEFAULT_DATE,
                  default_args={'owner': 'owner1'})

        task = DummyOperator(
            task_id='test_state_succeeded1',
            dag=dag,
            on_failure_callback=check_failure)

        session = settings.Session()

        dag.clear()
        dag.create_dagrun(run_id="test",
                          state=State.RUNNING,
                          execution_date=DEFAULT_DATE,
                          start_date=DEFAULT_DATE,
                          session=session)
        ti = TI(task=task, execution_date=DEFAULT_DATE)
        ti.refresh_from_db()
        job1 = LocalTaskJob(task_instance=ti,
                            ignore_ti_state=True,
                            executor=SequentialExecutor())
        from airflow.task.task_runner.standard_task_runner import StandardTaskRunner
        job1.task_runner = StandardTaskRunner(job1)
        process = multiprocessing.Process(target=job1.run)
        process.start()
        ti.refresh_from_db()
        for _ in range(0, 50):
            if ti.state == State.RUNNING:
                break
            time.sleep(0.1)
            ti.refresh_from_db()
        self.assertEqual(State.RUNNING, ti.state)
        ti.state = State.FAILED
        session.merge(ti)
        session.commit()

        job1.heartbeat_callback(session=None)
        self.assertTrue(data['called'])
        process.join(timeout=10)
        self.assertFalse(process.is_alive())
<|endoftext|>