<commit_before># Copyright Anne M. Archibald 2008
# Released under the scipy license
from numpy.testing import *

import numpy as np
from scipy.spatial import KDTree, distance, Rectangle

class ConsistencyTests:
    def test_nearest(self):
        x = self.x
        d, i = self.kdtree.query(x, 1)
        assert_almost_equal(d**2,np.sum((x-self.data[i])**2))
        eps = 1e-8
        assert np.all(np.sum((self.data-x[np.newaxis,:])**2,axis=1)>d**2-eps)

    def test_m_nearest(self):
        x = self.x
        m = self.m
        dd, ii = self.kdtree.query(x, m)
        d = np.amax(dd)
        i = ii[np.argmax(dd)]
        assert_almost_equal(d**2,np.sum((x-self.data[i])**2))
        eps = 1e-8
        assert_equal(np.sum(np.sum((self.data-x[np.newaxis,:])**2,axis=1)<d**2+eps),m)

    def test_points_near(self):
        x = self.x
        d = self.d
        dd, ii = self.kdtree.query(x, k=self.kdtree.n, distance_upper_bound=d)
        eps = 1e-8
        hits = 0
        for near_d, near_i in zip(dd,ii):
            if near_d==np.inf:
                continue
            hits += 1
            assert_almost_equal(near_d**2,np.sum((x-self.data[near_i])**2))
            assert near_d<d+eps, "near_d=%g should be less than %g" % (near_d,d)
        assert_equal(np.sum(np.sum((self.data-x[np.newaxis,:])**2,axis=1)<d**2+eps),hits)

    def test_points_near_l1(self):
        x = self.x
        d = self.d
        dd, ii = self.kdtree.query(x, k=self.kdtree.n, p=1, distance_upper_bound=d)
        eps = 1e-8
        hits = 0
        for near_d, near_i in zip(dd,ii):
            if near_d==np.inf:
                continue
            hits += 1
            assert_almost_equal(near_d,distance(x,self.data[near_i],1))
            assert near_d<d+eps, "near_d=%g should be less than %g" % (near_d,d)
        assert_equal(np.sum(distance(self.data,x,1)<d+eps),hits)
    def test_points_near_linf(self):
        x = self.x
        d = self.d
        dd, ii = self.kdtree.query(x, k=self.kdtree.n, p=np.inf, distance_upper_bound=d)
        eps = 1e-8
        hits = 0
        for near_d, near_i in zip(dd,ii):
            if near_d==np.inf:
                continue
            hits += 1
            assert_almost_equal(near_d,distance(x,self.data[near_i],np.inf))
            assert near_d<d+eps, "near_d=%g should be less than %g" % (near_d,d)
        assert_equal(np.sum(distance(self.data,x,np.inf)<d+eps),hits)

    def test_approx(self):
        x = self.x
        m = self.m
        eps = 0.1
        d_real, i_real = self.kdtree.query(x, m)
        d, i = self.kdtree.query(x, m, eps=eps)
        assert np.all(d<=d_real*(1+eps))

class test_random(ConsistencyTests):
    def setUp(self):
        self.n = 1000
        self.k = 4
        self.data = np.random.randn(self.n, self.k)
        self.kdtree = KDTree(self.data)
        self.x = np.random.randn(self.k)
        self.d = 0.2
        self.m = 10

class test_small(ConsistencyTests):
    def setUp(self):
        self.data = np.array([[0,0,0],
                              [0,0,1],
                              [0,1,0],
                              [0,1,1],
                              [1,0,0],
                              [1,0,1],
                              [1,1,0],
                              [1,1,1]])
        self.kdtree = KDTree(self.data)
        self.n = self.kdtree.n
        self.k = self.kdtree.k
        self.x = np.random.randn(3)
        self.d = 0.5
        self.m = 4

    def test_nearest(self):
        assert_array_equal(
                self.kdtree.query((0,0,0.1), 1),
                (0.1,0))
    def test_nearest_two(self):
        assert_array_equal(
                self.kdtree.query((0,0,0.1), 2),
                ([0.1,0.9],[0,1]))
class test_small_nonleaf(test_small):
    def setUp(self):
        self.data = np.array([[0,0,0],
                              [0,0,1],
                              [0,1,0],
                              [0,1,1],
                              [1,0,0],
                              [1,0,1],
                              [1,1,0],
                              [1,1,1]])
        self.kdtree = KDTree(self.data,leafsize=1)
        self.n = self.kdtree.n
        self.k = self.kdtree.k
        self.x = np.random.randn(3)
        self.d = 0.5
        self.m = 4

class test_vectorization:
    def setUp(self):
        self.data = np.array([[0,0,0],
                              [0,0,1],
                              [0,1,0],
                              [0,1,1],
                              [1,0,0],
                              [1,0,1],
                              [1,1,0],
                              [1,1,1]])
        self.kdtree = KDTree(self.data)

    def test_single_query(self):
        d, i = self.kdtree.query([0,0,0])
        assert isinstance(d,float)
        assert isinstance(i,int)

    def test_vectorized_query(self):
        d, i = self.kdtree.query(np.zeros((2,4,3)))
        assert_equal(np.shape(d),(2,4))
        assert_equal(np.shape(i),(2,4))

    def test_single_query_multiple_neighbors(self):
        s = 23
        kk = self.kdtree.n+s
        d, i = self.kdtree.query([0,0,0],k=kk)
        assert_equal(np.shape(d),(kk,))
        assert_equal(np.shape(i),(kk,))
        assert np.all(~np.isfinite(d[-s:]))
        assert np.all(i[-s:]==self.kdtree.n)
    def test_vectorized_query_multiple_neighbors(self):
        s = 23
        kk = self.kdtree.n+s
        d, i = self.kdtree.query(np.zeros((2,4,3)),k=kk)
        assert_equal(np.shape(d),(2,4,kk))
        assert_equal(np.shape(i),(2,4,kk))
        assert np.all(~np.isfinite(d[:,:,-s:]))
        assert np.all(i[:,:,-s:]==self.kdtree.n)
    def test_single_query_all_neighbors(self):
        d, i = self.kdtree.query([0,0,0],k=None,distance_upper_bound=1.1)
        assert isinstance(d,list)
        assert isinstance(i,list)
    def test_vectorized_query_all_neighbors(self):
        d, i = self.kdtree.query(np.zeros((2,4,3)),k=None,distance_upper_bound=1.1)
        assert_equal(np.shape(d),(2,4))
        assert_equal(np.shape(i),(2,4))

        assert isinstance(d[0,0],list)
        assert isinstance(i[0,0],list)

class ball_consistency:

    def test_in_ball(self):
        l = self.T.query_ball_point(self.x, self.d, p=self.p, eps=self.eps)
        for i in l:
            assert distance(self.data[i],self.x,self.p)<=self.d*(1.+self.eps)

    def test_found_all(self):
        c = np.ones(self.T.n,dtype=np.bool)
        l = self.T.query_ball_point(self.x, self.d, p=self.p, eps=self.eps)
        c[l] = False
        assert np.all(distance(self.data[c],self.x,self.p)>=self.d/(1.+self.eps))

class test_random_ball(ball_consistency):

    def setUp(self):
        n = 1000
        k = 4
        self.data = np.random.randn(n,k)
        self.T = KDTree(self.data)
        self.x = np.random.randn(k)
        self.p = 2.
        self.eps = 0
        self.d = 0.2

class test_random_ball_approx(test_random_ball):

    def setUp(self):
        test_random_ball.setUp(self)
        self.eps = 0.1

class test_random_ball_far(test_random_ball):

    def setUp(self):
        test_random_ball.setUp(self)
        self.d = 2.

class test_random_ball_l1(test_random_ball):

    def setUp(self):
        test_random_ball.setUp(self)
        self.p = 1

class test_random_ball_linf(test_random_ball):

    def setUp(self):
        test_random_ball.setUp(self)
        self.p = np.inf

def test_random_ball_vectorized():

    n = 20
    k = 5
    T = KDTree(np.random.randn(n,k))

    r = T.query_ball_point(np.random.randn(2,3,k),1)
    assert_equal(r.shape,(2,3))
    assert isinstance(r[0,0],list)

class two_trees_consistency:

    def test_all_in_ball(self):
        r = self.T1.query_ball_tree(self.T2, self.d, p=self.p, eps=self.eps)
        for i, l in enumerate(r):
            for j in l:
                assert distance(self.data1[i],self.data2[j],self.p)<=self.d*(1.+self.eps)
    def test_found_all(self):
        r = self.T1.query_ball_tree(self.T2, self.d, p=self.p, eps=self.eps)
        for i, l in enumerate(r):
            c = np.ones(self.T2.n,dtype=np.bool)
            c[l] = False
            assert np.all(distance(self.data2[c],self.data1[i],self.p)>=self.d/(1.+self.eps))

class test_two_random_trees(two_trees_consistency):

    def setUp(self):
        n = 100
        k = 4
        self.data1 = np.random.randn(n,k)
        self.T1 = KDTree(self.data1,leafsize=2)
        self.data2 = np.random.randn(n,k)
        self.T2 = KDTree(self.data2,leafsize=2)
        self.p = 2.
        self.eps = 0
        self.d = 0.2

class test_two_random_trees_far(test_two_random_trees):

    def setUp(self):
        test_two_random_trees.setUp(self)
        self.d = 2

class test_two_random_trees_linf(test_two_random_trees):

    def setUp(self):
        test_two_random_trees.setUp(self)
        self.p = np.inf

class test_rectangle:

    def setUp(self):
        self.rect = Rectangle([0,0],[1,1])

    def test_min_inside(self):
        assert_almost_equal(self.rect.min_distance_point([0.5,0.5]),0)
    def test_min_one_side(self):
        assert_almost_equal(self.rect.min_distance_point([0.5,1.5]),0.5)
    def test_min_two_sides(self):
        assert_almost_equal(self.rect.min_distance_point([2,2]),np.sqrt(2))
    def test_max_inside(self):
        assert_almost_equal(self.rect.max_distance_point([0.5,0.5]),1/np.sqrt(2))
    def test_max_one_side(self):
        assert_almost_equal(self.rect.max_distance_point([0.5,1.5]),np.hypot(0.5,1.5))
    def test_max_two_sides(self):
        assert_almost_equal(self.rect.max_distance_point([2,2]),2*np.sqrt(2))

    def test_split(self):
        less, greater = self.rect.split(0,0.1)
        assert_array_equal(less.maxes,[0.1,1])
        assert_array_equal(less.mins,[0,0])
        assert_array_equal(greater.maxes,[1,1])
        assert_array_equal(greater.mins,[0.1,0])

def test_distance_l2():
    assert_almost_equal(distance([0,0],[1,1],2),np.sqrt(2))
def test_distance_l1():
    assert_almost_equal(distance([0,0],[1,1],1),2)
def test_distance_linf():
    assert_almost_equal(distance([0,0],[1,1],np.inf),1)
def test_distance_vectorization():
    x = np.random.randn(10,1,3)
    y = np.random.randn(1,7,3)
    assert_equal(distance(x,y).shape,(10,7))

class test_count_neighbors:

    def setUp(self):
        n = 100
        k = 4
        self.T1 = KDTree(np.random.randn(n,k),leafsize=2)
        self.T2 = KDTree(np.random.randn(n,k),leafsize=2)

    def test_one_radius(self):
        r = 0.2
        assert_equal(self.T1.count_neighbors(self.T2, r),
                np.sum([len(l) for l in self.T1.query_ball_tree(self.T2,r)]))

    def test_large_radius(self):
        r = 1000
        assert_equal(self.T1.count_neighbors(self.T2, r),
                np.sum([len(l) for l in self.T1.query_ball_tree(self.T2,r)]))

    def test_multiple_radius(self):
        rs = np.exp(np.linspace(np.log(0.01),np.log(10),10))
        results = self.T1.count_neighbors(self.T2, rs)
        assert np.all(np.diff(results)>=0)
        for r,result in zip(rs, results):
            assert_equal(self.T1.count_neighbors(self.T2, r), result)
<commit_msg>Added boilerplate to allow running all tests in this module.<commit_after># Copyright Anne M. Archibald 2008
# Released under the scipy license
from numpy.testing import *

import numpy as np
from scipy.spatial import KDTree, distance, Rectangle

class ConsistencyTests:
    def test_nearest(self):
        x = self.x
        d, i = self.kdtree.query(x, 1)
        assert_almost_equal(d**2,np.sum((x-self.data[i])**2))
        eps = 1e-8
        assert np.all(np.sum((self.data-x[np.newaxis,:])**2,axis=1)>d**2-eps)

    def test_m_nearest(self):
        x = self.x
        m = self.m
        dd, ii = self.kdtree.query(x, m)
        d = np.amax(dd)
        i = ii[np.argmax(dd)]
        assert_almost_equal(d**2,np.sum((x-self.data[i])**2))
        eps = 1e-8
        assert_equal(np.sum(np.sum((self.data-x[np.newaxis,:])**2,axis=1)<d**2+eps),m)

    def test_points_near(self):
        x = self.x
        d = self.d
        dd, ii = self.kdtree.query(x, k=self.kdtree.n, distance_upper_bound=d)
        eps = 1e-8
        hits = 0
        for near_d, near_i in zip(dd,ii):
            if near_d==np.inf:
                continue
            hits += 1
            assert_almost_equal(near_d**2,np.sum((x-self.data[near_i])**2))
            assert near_d<d+eps, "near_d=%g should be less than %g" % (near_d,d)
        assert_equal(np.sum(np.sum((self.data-x[np.newaxis,:])**2,axis=1)<d**2+eps),hits)

    def test_points_near_l1(self):
        x = self.x
        d = self.d
        dd, ii = self.kdtree.query(x, k=self.kdtree.n, p=1, distance_upper_bound=d)
        eps = 1e-8
        hits = 0
        for near_d, near_i in zip(dd,ii):
            if near_d==np.inf:
                continue
            hits += 1
            assert_almost_equal(near_d,distance(x,self.data[near_i],1))
            assert near_d<d+eps, "near_d=%g should be less than %g" % (near_d,d)
        assert_equal(np.sum(distance(self.data,x,1)<d+eps),hits)
    def test_points_near_linf(self):
        x = self.x
        d = self.d
        dd, ii = self.kdtree.query(x, k=self.kdtree.n, p=np.inf, distance_upper_bound=d)
        eps = 1e-8
        hits = 0
        for near_d, near_i in zip(dd,ii):
            if near_d==np.inf:
                continue
            hits += 1
            assert_almost_equal(near_d,distance(x,self.data[near_i],np.inf))
            assert near_d<d+eps, "near_d=%g should be less than %g" % (near_d,d)
        assert_equal(np.sum(distance(self.data,x,np.inf)<d+eps),hits)

    def test_approx(self):
        x = self.x
        m = self.m
        eps = 0.1
        d_real, i_real = self.kdtree.query(x, m)
        d, i = self.kdtree.query(x, m, eps=eps)
        assert np.all(d<=d_real*(1+eps))

class test_random(ConsistencyTests):
    def setUp(self):
        self.n = 1000
        self.k = 4
        self.data = np.random.randn(self.n, self.k)
        self.kdtree = KDTree(self.data)
        self.x = np.random.randn(self.k)
        self.d = 0.2
        self.m = 10

class test_small(ConsistencyTests):
    def setUp(self):
        self.data = np.array([[0,0,0],
                              [0,0,1],
                              [0,1,0],
                              [0,1,1],
                              [1,0,0],
                              [1,0,1],
                              [1,1,0],
                              [1,1,1]])
        self.kdtree = KDTree(self.data)
        self.n = self.kdtree.n
        self.k = self.kdtree.k
        self.x = np.random.randn(3)
        self.d = 0.5
        self.m = 4

    def test_nearest(self):
        assert_array_equal(
                self.kdtree.query((0,0,0.1), 1),
                (0.1,0))
    def test_nearest_two(self):
        assert_array_equal(
                self.kdtree.query((0,0,0.1), 2),
                ([0.1,0.9],[0,1]))
class test_small_nonleaf(test_small):
    def setUp(self):
        self.data = np.array([[0,0,0],
                              [0,0,1],
                              [0,1,0],
                              [0,1,1],
                              [1,0,0],
                              [1,0,1],
                              [1,1,0],
                              [1,1,1]])
        self.kdtree = KDTree(self.data,leafsize=1)
        self.n = self.kdtree.n
        self.k = self.kdtree.k
        self.x = np.random.randn(3)
        self.d = 0.5
        self.m = 4

class test_vectorization:
    def setUp(self):
        self.data = np.array([[0,0,0],
                              [0,0,1],
                              [0,1,0],
                              [0,1,1],
                              [1,0,0],
                              [1,0,1],
                              [1,1,0],
                              [1,1,1]])
        self.kdtree = KDTree(self.data)

    def test_single_query(self):
        d, i = self.kdtree.query([0,0,0])
        assert isinstance(d,float)
        assert isinstance(i,int)

    def test_vectorized_query(self):
        d, i = self.kdtree.query(np.zeros((2,4,3)))
        assert_equal(np.shape(d),(2,4))
        assert_equal(np.shape(i),(2,4))

    def test_single_query_multiple_neighbors(self):
        s = 23
        kk = self.kdtree.n+s
        d, i = self.kdtree.query([0,0,0],k=kk)
        assert_equal(np.shape(d),(kk,))
        assert_equal(np.shape(i),(kk,))
        assert np.all(~np.isfinite(d[-s:]))
        assert np.all(i[-s:]==self.kdtree.n)
    def test_vectorized_query_multiple_neighbors(self):
        s = 23
        kk = self.kdtree.n+s
        d, i = self.kdtree.query(np.zeros((2,4,3)),k=kk)
        assert_equal(np.shape(d),(2,4,kk))
        assert_equal(np.shape(i),(2,4,kk))
        assert np.all(~np.isfinite(d[:,:,-s:]))
        assert np.all(i[:,:,-s:]==self.kdtree.n)
    def test_single_query_all_neighbors(self):
        d, i = self.kdtree.query([0,0,0],k=None,distance_upper_bound=1.1)
        assert isinstance(d,list)
        assert isinstance(i,list)
    def test_vectorized_query_all_neighbors(self):
        d, i = self.kdtree.query(np.zeros((2,4,3)),k=None,distance_upper_bound=1.1)
        assert_equal(np.shape(d),(2,4))
        assert_equal(np.shape(i),(2,4))

        assert isinstance(d[0,0],list)
        assert isinstance(i[0,0],list)

class ball_consistency:

    def test_in_ball(self):
        l = self.T.query_ball_point(self.x, self.d, p=self.p, eps=self.eps)
        for i in l:
            assert distance(self.data[i],self.x,self.p)<=self.d*(1.+self.eps)

    def test_found_all(self):
        c = np.ones(self.T.n,dtype=np.bool)
        l = self.T.query_ball_point(self.x, self.d, p=self.p, eps=self.eps)
        c[l] = False
        assert np.all(distance(self.data[c],self.x,self.p)>=self.d/(1.+self.eps))

class test_random_ball(ball_consistency):

    def setUp(self):
        n = 1000
        k = 4
        self.data = np.random.randn(n,k)
        self.T = KDTree(self.data)
        self.x = np.random.randn(k)
        self.p = 2.
        self.eps = 0
        self.d = 0.2

class test_random_ball_approx(test_random_ball):

    def setUp(self):
        test_random_ball.setUp(self)
        self.eps = 0.1

class test_random_ball_far(test_random_ball):

    def setUp(self):
        test_random_ball.setUp(self)
        self.d = 2.

class test_random_ball_l1(test_random_ball):

    def setUp(self):
        test_random_ball.setUp(self)
        self.p = 1

class test_random_ball_linf(test_random_ball):

    def setUp(self):
        test_random_ball.setUp(self)
        self.p = np.inf

def test_random_ball_vectorized():

    n = 20
    k = 5
    T = KDTree(np.random.randn(n,k))

    r = T.query_ball_point(np.random.randn(2,3,k),1)
    assert_equal(r.shape,(2,3))
    assert isinstance(r[0,0],list)

class two_trees_consistency:

    def test_all_in_ball(self):
        r = self.T1.query_ball_tree(self.T2, self.d, p=self.p, eps=self.eps)
        for i, l in enumerate(r):
            for j in l:
                assert distance(self.data1[i],self.data2[j],self.p)<=self.d*(1.+self.eps)
    def test_found_all(self):
        r = self.T1.query_ball_tree(self.T2, self.d, p=self.p, eps=self.eps)
        for i, l in enumerate(r):
            c = np.ones(self.T2.n,dtype=np.bool)
            c[l] = False
            assert np.all(distance(self.data2[c],self.data1[i],self.p)>=self.d/(1.+self.eps))

class test_two_random_trees(two_trees_consistency):

    def setUp(self):
        n = 100
        k = 4
        self.data1 = np.random.randn(n,k)
        self.T1 = KDTree(self.data1,leafsize=2)
        self.data2 = np.random.randn(n,k)
        self.T2 = KDTree(self.data2,leafsize=2)
        self.p = 2.
        self.eps = 0
        self.d = 0.2

class test_two_random_trees_far(test_two_random_trees):

    def setUp(self):
        test_two_random_trees.setUp(self)
        self.d = 2

class test_two_random_trees_linf(test_two_random_trees):

    def setUp(self):
        test_two_random_trees.setUp(self)
        self.p = np.inf

class test_rectangle:

    def setUp(self):
        self.rect = Rectangle([0,0],[1,1])

    def test_min_inside(self):
        assert_almost_equal(self.rect.min_distance_point([0.5,0.5]),0)
    def test_min_one_side(self):
        assert_almost_equal(self.rect.min_distance_point([0.5,1.5]),0.5)
    def test_min_two_sides(self):
        assert_almost_equal(self.rect.min_distance_point([2,2]),np.sqrt(2))
    def test_max_inside(self):
        assert_almost_equal(self.rect.max_distance_point([0.5,0.5]),1/np.sqrt(2))
    def test_max_one_side(self):
        assert_almost_equal(self.rect.max_distance_point([0.5,1.5]),np.hypot(0.5,1.5))
    def test_max_two_sides(self):
        assert_almost_equal(self.rect.max_distance_point([2,2]),2*np.sqrt(2))

    def test_split(self):
        less, greater = self.rect.split(0,0.1)
        assert_array_equal(less.maxes,[0.1,1])
        assert_array_equal(less.mins,[0,0])
        assert_array_equal(greater.maxes,[1,1])
        assert_array_equal(greater.mins,[0.1,0])

def test_distance_l2():
    assert_almost_equal(distance([0,0],[1,1],2),np.sqrt(2))
def test_distance_l1():
    assert_almost_equal(distance([0,0],[1,1],1),2)
def test_distance_linf():
    assert_almost_equal(distance([0,0],[1,1],np.inf),1)
def test_distance_vectorization():
    x = np.random.randn(10,1,3)
    y = np.random.randn(1,7,3)
    assert_equal(distance(x,y).shape,(10,7))

class test_count_neighbors:

    def setUp(self):
        n = 100
        k = 4
        self.T1 = KDTree(np.random.randn(n,k),leafsize=2)
        self.T2 = KDTree(np.random.randn(n,k),leafsize=2)

    def test_one_radius(self):
        r = 0.2
        assert_equal(self.T1.count_neighbors(self.T2, r),
                np.sum([len(l) for l in self.T1.query_ball_tree(self.T2,r)]))

    def test_large_radius(self):
        r = 1000
        assert_equal(self.T1.count_neighbors(self.T2, r),
                np.sum([len(l) for l in self.T1.query_ball_tree(self.T2,r)]))

    def test_multiple_radius(self):
        rs = np.exp(np.linspace(np.log(0.01),np.log(10),10))
        results = self.T1.count_neighbors(self.T2, rs)
        assert np.all(np.diff(results)>=0)
        for r,result in zip(rs, results):
            assert_equal(self.T1.count_neighbors(self.T2, r), result)

if __name__=="__main__":
    run_module_suite()
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, unicode_literals

from sqlalchemy.sql import func

from changes.api.base import APIView
from changes.constants import Status
from changes.config import db, redis
from changes.models import Job, JobPlan, JobStep

class JobStepAllocateAPIView(APIView):
    def find_next_jobstep(self):
        # find projects with pending allocations
        project_list = [p for p, in db.session.query(
            Job.project_id,
        ).filter(
            Job.status == Status.pending_allocation,
        ).group_by(
            Job.project_id
        )]
        if not project_list:
            return None

        # TODO(dcramer): this should be configurably and handle more cases
        # than just 'active job' as that can be 1 step or 100 steps
        # find the total number of job steps in progress per project
        # hard limit of 10 active jobs per project
        unavail_projects = [
            p for p, c in
            db.session.query(
                Job.project_id, func.count(Job.project_id),
            ).filter(
                Job.status.in_([Status.allocated, Status.in_progress]),
                Job.project_id.in_(project_list),
            ).group_by(
                Job.project_id,
            )
            if c >= 10
        ]

        filters = [
            JobStep.status == Status.pending_allocation,
        ]
        if unavail_projects:
            filters.append(~JobStep.project_id.in_(unavail_projects))

        # prioritize a job that's has already started
        existing = JobStep.query.join(
            'job',
        ).filter(
            Job.status.in_([Status.allocated, Status.in_progress]),
            *filters
        ).order_by(JobStep.date_created.desc()).first()
        if existing:
            return existing

        # now allow any prioritized project, based on order
        return JobStep.query.filter(
            *filters
        ).order_by(JobStep.date_created.desc()).first()

    def post(self):
        with redis.lock('jobstep:allocate'):
            to_allocate = self.find_next_jobstep()

            # Should 204, but flask/werkzeug throws StopIteration (bug!) for tests
            if to_allocate is None:
                return self.respond([])

            to_allocate.status = Status.allocated
            db.session.add(to_allocate)
            db.session.flush()

        jobplan, buildstep = JobPlan.get_build_step_for_job(to_allocate.job_id)

        assert jobplan and buildstep

        context = self.serialize(to_allocate)
        context['project'] = self.serialize(to_allocate.project)
        context['resources'] = {
            'cpus': 4,
            'mem': 8 * 1024,
        }
        context['cmd'] = buildstep.get_allocation_command(to_allocate)

        return self.respond([context])
<commit_msg>Various tweaks to queue logic<commit_after>from __future__ import absolute_import, division, unicode_literals

from sqlalchemy.sql import func

from changes.api.base import APIView
from changes.constants import Status
from changes.config import db, redis
from changes.models import Job, JobPlan, JobStep

class JobStepAllocateAPIView(APIView):
    def find_next_jobstep(self):
        # find projects with pending allocations
        project_list = [p for p, in db.session.query(
            Job.project_id,
        ).filter(
            Job.status == Status.pending_allocation,
        ).group_by(
            Job.project_id
        )]
        if not project_list:
            return None

        # TODO(dcramer): this should be configurably and handle more cases
        # than just 'active job' as that can be 1 step or 100 steps
        # find the total number of job steps in progress per project
        # hard limit of 10 active jobs per project
        unavail_projects = [
            p for p, c in
            db.session.query(
                Job.project_id, func.count(Job.project_id),
            ).filter(
                Job.status.in_([Status.allocated, Status.in_progress]),
                Job.project_id.in_(project_list),
            ).group_by(
                Job.project_id,
            )
            if c >= 10
        ]

        filters = [
            JobStep.status == Status.pending_allocation,
        ]
        if unavail_projects:
            filters.append(~JobStep.project_id.in_(unavail_projects))

        # prioritize a job that's has already started
        existing = JobStep.query.join(
            'job',
        ).filter(
            Job.status.in_([Status.allocated, Status.in_progress]),
            *filters
        ).order_by(JobStep.date_created.asc()).first()
        if existing:
            return existing

        # now allow any prioritized project, based on order
        existing = JobStep.query.filter(
            *filters
        ).order_by(JobStep.date_created.asc()).first()
        if existing:
            return existing

        # TODO(dcramer): we want to burst but not go too far. For now just
        # let burst
        return JobStep.query.filter(
            JobStep.status == Status.pending_allocation,
        ).order_by(JobStep.date_created.asc()).first()

    def post(self):
        with redis.lock('jobstep:allocate'):
            to_allocate = self.find_next_jobstep()

            # Should 204, but flask/werkzeug throws StopIteration (bug!) for tests
            if to_allocate is None:
                return self.respond([])

            to_allocate.status = Status.allocated
            db.session.add(to_allocate)
            db.session.flush()

        jobplan, buildstep = JobPlan.get_build_step_for_job(to_allocate.job_id)

        assert jobplan and buildstep

        context = self.serialize(to_allocate)
        context['project'] = self.serialize(to_allocate.project)
        context['resources'] = {
            'cpus': 4,
            'mem': 8 * 1024,
        }
        context['cmd'] = buildstep.get_allocation_command(to_allocate)

        return self.respond([context])
<|endoftext|>


<commit_before>"""
building properties algorithm
"""

# HISTORY:
# J. A. Fonseca  script development          22.03.15

from __future__ import division
from __future__ import absolute_import

import numpy as np
import pandas as pd
from cea.utilities.dbfreader import dbf2df, df2dbf
import cea.inputlocator

__author__ = "Jimeno A. Fonseca"
__copyright__ = "Copyright 2015, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Jimeno A. Fonseca", "Daren Thomas", "Martin Mosteiro"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

def properties(locator, prop_architecture_flag, prop_hvac_flag, prop_comfort_flag, prop_internal_loads_flag):
    """
    algorithm to query building properties from statistical database
    Archetypes_HVAC_properties.csv. for more info check the integrated demand
    model of Fonseca et al. 2015. Appl. energy.

    :param InputLocator locator: an InputLocator instance set to the scenario to work on
    :param boolean prop_architecture_flag: if True, get properties about the construction and architecture.
    :param boolean prop_comfort_flag: if True, get properties about thermal comfort.
    :param boolean prop_hvac_flag: if True, get properties about types of HVAC systems, otherwise False.
    :param boolean prop_internal_loads_flag: if True, get properties about internal loads, otherwise False.

    The following files are created by this script, depending on which flags were set:

    - building_HVAC: .dbf
        describes the queried properties of HVAC systems.

    - architecture.dbf
        describes the queried properties of architectural features

    - building_thermal: .shp
        describes the queried thermal properties of buildings

    - indoor_comfort.shp
        describes the queried thermal properties of buildings
    """

    # get occupancy and age files
    building_occupancy_df = dbf2df(locator.get_building_occupancy())
    list_uses = list(building_occupancy_df.drop(['PFloor', 'Name'], axis=1).columns)  # parking excluded in U-Values
    building_age_df = dbf2df(locator.get_building_age())

    # get occupant densities from archetypes schedules
    occupant_densities = {}
    for use in list_uses:
        archetypes_schedules = pd.read_excel(locator.get_archetypes_schedules(), use).T
        area_per_occupant = archetypes_schedules['density'].values[:1][0]
        if area_per_occupant > 0:
            occupant_densities[use] = 1 / area_per_occupant
        else:
            occupant_densities[use] = 0

    # prepare shapefile to store results (a shapefile with only names of buildings
    names_df = building_age_df[['Name']]

    # define main use:
    building_occupancy_df['mainuse'] = calc_mainuse(building_occupancy_df, list_uses)

    # dataframe with jonned data for categories
    categories_df = building_occupancy_df.merge(building_age_df, on='Name')

    # get properties about the construction and architecture
    if prop_architecture_flag:
        architecture_DB = get_database(locator.get_archetypes_properties(), 'ARCHITECTURE')
        architecture_DB['Code'] = architecture_DB.apply(lambda x: calc_code(x['building_use'], x['year_start'],
                                                                            x['year_end'], x['standard']), axis=1)
        categories_df['cat_architecture'] = calc_category(architecture_DB, categories_df)

        prop_architecture_df = categories_df.merge(architecture_DB, left_on='cat_architecture', right_on='Code')

        # adjust 'Hs' for multiuse buildings
        prop_architecture_df['Hs'] = correct_archetype_areas(prop_architecture_df, architecture_DB, list_uses)

        # write to shapefile
        prop_architecture_df_merged = names_df.merge(prop_architecture_df, on="Name")
        fields = ['Name', 'Hs', 'win_wall', 'type_cons', 'type_leak', 'type_roof', 'type_wall', 'type_win',
                  'type_shade']
        df2dbf(prop_architecture_df_merged[fields], locator.get_building_architecture())

    # get properties about types of HVAC systems
    if prop_hvac_flag:
        HVAC_DB = get_database(locator.get_archetypes_properties(), 'HVAC')
        HVAC_DB['Code'] = HVAC_DB.apply(lambda x: calc_code(x['building_use'], x['year_start'],
                                                            x['year_end'], x['standard']), axis=1)

        categories_df['cat_HVAC'] = calc_category(HVAC_DB, categories_df)

        # define HVAC systems types
        prop_HVAC_df = categories_df.merge(HVAC_DB, left_on='cat_HVAC', right_on='Code')

        # write to shapefile
        prop_HVAC_df_merged = names_df.merge(prop_HVAC_df, on="Name")
        fields = ['Name', 'type_cs', 'type_hs', 'type_dhw', 'type_ctrl', 'type_vent']
        df2dbf(prop_HVAC_df_merged[fields], locator.get_building_hvac())

    if prop_comfort_flag:
        comfort_DB = get_database(locator.get_archetypes_properties(), 'INDOOR_COMFORT')

        # define comfort
        prop_comfort_df = categories_df.merge(comfort_DB, left_on='mainuse', right_on='Code')

        # write to shapefile
        prop_comfort_df_merged = names_df.merge(prop_comfort_df, on="Name")
        prop_comfort_df_merged = calculate_average_multiuse(prop_comfort_df_merged, occupant_densities, list_uses,
                                                            comfort_DB)
        fields = ['Name', 'Tcs_set_C', 'Ths_set_C', 'Tcs_setb_C', 'Ths_setb_C', 'Ve_lps']
        df2dbf(prop_comfort_df_merged[fields], locator.get_building_comfort())

    if prop_internal_loads_flag:
        internal_DB = get_database(locator.get_archetypes_properties(), 'INTERNAL_LOADS')

        # define comfort
        prop_internal_df = categories_df.merge(internal_DB, left_on='mainuse', right_on='Code')

        # write to shapefile
        prop_internal_df_merged = names_df.merge(prop_internal_df, on="Name")
        prop_internal_df_merged = calculate_average_multiuse(prop_internal_df_merged, occupant_densities, list_uses,
                                                             internal_DB)
        fields = ['Name', 'Qs_Wp', 'X_ghp', 'Ea_Wm2', 'El_Wm2', 'Epro_Wm2', 'Ere_Wm2', 'Ed_Wm2', 'Vww_lpd', 'Vw_lpd']
        df2dbf(prop_internal_df_merged[fields], locator.get_building_internal())

def calc_code(code1, code2, code3, code4):
    return str(code1) + str(code2) + str(code3) + str(code4)

def calc_mainuse(uses_df, uses):
    databaseclean = uses_df[uses].transpose()
    array_max = np.array(databaseclean[databaseclean[:] > 0].idxmax(skipna=True), dtype='S10')
    for i in range(len(array_max)):
        if databaseclean[i][array_max[i]] != 1:
            databaseclean[i][array_max[i]] = 0
    array_second = np.array(databaseclean[databaseclean[:] > 0].idxmax(skipna=True), dtype='S10')
    mainuse = np.array(map(calc_comparison, array_second, array_max))

    return mainuse

def get_database(path_database, sheet):
    database = pd.read_excel(path_database, sheet)
    return database

def calc_comparison(array_second, array_max):
    if array_max == 'PARKING':
        if array_second != 'PARKING':
            array_max = array_second
    return array_max

def calc_category(archetype_DB, age):
    category = []
    for row in age.index:
        if age.loc[row, 'envelope'] > age.loc[row, 'built']:
            category.append(archetype_DB[(archetype_DB['year_start'] <= age.loc[row, 'envelope']) & \
                                         (archetype_DB['year_end'] >= age.loc[row, 'envelope']) & \
                                         (archetype_DB['building_use'] == age.loc[row, 'mainuse']) & \
                                         (archetype_DB['standard'] == 'R')].Code.values[0])
        else:
            category.append(archetype_DB[(archetype_DB['year_start'] <= age.loc[row, 'built']) & \
                                         (archetype_DB['year_end'] >= age.loc[row, 'built']) & \
                                         (archetype_DB['building_use'] == age.loc[row, 'mainuse']) & \
                                         (archetype_DB['standard'] == 'C')].Code.values[0])
        if 0 < age.loc[row, 'envelope'] < age.loc[row, 'built']:
            print 'Incorrect renovation year in building ' + age['Name'][row] + \
                  ': renovation year is lower than building age'
        if age.loc[row, 'envelope'] == age.loc[row, 'built']:
            print 'Incorrect renovation year in building ' + age['Name'][
                row] + ': if building is not renovated, the year needs to be set to 0'
    return category

def correct_archetype_areas(prop_architecture_df, architecture_DB, list_uses):
    """
        Corrects the heated area 'Hs' for buildings with multiple uses.

         :var prop_architecture_df: DataFrame containing each building's occupancy, construction and renovation data as
         well as the architectural properties obtained from the archetypes.
         :type prop_architecture_df: DataFrame
         :var architecture_DB: architecture database for each archetype
         :type architecture_DB: DataFrame
         :var list_uses: list of all occupancy types in the project
         :type list_uses: list[str]

         :return Hs_list: the corrected values for 'Hs' for each building
         :type Hs_list: list[float]
    """

    indexed_DB = architecture_DB.set_index('Code')

    # weighted average of values
    def calc_average(last, current, share_of_use):
        return last + current * share_of_use

    Hs_list = []

    for building in prop_architecture_df.index:
        Hs = 0
        for use in list_uses:
            # if the use is present in the building, find the building archetype properties for that use
            if prop_architecture_df[use][building] > 0:
                # get archetype code for the current occupancy type
                current_use_code = use + str(prop_architecture_df['year_start'][building]) + \
                                   str(prop_architecture_df['year_end'][building]) + \
                                   str(prop_architecture_df['standard'][building])
                # recalculate heated floor area as an average of the archetype value for each occupancy type in the
                # building
                Hs = calc_average(Hs, indexed_DB['Hs'][current_use_code], prop_architecture_df[use][building])
        Hs_list.append(Hs)

    return Hs_list

def calculate_average_multiuse(properties_df, occupant_densities, list_uses, properties_DB):
    '''
    This script calculates the average internal loads and ventilation properties for multiuse buildings.

    :param properties_df: DataFrame containing the building's occupancy type and the corresponding indoor comfort
    properties or internal loads.
    :type properties_df: DataFrame
    :param occupant_densities: DataFrame containing the number of people per square meter for each occupancy type based
    on the archetypes
    :type occupant_densities: Dict
    :param list_uses: list of uses in the project
    :type list_uses: list[str]
    :param properties_DB: DataFrame containing each occupancy type's indoor comfort properties or internal loads based
    on the corresponding archetypes
    :type properties_DB: DataFrame

    :return properties_df: the same DataFrame as the input parameter, but with the updated properties for multiuse
    buildings
    '''

    indexed_DB = properties_DB.set_index('Code')

    for column in properties_df.columns:
        if column in ['Ve_lps', 'Qs_Wp', 'X_ghp', 'Vww_lpd', 'Vw_lpd']:
            # some properties are imported from the Excel files as int instead of float
            properties_df[column] = properties_df[column].astype(float)
            for building in properties_df.index:
                column_total = 0
                people_total = 0
                for use in list_uses:
                    if use in properties_df.columns:
                        column_total += properties_df[use][building] * occupant_densities[use] * indexed_DB[column][use]
                        people_total += properties_df[use][building] * occupant_densities[use]
                if people_total > 0:
                    properties_df.loc[building, column] = column_total / people_total
                else:
                    properties_df.loc[building, column] = 0

        elif column in ['Ea_Wm2', 'El_Wm2', 'Epro_Wm2', 'Ere_Wm2', 'Ed_Wm2']:
            for building in properties_df.index:
                average = 0
                for use in list_uses:
                    average += properties_df[use][building] * indexed_DB[column][use]
                properties_df.loc[building, column] = average

    return properties_df

def run_as_script(scenario_path=None, prop_thermal_flag=True, prop_architecture_flag=True, prop_hvac_flag=True,
                  prop_comfort_flag=True, prop_internal_loads_flag=True):
    """
    Run the properties script with input from the reference case and compare the results. This ensures that changes
    made to this script (e.g. refactorings) do not stop the script from working and also that the results stay the same.
    """
    import cea.globalvar
    gv = cea.globalvar.GlobalVariables()
    if not scenario_path:
        scenario_path = r'C:\reference-case-open1\baseline2'  # gv.scenario_reference
    locator = cea.inputlocator.InputLocator(scenario_path=scenario_path)
    properties(locator=locator, prop_architecture_flag=prop_architecture_flag,
               prop_hvac_flag=prop_hvac_flag, prop_comfort_flag=prop_comfort_flag,
               prop_internal_loads_flag=prop_internal_loads_flag)

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--scenario', help='Path to the scenario folder')
    args = parser.parse_args()

    run_as_script(scenario_path=args.scenario)
<commit_msg>Removing hard-coded reference case<commit_after>"""
building properties algorithm
"""

# HISTORY:
# J. A. Fonseca  script development          22.03.15

from __future__ import division
from __future__ import absolute_import

import numpy as np
import pandas as pd
from cea.utilities.dbfreader import dbf2df, df2dbf
import cea.inputlocator

__author__ = "Jimeno A. Fonseca"
__copyright__ = "Copyright 2015, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Jimeno A. Fonseca", "Daren Thomas", "Martin Mosteiro"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

def properties(locator, prop_architecture_flag, prop_hvac_flag, prop_comfort_flag, prop_internal_loads_flag):
    """
    algorithm to query building properties from statistical database
    Archetypes_HVAC_properties.csv. for more info check the integrated demand
    model of Fonseca et al. 2015. Appl. energy.

    :param InputLocator locator: an InputLocator instance set to the scenario to work on
    :param boolean prop_architecture_flag: if True, get properties about the construction and architecture.
    :param boolean prop_comfort_flag: if True, get properties about thermal comfort.
    :param boolean prop_hvac_flag: if True, get properties about types of HVAC systems, otherwise False.
    :param boolean prop_internal_loads_flag: if True, get properties about internal loads, otherwise False.

    The following files are created by this script, depending on which flags were set:

    - building_HVAC: .dbf
        describes the queried properties of HVAC systems.

    - architecture.dbf
        describes the queried properties of architectural features

    - building_thermal: .shp
        describes the queried thermal properties of buildings

    - indoor_comfort.shp
        describes the queried thermal properties of buildings
    """

    # get occupancy and age files
    building_occupancy_df = dbf2df(locator.get_building_occupancy())
    list_uses = list(building_occupancy_df.drop(['PFloor', 'Name'], axis=1).columns)  # parking excluded in U-Values
    building_age_df = dbf2df(locator.get_building_age())

    # get occupant densities from archetypes schedules
    occupant_densities = {}
    for use in list_uses:
        archetypes_schedules = pd.read_excel(locator.get_archetypes_schedules(), use).T
        area_per_occupant = archetypes_schedules['density'].values[:1][0]
        if area_per_occupant > 0:
            occupant_densities[use] = 1 / area_per_occupant
        else:
            occupant_densities[use] = 0

    # prepare shapefile to store results (a shapefile with only names of buildings
    names_df = building_age_df[['Name']]

    # define main use:
    building_occupancy_df['mainuse'] = calc_mainuse(building_occupancy_df, list_uses)

    # dataframe with jonned data for categories
    categories_df = building_occupancy_df.merge(building_age_df, on='Name')

    # get properties about the construction and architecture
    if prop_architecture_flag:
        architecture_DB = get_database(locator.get_archetypes_properties(), 'ARCHITECTURE')
        architecture_DB['Code'] = architecture_DB.apply(lambda x: calc_code(x['building_use'], x['year_start'],
                                                                            x['year_end'], x['standard']), axis=1)
        categories_df['cat_architecture'] = calc_category(architecture_DB, categories_df)

        prop_architecture_df = categories_df.merge(architecture_DB, left_on='cat_architecture', right_on='Code')

        # adjust 'Hs' for multiuse buildings
        prop_architecture_df['Hs'] = correct_archetype_areas(prop_architecture_df, architecture_DB, list_uses)

        # write to shapefile
        prop_architecture_df_merged = names_df.merge(prop_architecture_df, on="Name")
        fields = ['Name', 'Hs', 'win_wall', 'type_cons', 'type_leak', 'type_roof', 'type_wall', 'type_win',
                  'type_shade']
        df2dbf(prop_architecture_df_merged[fields], locator.get_building_architecture())

    # get properties about types of HVAC systems
    if prop_hvac_flag:
        HVAC_DB = get_database(locator.get_archetypes_properties(), 'HVAC')
        HVAC_DB['Code'] = HVAC_DB.apply(lambda x: calc_code(x['building_use'], x['year_start'],
                                                            x['year_end'], x['standard']), axis=1)

        categories_df['cat_HVAC'] = calc_category(HVAC_DB, categories_df)

        # define HVAC systems types
        prop_HVAC_df = categories_df.merge(HVAC_DB, left_on='cat_HVAC', right_on='Code')

        # write to shapefile
        prop_HVAC_df_merged = names_df.merge(prop_HVAC_df, on="Name")
        fields = ['Name', 'type_cs', 'type_hs', 'type_dhw', 'type_ctrl', 'type_vent']
        df2dbf(prop_HVAC_df_merged[fields], locator.get_building_hvac())

    if prop_comfort_flag:
        comfort_DB = get_database(locator.get_archetypes_properties(), 'INDOOR_COMFORT')

        # define comfort
        prop_comfort_df = categories_df.merge(comfort_DB, left_on='mainuse', right_on='Code')

        # write to shapefile
        prop_comfort_df_merged = names_df.merge(prop_comfort_df, on="Name")
        prop_comfort_df_merged = calculate_average_multiuse(prop_comfort_df_merged, occupant_densities, list_uses,
                                                            comfort_DB)
        fields = ['Name', 'Tcs_set_C', 'Ths_set_C', 'Tcs_setb_C', 'Ths_setb_C', 'Ve_lps']
        df2dbf(prop_comfort_df_merged[fields], locator.get_building_comfort())

    if prop_internal_loads_flag:
        internal_DB = get_database(locator.get_archetypes_properties(), 'INTERNAL_LOADS')

        # define comfort
        prop_internal_df = categories_df.merge(internal_DB, left_on='mainuse', right_on='Code')

        # write to shapefile
        prop_internal_df_merged = names_df.merge(prop_internal_df, on="Name")
        prop_internal_df_merged = calculate_average_multiuse(prop_internal_df_merged, occupant_densities, list_uses,
                                                             internal_DB)
        fields = ['Name', 'Qs_Wp', 'X_ghp', 'Ea_Wm2', 'El_Wm2', 'Epro_Wm2', 'Ere_Wm2', 'Ed_Wm2', 'Vww_lpd', 'Vw_lpd']
        df2dbf(prop_internal_df_merged[fields], locator.get_building_internal())

def calc_code(code1, code2, code3, code4):
    return str(code1) + str(code2) + str(code3) + str(code4)

def calc_mainuse(uses_df, uses):
    databaseclean = uses_df[uses].transpose()
    array_max = np.array(databaseclean[databaseclean[:] > 0].idxmax(skipna=True), dtype='S10')
    for i in range(len(array_max)):
        if databaseclean[i][array_max[i]] != 1:
            databaseclean[i][array_max[i]] = 0
    array_second = np.array(databaseclean[databaseclean[:] > 0].idxmax(skipna=True), dtype='S10')
    mainuse = np.array(map(calc_comparison, array_second, array_max))

    return mainuse

def get_database(path_database, sheet):
    database = pd.read_excel(path_database, sheet)
    return database

def calc_comparison(array_second, array_max):
    if array_max == 'PARKING':
        if array_second != 'PARKING':
            array_max = array_second
    return array_max

def calc_category(archetype_DB, age):
    category = []
    for row in age.index:
        if age.loc[row, 'envelope'] > age.loc[row, 'built']:
            category.append(archetype_DB[(archetype_DB['year_start'] <= age.loc[row, 'envelope']) & \
                                         (archetype_DB['year_end'] >= age.loc[row, 'envelope']) & \
                                         (archetype_DB['building_use'] == age.loc[row, 'mainuse']) & \
                                         (archetype_DB['standard'] == 'R')].Code.values[0])
        else:
            category.append(archetype_DB[(archetype_DB['year_start'] <= age.loc[row, 'built']) & \
                                         (archetype_DB['year_end'] >= age.loc[row, 'built']) & \
                                         (archetype_DB['building_use'] == age.loc[row, 'mainuse']) & \
                                         (archetype_DB['standard'] == 'C')].Code.values[0])
        if 0 < age.loc[row, 'envelope'] < age.loc[row, 'built']:
            print 'Incorrect renovation year in building ' + age['Name'][row] + \
                  ': renovation year is lower than building age'
        if age.loc[row, 'envelope'] == age.loc[row, 'built']:
            print 'Incorrect renovation year in building ' + age['Name'][
                row] + ': if building is not renovated, the year needs to be set to 0'
    return category

def correct_archetype_areas(prop_architecture_df, architecture_DB, list_uses):
    """
        Corrects the heated area 'Hs' for buildings with multiple uses.

         :var prop_architecture_df: DataFrame containing each building's occupancy, construction and renovation data as
         well as the architectural properties obtained from the archetypes.
         :type prop_architecture_df: DataFrame
         :var architecture_DB: architecture database for each archetype
         :type architecture_DB: DataFrame
         :var list_uses: list of all occupancy types in the project
         :type list_uses: list[str]

         :return Hs_list: the corrected values for 'Hs' for each building
         :type Hs_list: list[float]
    """

    indexed_DB = architecture_DB.set_index('Code')

    # weighted average of values
    def calc_average(last, current, share_of_use):
        return last + current * share_of_use

    Hs_list = []

    for building in prop_architecture_df.index:
        Hs = 0
        for use in list_uses:
            # if the use is present in the building, find the building archetype properties for that use
            if prop_architecture_df[use][building] > 0:
                # get archetype code for the current occupancy type
                current_use_code = use + str(prop_architecture_df['year_start'][building]) + \
                                   str(prop_architecture_df['year_end'][building]) + \
                                   str(prop_architecture_df['standard'][building])
                # recalculate heated floor area as an average of the archetype value for each occupancy type in the
                # building
                Hs = calc_average(Hs, indexed_DB['Hs'][current_use_code], prop_architecture_df[use][building])
        Hs_list.append(Hs)

    return Hs_list

def calculate_average_multiuse(properties_df, occupant_densities, list_uses, properties_DB):
    '''
    This script calculates the average internal loads and ventilation properties for multiuse buildings.

    :param properties_df: DataFrame containing the building's occupancy type and the corresponding indoor comfort
    properties or internal loads.
    :type properties_df: DataFrame
    :param occupant_densities: DataFrame containing the number of people per square meter for each occupancy type based
    on the archetypes
    :type occupant_densities: Dict
    :param list_uses: list of uses in the project
    :type list_uses: list[str]
    :param properties_DB: DataFrame containing each occupancy type's indoor comfort properties or internal loads based
    on the corresponding archetypes
    :type properties_DB: DataFrame

    :return properties_df: the same DataFrame as the input parameter, but with the updated properties for multiuse
    buildings
    '''

    indexed_DB = properties_DB.set_index('Code')

    for column in properties_df.columns:
        if column in ['Ve_lps', 'Qs_Wp', 'X_ghp', 'Vww_lpd', 'Vw_lpd']:
            # some properties are imported from the Excel files as int instead of float
            properties_df[column] = properties_df[column].astype(float)
            for building in properties_df.index:
                column_total = 0
                people_total = 0
                for use in list_uses:
                    if use in properties_df.columns:
                        column_total += properties_df[use][building] * occupant_densities[use] * indexed_DB[column][use]
                        people_total += properties_df[use][building] * occupant_densities[use]
                if people_total > 0:
                    properties_df.loc[building, column] = column_total / people_total
                else:
                    properties_df.loc[building, column] = 0

        elif column in ['Ea_Wm2', 'El_Wm2', 'Epro_Wm2', 'Ere_Wm2', 'Ed_Wm2']:
            for building in properties_df.index:
                average = 0
                for use in list_uses:
                    average += properties_df[use][building] * indexed_DB[column][use]
                properties_df.loc[building, column] = average

    return properties_df

def run_as_script(scenario_path=None, prop_thermal_flag=True, prop_architecture_flag=True, prop_hvac_flag=True,
                  prop_comfort_flag=True, prop_internal_loads_flag=True):
    """
    Run the properties script with input from the reference case and compare the results. This ensures that changes
    made to this script (e.g. refactorings) do not stop the script from working and also that the results stay the same.
    """
    import cea.globalvar
    gv = cea.globalvar.GlobalVariables()
    if not scenario_path:
        scenario_path = gv.scenario_reference
    locator = cea.inputlocator.InputLocator(scenario_path=scenario_path)
    properties(locator=locator, prop_architecture_flag=prop_architecture_flag,
               prop_hvac_flag=prop_hvac_flag, prop_comfort_flag=prop_comfort_flag,
               prop_internal_loads_flag=prop_internal_loads_flag)

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--scenario', help='Path to the scenario folder')
    args = parser.parse_args()

    run_as_script(scenario_path=args.scenario)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import os
import sys
from io import open
import time
import traceback

from django.conf import settings
from django.contrib.auth.models import Group
from django.core.mail import EmailMultiAlternatives
from django.core.management.base import BaseCommand

from djangotrellostats.apps.fetch.fetchers.trello import BoardFetcher, Initializer
from djangotrellostats.apps.members.models import Member

class Command(BaseCommand):
    help = u'Fetch board data'

    FETCH_LOCK_FILE_PATH = u"/tmp/django-trello-stats-fetch-lock.txt"

    def __init__(self, stdout=None, stderr=None, no_color=False):
        super(Command, self).__init__(stdout, stderr, no_color)
        self.member = None
        self.start_time = None
        self.end_time = None

    def add_arguments(self, parser):
        parser.add_argument('member_trello_username', nargs='+', type=str)

    def start(self):
        self.start_time = time.time()
        # Check if lock file exists. If it exists, thrown an error
        if os.path.isfile(Command.FETCH_LOCK_FILE_PATH):
            self.stdout.write(self.style.ERROR(u"Lock is in place. Unable to fetch"))
            return False

        # Creates a new lock file
        self.stdout.write(self.style.SUCCESS(u"Lock does not exist. Creating..."))
        with open(Command.FETCH_LOCK_FILE_PATH, 'w', encoding="utf-8") as lock_file:
            lock_file.write(u"Fetching data for boards")

        self.stdout.write(self.style.SUCCESS(u"Lock file created"))
        return True

    def end(self):
        self.end_time = time.time()
        # Lock file must exist
        if not os.path.isfile(Command.FETCH_LOCK_FILE_PATH):
            self.stdout.write(self.style.ERROR(u"Lock does not exist. Cancel operation"))
            return False

        # Deleting the lock file
        os.remove(Command.FETCH_LOCK_FILE_PATH)

        self.stdout.write(self.style.SUCCESS(u"Lock file deleted"))

    def elapsed_time(self):
        return self.end_time - self.start_time

    def handle(self, *args, **options):
        try:
            member_trello_username = options['member_trello_username'][0]
        except (IndexError, KeyError)as e:
            self.stdout.write(self.style.ERROR("member_username is mandatory"))
            raise AssertionError("member_username is mandatory")

        self.member = Member.objects.get(trello_username=member_trello_username)
        if not self.member.is_initialized():
            error_message = u"Member {0} is not initialized".format(self.member.trello_username)
            self.stderr.write(self.style.SUCCESS(error_message))
            raise AssertionError(error_message)

        if not self.start():
            self.stdout.write(self.style.ERROR("Unable to fetch"))
            error_message = u"Unable to fetch boards. Is the lock file present?"
            self.warn_administrators(subject=u"Unable to fetch boards",
                                     message=error_message)
            raise AssertionError(error_message)

        fetch_ok = False

        # Make two retries to the fetch procsess
        try:
            self.fetch()
            fetch_ok = True
        # If after two retries the exception persists, warn the administrators
        except Exception as e:
            fetch_ok = False
            self.warn_administrators(subject=u"Unable to fetch boards",
                                     message=u"We tried initializing the boards but it didn't work out. {0}".format(traceback.format_exc()))
        # Always delete the lock file
        finally:
            self.end()

        if fetch_ok:
            self.stdout.write(self.style.SUCCESS(u"All boards fetched successfully {0}".format(self.elapsed_time())))

    # Fetch boards
    def fetch(self):
        last_board = None
        try:
            # For each board that is ready, fetch it
            for board in self.member.created_boards.filter(has_to_be_fetched=True):
                if board.is_ready():
                    last_board = board
                    self.stdout.write(self.style.SUCCESS(u"Board {0} is ready".format(board.name)))
                    self.fetch_board(board)
                    self.stdout.write(self.style.SUCCESS(u"Board {0} fetched successfully".format(board.name)))
                else:
                    self.stdout.write(self.style.ERROR(u"Board {0} is not ready".format(board.name)))

        # If there is any exception, warn the administrators
        except Exception as e:
            if last_board:
                error_message = u"Error when fetching boards. Board {0} fetch failed.".format(last_board.name)
            else:
                error_message = u"Error when fetching boards."
            self.warn_administrators(subject=error_message, message=traceback.format_exc())
            self.stdout.write(self.style.ERROR(error_message))

    # Fetch one board
    def fetch_board(self, board):
        num_retries = 0
        while num_retries < 2:
            try:
                num_retries += 1
                board_fetcher = BoardFetcher(board)
                board_fetcher.fetch(debug=False)
            except Exception as e:
                if num_retries < 2:
                    # Initialize board if needed
                    initializer = Initializer(self.member, debug=False)
                    initializer.init(board.uuid)
                else:
                    raise e

    # Warn administrators of an error
    def warn_administrators(self, subject, message):
        email_subject = u"[DjangoTrelloStats] [Warning] {0}".format(subject)

        administrator_group = Group.objects.get(name=settings.ADMINISTRATOR_GROUP)
        administrator_users = administrator_group.user_set.all()
        for administrator_user in administrator_users:
            email_message = EmailMultiAlternatives(email_subject, message, settings.EMAIL_HOST_USER, [administrator_user.email])
            email_message.send()
<commit_msg>Ignore old locks when fetching data<commit_after>from __future__ import unicode_literals

import os
import sys
from io import open
import time
import traceback

import pytz
from django.conf import settings
from django.contrib.auth.models import Group
from django.core.mail import EmailMultiAlternatives
from django.core.management.base import BaseCommand
from django.utils import timezone

from djangotrellostats.apps.fetch.fetchers.trello import BoardFetcher, Initializer
from djangotrellostats.apps.members.models import Member
from datetime import datetime, timedelta

class Command(BaseCommand):
    help = u'Fetch board data'

    FETCH_LOCK_FILE_PATH = u"/tmp/django-trello-stats-fetch-lock.txt"

    def __init__(self, stdout=None, stderr=None, no_color=False):
        super(Command, self).__init__(stdout, stderr, no_color)
        self.member = None
        self.start_time = None
        self.end_time = None

    def add_arguments(self, parser):
        parser.add_argument('member_trello_username', nargs='+', type=str)

    def start(self):
        self.start_time = time.time()
        # Check if lock file exists. If it exists and has been there for less than 30 minutes, return an error code
        if os.path.isfile(Command.FETCH_LOCK_FILE_PATH):
            # If the file is not old (has been there for less than 30 minutes), return an error code
            if not Command._lock_file_is_old():
                self.stdout.write(self.style.ERROR(u"Lock is in place. Unable to fetch"))
                return False
            os.remove(Command.FETCH_LOCK_FILE_PATH)

        # Creates a new lock file
        self.stdout.write(self.style.SUCCESS(u"Lock does not exist. Creating..."))
        with open(Command.FETCH_LOCK_FILE_PATH, 'w', encoding="utf-8") as lock_file:
            lock_file.write(u"Fetching data for boards")

        self.stdout.write(self.style.SUCCESS(u"Lock file created"))
        return True

    # We consider a lock file is old when has been there 30 minutes or more
    @staticmethod
    def _lock_file_is_old():
        lock_file_modification_datetime = Command._get_lock_file_last_modification_datetime()
        return lock_file_modification_datetime > timezone.now() - timedelta(minutes=30)

    # Gets last modification datetime of the lock file
    @staticmethod
    def _get_lock_file_last_modification_datetime():
        t = os.path.getmtime(Command.FETCH_LOCK_FILE_PATH)
        local_timezone = pytz.timezone(settings.TIME_ZONE)
        return local_timezone.localize(datetime.fromtimestamp(t))

    def end(self):
        self.end_time = time.time()
        # Lock file must exist
        if not os.path.isfile(Command.FETCH_LOCK_FILE_PATH):
            self.stdout.write(self.style.ERROR(u"Lock does not exist. Cancel operation"))
            return False

        # Deleting the lock file
        os.remove(Command.FETCH_LOCK_FILE_PATH)

        self.stdout.write(self.style.SUCCESS(u"Lock file deleted"))

    def elapsed_time(self):
        return self.end_time - self.start_time

    def handle(self, *args, **options):
        try:
            member_trello_username = options['member_trello_username'][0]
        except (IndexError, KeyError)as e:
            self.stdout.write(self.style.ERROR("member_username is mandatory"))
            raise AssertionError("member_username is mandatory")

        self.member = Member.objects.get(trello_username=member_trello_username)
        if not self.member.is_initialized():
            error_message = u"Member {0} is not initialized".format(self.member.trello_username)
            self.stderr.write(self.style.SUCCESS(error_message))
            raise AssertionError(error_message)

        if not self.start():
            self.stdout.write(self.style.ERROR("Unable to fetch"))
            error_message = u"Unable to fetch boards. Is the lock file present?"
            self.warn_administrators(subject=u"Unable to fetch boards",
                                     message=error_message)
            raise AssertionError(error_message)

        fetch_ok = False

        # Make two retries to the fetch procsess
        try:
            self.fetch()
            fetch_ok = True
        # If after two retries the exception persists, warn the administrators
        except Exception as e:
            fetch_ok = False
            self.warn_administrators(subject=u"Unable to fetch boards",
                                     message=u"We tried initializing the boards but it didn't work out. {0}".format(traceback.format_exc()))
        # Always delete the lock file
        finally:
            self.end()

        if fetch_ok:
            self.stdout.write(self.style.SUCCESS(u"All boards fetched successfully {0}".format(self.elapsed_time())))

    # Fetch boards
    def fetch(self):
        last_board = None
        try:
            # For each board that is ready, fetch it
            for board in self.member.created_boards.filter(has_to_be_fetched=True):
                if board.is_ready():
                    last_board = board
                    self.stdout.write(self.style.SUCCESS(u"Board {0} is ready".format(board.name)))
                    self.fetch_board(board)
                    self.stdout.write(self.style.SUCCESS(u"Board {0} fetched successfully".format(board.name)))
                else:
                    self.stdout.write(self.style.ERROR(u"Board {0} is not ready".format(board.name)))

        # If there is any exception, warn the administrators
        except Exception as e:
            if last_board:
                error_message = u"Error when fetching boards. Board {0} fetch failed.".format(last_board.name)
            else:
                error_message = u"Error when fetching boards."
            self.warn_administrators(subject=error_message, message=traceback.format_exc())
            self.stdout.write(self.style.ERROR(error_message))

    # Fetch one board
    def fetch_board(self, board):
        num_retries = 0
        while num_retries < 2:
            try:
                num_retries += 1
                board_fetcher = BoardFetcher(board)
                board_fetcher.fetch(debug=False)
            except Exception as e:
                if num_retries < 2:
                    # Initialize board if needed
                    initializer = Initializer(self.member, debug=False)
                    initializer.init(board.uuid)
                else:
                    raise e

    # Warn administrators of an error
    def warn_administrators(self, subject, message):
        email_subject = u"[DjangoTrelloStats] [Warning] {0}".format(subject)

        administrator_group = Group.objects.get(name=settings.ADMINISTRATOR_GROUP)
        administrator_users = administrator_group.user_set.all()
        for administrator_user in administrator_users:
            email_message = EmailMultiAlternatives(email_subject, message, settings.EMAIL_HOST_USER, [administrator_user.email])
            email_message.send()
<|endoftext|>


<commit_before># Abilitazione/Disattivazione del development environment
DEBUG = True

# Definizione della directory dell'applicazione
import os
BASE_DIR = os.path.abspath(os.path.dirname(__file__))

# Definizione del db

# Application threads. Uso due thread per processore disponibile
# uno per gestire le request in ingresso e uno per i processi
# in background.
THREADS_PER_PAGE = 2

# Abilito protezione contro i *Cross-site Request Forgery (CSRF)*
CSRF_ENABLED = True

# Secret key for signing the data.
CSRF_SESSION_KEY = "secret"

MAPS_API_KEY = ''
CAPTCHA_KEY = ''
<commit_msg>Rimosso config originale<commit_after># Abilitazione/Disattivazione del development environment
DEBUG = True

# Definizione della directory dell'applicazione
import os
BASE_DIR = os.path.abspath(os.path.dirname(__file__))

# Definizione del db

# Application threads. Uso due thread per processore disponibile
# uno per gestire le request in ingresso e uno per i processi
# in background.
THREADS_PER_PAGE = 2

# Abilito protezione contro i *Cross-site Request Forgery (CSRF)*
CSRF_ENABLED = True

# Secret key for signing the data.
CSRF_SESSION_KEY = "secret"

MAPS_API_KEY = 'qui ci va la chiave per google maps'
CAPTCHA_KEY = 'qui ci va la chiave dei recaptcha'
<|endoftext|>


<commit_before>"""
SciPy --- A scientific computing package for Python

Available subpackages
======================
"""

#try:
    #import gui_thread
#except:
    #print "Warning:  wxPython not loaded"

from scipy_version import scipy_version as __version__

# SciPy has levels
# Level 0 -- Numeric and core routines in scipy_base
#
# Level 1 -- Level 0 + fft, special, linalg, stats
# Level 1a -- Core routines which depend on Level 1.

# Level 2 -- plotting interface.
#
# Level 3
# Packages which define own functions plus depend on Levels 0-2.
#
# Level 0, 1, 2 should be imported in order and then other levels imported
#   as available.

import Numeric
import os,sys
import string

# Level 0
# modules to import under the scipy namespace
from scipy_base import *
import Matrix
mat = Matrix.Matrix

from helpmod import *

try:
    id(help)
except NameError:
    help = info

# Level 1
# these modules will just be imported (not subsumed)
import special, io, linalg, stats, fftpack
from fftpack import fft, fftn, fft2, ifft, ifft2, ifftn, fftshift, ifftshift, cont_ft, fftfreq, zeropad
from stats import mean, median, std, cov, corrcoef
from special import isinf, isfinite, isnan

# Functions to be subsumed that need Level 0 and Level 1
from common import *

try:
    from pilutil import *
except ImportError:
    pass

try:
    import fftw
except ImportError:
    pass  # Not default anymore

# Level 2
import optimize, integrate, signal, special, interpolate, cow, ga, cluster, weave

# Level 3
_plot = []
try:
    if sys.platform != 'win32':
        import xplt
        #__all__.append('xplt')
        _plot.append('xplt')
except ImportError:
    pass

try:
#   gplt on win32 messes up focus and takes up 99%
#   of processor -- works fine on *nix.
    if sys.platform != 'win32':
        import gplt
        #__all__.append('gplt')
        _plot.append('gplt')
except ImportError:
    pass

#if _plot == []:
#    print "Warning: No plotting available."
#else:
#    print "Plotting methods available: ", _plot

# removed gplt because it screws up imports on win32
_pkgs = ['cluster','cow','fftpack','fftw','ga','integrate',
         'interpolate', 'io', 'linalg', 'optimize', 'plt',
         'signal', 'sparse', 'special', 'stats', 'weave',
         'xplt'] #,'gplt']

if sys.platform != 'win32':
    _pkgs.append('gplt')

_pkg_doc = ['Vector Quantization / Kmeans',
            'Cluster of Workstations',
            'FFT algorithms',
            'FFT algorithms using fftw',
            'Genetic Algorithms',
            'Plotting with gnuplot',
            'Integration',
            'Interpolation',
            'Input/Output',
            'Linear Algebra',
            'Optimization',
            'Plotting with wxPython',
            'Signal Processing',
            'Sparse Matrices',
            'Special Functions',
            'Statistics',
            'C/C++ integration',
            'Plotting with X']

def _extend_doc(st,p,pd):
    _lengths = [len(x) for x in p]
    _ml = max(_lengths)
    k = 0
    for name in p:
        try:
            exec "import %s" % name
            st += "%s%s --- %s\n" % (name, ' '*(_ml-_lengths[k]), pd[k])
        except ImportError:
            pass
        k += 1
    return st

__doc__ = _extend_doc(__doc__,_pkgs,_pkg_doc)

#---- testing ----#

def test(level=1):
    """ From this top level, there are possibly many many tests.
        Test only the quick tests by default.
    """
    import unittest
    runner = unittest.TextTestRunner()
    runner.run(test_suite(level))
    return runner

def test_all(level=10):
    test(level)

def test_suite(level = 1):
    import scipy_test.testing
    import scipy
    ignore = ['xplt','plt','gplt','gui_thread','sparse','scipy_version']
    suites = [scipy_test.testing.harvest_test_suites(scipy,ignore,level=level)]
    suites += [scipy_test.testing.harvest_test_suites(weave,ignore,level=level)]
    import unittest
    return unittest.TestSuite(suites)<commit_msg>removed plt from _pkgs so that it isn't imported at start up.  This causes problems on Unix machines when DISPLAY isn't set and GTK isn't available.<commit_after>"""
SciPy --- A scientific computing package for Python

Available subpackages
======================
"""

#try:
    #import gui_thread
#except:
    #print "Warning:  wxPython not loaded"

from scipy_version import scipy_version as __version__

# SciPy has levels
# Level 0 -- Numeric and core routines in scipy_base
#
# Level 1 -- Level 0 + fft, special, linalg, stats
# Level 1a -- Core routines which depend on Level 1.

# Level 2 -- plotting interface.
#
# Level 3
# Packages which define own functions plus depend on Levels 0-2.
#
# Level 0, 1, 2 should be imported in order and then other levels imported
#   as available.

import Numeric
import os,sys
import string

# Level 0
# modules to import under the scipy namespace
from scipy_base import *
import Matrix
mat = Matrix.Matrix

from helpmod import *

try:
    id(help)
except NameError:
    help = info

# Level 1
# these modules will just be imported (not subsumed)
import special, io, linalg, stats, fftpack
from fftpack import fft, fftn, fft2, ifft, ifft2, ifftn, fftshift, ifftshift, cont_ft, fftfreq, zeropad
from stats import mean, median, std, cov, corrcoef
from special import isinf, isfinite, isnan

# Functions to be subsumed that need Level 0 and Level 1
from common import *

try:
    from pilutil import *
except ImportError:
    pass

try:
    import fftw
except ImportError:
    pass  # Not default anymore

# Level 2
import optimize, integrate, signal, special, interpolate, cow, ga, cluster, weave

# Level 3
_plot = []
try:
    if sys.platform != 'win32':
        import xplt
        #__all__.append('xplt')
        _plot.append('xplt')
except ImportError:
    pass

try:
#   gplt on win32 messes up focus and takes up 99%
#   of processor -- works fine on *nix.
    if sys.platform != 'win32':
        import gplt
        #__all__.append('gplt')
        _plot.append('gplt')
except ImportError:
    pass

#if _plot == []:
#    print "Warning: No plotting available."
#else:
#    print "Plotting methods available: ", _plot

# removed gplt because it screws up imports on win32
# removed plt cause it screws up unix when gtk isn't available.
_pkgs = ['cluster','cow','fftpack','fftw','ga','integrate',
         'interpolate', 'io', 'linalg', 'optimize',
         'signal', 'sparse', 'special', 'stats', 'weave',
         'xplt'] #,'plt', 'gplt']

if sys.platform != 'win32':
    _pkgs.append('gplt')

_pkg_doc = ['Vector Quantization / Kmeans',
            'Cluster of Workstations',
            'FFT algorithms',
            'FFT algorithms using fftw',
            'Genetic Algorithms',
            'Plotting with gnuplot',
            'Integration',
            'Interpolation',
            'Input/Output',
            'Linear Algebra',
            'Optimization',
            'Plotting with wxPython',
            'Signal Processing',
            'Sparse Matrices',
            'Special Functions',
            'Statistics',
            'C/C++ integration',
            'Plotting with X']

def _extend_doc(st,p,pd):
    _lengths = [len(x) for x in p]
    _ml = max(_lengths)
    k = 0
    for name in p:
        try:
            exec "import %s" % name
            st += "%s%s --- %s\n" % (name, ' '*(_ml-_lengths[k]), pd[k])
        except ImportError:
            pass
        k += 1
    return st

__doc__ = _extend_doc(__doc__,_pkgs,_pkg_doc)

#---- testing ----#

def test(level=1):
    """ From this top level, there are possibly many many tests.
        Test only the quick tests by default.
    """
    import unittest
    runner = unittest.TextTestRunner()
    runner.run(test_suite(level))
    return runner

def test_all(level=10):
    test(level)

def test_suite(level = 1):
    import scipy_test.testing
    import scipy
    ignore = ['xplt','plt','gplt','gui_thread','sparse','scipy_version']
    suites = [scipy_test.testing.harvest_test_suites(scipy,ignore,level=level)]
    suites += [scipy_test.testing.harvest_test_suites(weave,ignore,level=level)]
    import unittest
    return unittest.TestSuite(suites)<|endoftext|>