<commit_before>import unittest
import test.support

import io
import ast
import _ast
import unparse

for_else = """\
def f():
    for x in range(10):
        break
    else:
        y = 2
    z = 3
"""

while_else = """\
def g():
    while True:
        break
    else:
        y = 2
    z = 3
"""

relative_import = """\
from . import fred
from .. import barney
from .australia import shrimp as prawns
"""

nonlocal_ex = """\
def f():
    x = 1
    def g():
        nonlocal x
        x = 2
        y = 7
        def h():
            nonlocal x, y
"""

# also acts as test for 'except ... as ...'
raise_from = """\
try:
    1 / 0
except ZeroDivisionError as e:
    raise ArithmeticError from e
"""

class_decorator = """\
@f1(arg)
@f2
class Foo: pass
"""

class UnparseTestCase(unittest.TestCase):
    # Tests for specific bugs found in earlier versions of unparse

    def check_roundtrip(self, code1, filename="internal"):
        ast1 = compile(code1, filename, "exec", _ast.PyCF_ONLY_AST)
        unparse_buffer = io.StringIO()
        unparse.Unparser(ast1, unparse_buffer)
        code2 = unparse_buffer.getvalue()
        ast2 = compile(code2, filename, "exec", _ast.PyCF_ONLY_AST)
        self.assertEqual(ast.dump(ast1), ast.dump(ast2))

    def test_del_statement(self):
        self.check_roundtrip("del x, y, z")

    def test_shifts(self):
        self.check_roundtrip("45 << 2")
        self.check_roundtrip("13 >> 7")

    def test_for_else(self):
        self.check_roundtrip(for_else)

    def test_while_else(self):
        self.check_roundtrip(while_else)

    def test_unary_parens(self):
        self.check_roundtrip("(-1)**7")
        self.check_roundtrip("not True or False")
        self.check_roundtrip("True or not False")

    def test_integer_parens(self):
        self.check_roundtrip("3 .__abs__()")

    def test_huge_float(self):
        self.check_roundtrip("1e1000")
        self.check_roundtrip("-1e1000")

    def test_lambda_parentheses(self):
        self.check_roundtrip("(lambda: int)()")

    def test_chained_comparisons(self):
        self.check_roundtrip("1 < 4 <= 5")
        self.check_roundtrip("a is b is c is not d")

    def test_function_arguments(self):
        self.check_roundtrip("def f(): pass")
        self.check_roundtrip("def f(a): pass")
        self.check_roundtrip("def f(b = 2): pass")
        self.check_roundtrip("def f(a, b): pass")
        self.check_roundtrip("def f(a, b = 2): pass")
        self.check_roundtrip("def f(a = 5, b = 2): pass")
        self.check_roundtrip("def f(*, a = 1, b = 2): pass")
        self.check_roundtrip("def f(*, a = 1, b): pass")
        self.check_roundtrip("def f(*, a, b = 2): pass")
        self.check_roundtrip("def f(a, b = None, *, c, **kwds): pass")
        self.check_roundtrip("def f(a=2, *args, c=5, d, **kwds): pass")
        self.check_roundtrip("def f(*args, **kwargs): pass")

    def test_relative_import(self):
        self.check_roundtrip(relative_import)

    def test_nonlocal(self):
        self.check_roundtrip(nonlocal_ex)

    def test_raise_from(self):
        self.check_roundtrip(raise_from)

    def test_bytes(self):
        self.check_roundtrip("b'123'")

    def test_annotations(self):
        self.check_roundtrip("def f(a : int): pass")
        self.check_roundtrip("def f(a: int = 5): pass")
        self.check_roundtrip("def f(*args: [int]): pass")
        self.check_roundtrip("def f(**kwargs: dict): pass")
        self.check_roundtrip("def f() -> None: pass")

    def test_set_literal(self):
        self.check_roundtrip("{'a', 'b', 'c'}")

    def test_set_comprehension(self):
        self.check_roundtrip("{x for x in range(5)}")

    def test_dict_comprehension(self):
        self.check_roundtrip("{x: x*x for x in range(10)}")

    def test_class_decorators(self):
        self.check_roundtrip(class_decorator)

def test_main():
    test.support.run_unittest(UnparseTestCase)

if __name__ == '__main__':
    test_main()
<commit_msg>test_unparse.py: Do roundtrip testing for all Python files in Lib and Lib/test.<commit_after>import unittest
import test.support
import io
import os
import tokenize
import ast
import unparse

def read_pyfile(filename):
    """Read and return the contents of a Python source file (as a
    string), taking into account the file encoding."""
    with open(filename, "rb") as pyfile:
        encoding = tokenize.detect_encoding(pyfile.readline)[0]
    with open(filename, "r", encoding=encoding) as pyfile:
        source = pyfile.read()
    return source

for_else = """\
def f():
    for x in range(10):
        break
    else:
        y = 2
    z = 3
"""

while_else = """\
def g():
    while True:
        break
    else:
        y = 2
    z = 3
"""

relative_import = """\
from . import fred
from .. import barney
from .australia import shrimp as prawns
"""

nonlocal_ex = """\
def f():
    x = 1
    def g():
        nonlocal x
        x = 2
        y = 7
        def h():
            nonlocal x, y
"""

# also acts as test for 'except ... as ...'
raise_from = """\
try:
    1 / 0
except ZeroDivisionError as e:
    raise ArithmeticError from e
"""

class_decorator = """\
@f1(arg)
@f2
class Foo: pass
"""

class ASTTestCase(unittest.TestCase):
    def assertASTEqual(self, ast1, ast2):
        self.assertEqual(ast.dump(ast1), ast.dump(ast2))

    def check_roundtrip(self, code1, filename="internal"):
        ast1 = compile(code1, filename, "exec", ast.PyCF_ONLY_AST)
        unparse_buffer = io.StringIO()
        unparse.Unparser(ast1, unparse_buffer)
        code2 = unparse_buffer.getvalue()
        ast2 = compile(code2, filename, "exec", ast.PyCF_ONLY_AST)
        self.assertASTEqual(ast1, ast2)

class UnparseTestCase(ASTTestCase):
    # Tests for specific bugs found in earlier versions of unparse

    def test_del_statement(self):
        self.check_roundtrip("del x, y, z")

    def test_shifts(self):
        self.check_roundtrip("45 << 2")
        self.check_roundtrip("13 >> 7")

    def test_for_else(self):
        self.check_roundtrip(for_else)

    def test_while_else(self):
        self.check_roundtrip(while_else)

    def test_unary_parens(self):
        self.check_roundtrip("(-1)**7")
        self.check_roundtrip("not True or False")
        self.check_roundtrip("True or not False")

    def test_integer_parens(self):
        self.check_roundtrip("3 .__abs__()")

    def test_huge_float(self):
        self.check_roundtrip("1e1000")
        self.check_roundtrip("-1e1000")

    def test_lambda_parentheses(self):
        self.check_roundtrip("(lambda: int)()")

    def test_chained_comparisons(self):
        self.check_roundtrip("1 < 4 <= 5")
        self.check_roundtrip("a is b is c is not d")

    def test_function_arguments(self):
        self.check_roundtrip("def f(): pass")
        self.check_roundtrip("def f(a): pass")
        self.check_roundtrip("def f(b = 2): pass")
        self.check_roundtrip("def f(a, b): pass")
        self.check_roundtrip("def f(a, b = 2): pass")
        self.check_roundtrip("def f(a = 5, b = 2): pass")
        self.check_roundtrip("def f(*, a = 1, b = 2): pass")
        self.check_roundtrip("def f(*, a = 1, b): pass")
        self.check_roundtrip("def f(*, a, b = 2): pass")
        self.check_roundtrip("def f(a, b = None, *, c, **kwds): pass")
        self.check_roundtrip("def f(a=2, *args, c=5, d, **kwds): pass")
        self.check_roundtrip("def f(*args, **kwargs): pass")

    def test_relative_import(self):
        self.check_roundtrip(relative_import)

    def test_nonlocal(self):
        self.check_roundtrip(nonlocal_ex)

    def test_raise_from(self):
        self.check_roundtrip(raise_from)

    def test_bytes(self):
        self.check_roundtrip("b'123'")

    def test_annotations(self):
        self.check_roundtrip("def f(a : int): pass")
        self.check_roundtrip("def f(a: int = 5): pass")
        self.check_roundtrip("def f(*args: [int]): pass")
        self.check_roundtrip("def f(**kwargs: dict): pass")
        self.check_roundtrip("def f() -> None: pass")

    def test_set_literal(self):
        self.check_roundtrip("{'a', 'b', 'c'}")

    def test_set_comprehension(self):
        self.check_roundtrip("{x for x in range(5)}")

    def test_dict_comprehension(self):
        self.check_roundtrip("{x: x*x for x in range(10)}")

    def test_class_decorators(self):
        self.check_roundtrip(class_decorator)

class DirectoryTestCase(ASTTestCase):
    """Test roundtrip behaviour on all files in Lib and Lib/test."""

    # test directories, relative to the root of the distribution
    test_directories = 'Lib', os.path.join('Lib', 'test')

    def test_files(self):
        # get names of files to test
        dist_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)

        names = []
        for d in self.test_directories:
            test_dir = os.path.join(dist_dir, d)
            for n in os.listdir(test_dir):
                if n.endswith('.py') and not n.startswith('bad'):
                    names.append(os.path.join(test_dir, n))

        for filename in names:
            if test.support.verbose:
                print('Testing %s' % filename)
            source = read_pyfile(filename)
            self.check_roundtrip(source)

def test_main():
    test.support.run_unittest(UnparseTestCase, DirectoryTestCase)

if __name__ == '__main__':
    test_main()
<|endoftext|>


<commit_before>import unittest
import pandas as pd
import numpy as np
import random
import string
import pyarrow.parquet as pq
import numba
import hpat
from hpat import hiframes_sort
from hpat.str_arr_ext import StringArray
from hpat.tests.test_utils import (count_array_REPs, count_parfor_REPs,
                            count_parfor_OneDs, count_array_OneDs, dist_IR_contains,
                            get_start_end)

class TestJoin(unittest.TestCase):
    def test_join1(self):
        def test_impl(n):
            df1 = pd.DataFrame({'key1': np.arange(n)+3, 'A': np.arange(n)+1.0})
            df2 = pd.DataFrame({'key2': 2*np.arange(n)+1, 'B': n+np.arange(n)+1.0})
            df3 = pd.merge(df1, df2, left_on='key1', right_on='key2')
            return df3.B.sum()

        hpat_func = hpat.jit(test_impl)
        n = 11
        self.assertEqual(hpat_func(n), test_impl(n))
        self.assertEqual(count_array_REPs(), 0)
        self.assertEqual(count_parfor_REPs(), 0)
        n = 11111
        self.assertEqual(hpat_func(n), test_impl(n))

    def test_join1_seq(self):
        def test_impl(n):
            df1 = pd.DataFrame({'key1': np.arange(n)+3, 'A': np.arange(n)+1.0})
            df2 = pd.DataFrame({'key2': 2*np.arange(n)+1, 'B': n+np.arange(n)+1.0})
            df3 = df1.merge(df2, left_on='key1', right_on='key2')
            return df3.B

        hpat_func = hpat.jit(test_impl)
        n = 11
        self.assertEqual(hpat_func(n).sum(), test_impl(n).sum())
        self.assertEqual(count_array_OneDs(), 0)
        self.assertEqual(count_parfor_OneDs(), 0)
        n = 11111
        self.assertEqual(hpat_func(n).sum(), test_impl(n).sum())

    def test_join1_seq_str(self):
        def test_impl():
            df1 = pd.DataFrame({'key1': ['foo', 'bar', 'baz']})
            df2 = pd.DataFrame({'key2': ['baz', 'bar', 'baz'], 'B': ['b', 'zzz', 'ss']})
            df3 = pd.merge(df1, df2, left_on='key1', right_on='key2')
            return df3.B

        hpat_func = hpat.jit(test_impl)
        self.assertEqual(list(hpat_func()), list(test_impl()))

    def test_join1_seq_str_na(self):
        # test setting NA in string data column
        def test_impl():
            df1 = pd.DataFrame({'key1': ['foo', 'bar', 'baz']})
            df2 = pd.DataFrame({'key2': ['baz', 'bar', 'baz'], 'B': ['b', 'zzz', 'ss']})
            df3 = df1.merge(df2, left_on='key1', right_on='key2', how='left')
            return df3.B

        hpat_func = hpat.jit(test_impl)
        self.assertEqual(set(hpat_func()), set(test_impl()))

    def test_join_mutil_seq1(self):
        def test_impl(df1, df2):
            return df1.merge(df2, on=['A', 'B'])

        hpat_func = hpat.jit(test_impl)
        df1 = pd.DataFrame({'A': [3,1,1,3,4],
                            'B': [1,2,3,2,3],
                            'C': [7,8,9,4,5]})

        df2 = pd.DataFrame({'A': [2,1,4,4,3],
                            'B': [1,3,2,3,2],
                            'D': [1,2,3,4,8]})

        pd.testing.assert_frame_equal(hpat_func(df1, df2), test_impl(df1, df2))

    def test_join_mutil_parallel1(self):
        def test_impl(A1, B1, C1, A2, B2, D2):
            df1 = pd.DataFrame({'A': A1, 'B': B1, 'C': C1})
            df2 = pd.DataFrame({'A': A2, 'B': B2, 'D': D2})
            df3 = df1.merge(df2, on=['A', 'B'])
            return df3.C.sum() + df3.D.sum()

        hpat_func = hpat.jit(locals={
            'A1:input': 'distributed',
            'B1:input': 'distributed',
            'C1:input': 'distributed',
            'A2:input': 'distributed',
            'B2:input': 'distributed',
            'D2:input': 'distributed',})(test_impl)
        df1 = pd.DataFrame({'A': [3,1,1,3,4],
                            'B': [1,2,3,2,3],
                            'C': [7,8,9,4,5]})

        df2 = pd.DataFrame({'A': [2,1,4,4,3],
                            'B': [1,3,2,3,2],
                            'D': [1,2,3,4,8]})

        start, end = get_start_end(len(df1))
        h_A1 = df1.A.values[start:end]
        h_B1 = df1.B.values[start:end]
        h_C1 = df1.C.values[start:end]
        h_A2 = df2.A.values[start:end]
        h_B2 = df2.B.values[start:end]
        h_D2 = df2.D.values[start:end]
        p_A1 = df1.A.values
        p_B1 = df1.B.values
        p_C1 = df1.C.values
        p_A2 = df2.A.values
        p_B2 = df2.B.values
        p_D2 = df2.D.values
        h_res = hpat_func(h_A1, h_B1, h_C1, h_A2, h_B2, h_D2)
        p_res = test_impl(p_A1, p_B1, p_C1, p_A2, p_B2, p_D2)
        self.assertEqual(h_res, p_res)

    def test_merge_asof_seq1(self):
        def test_impl(df1, df2):
            return pd.merge_asof(df1, df2, on='time')

        hpat_func = hpat.jit(test_impl)
        df1 = pd.DataFrame(
            {'time': pd.DatetimeIndex(
                ['2017-01-03', '2017-01-06', '2017-02-21']), 'B': [4, 5, 6]})
        df2 = pd.DataFrame(
            {'time': pd.DatetimeIndex(
                ['2017-01-01', '2017-01-02', '2017-01-04', '2017-02-23',
                '2017-02-25']), 'A': [2,3,7,8,9]})
        pd.testing.assert_frame_equal(hpat_func(df1, df2), test_impl(df1, df2))

    def test_merge_asof_parallel1(self):
        def test_impl():
            df1 = pd.read_parquet('asof1.pq')
            df2 = pd.read_parquet('asof2.pq')
            df3 = pd.merge_asof(df1, df2, on='time')
            return (df3.A.sum(), df3.time.max(), df3.B.sum())

        hpat_func = hpat.jit(test_impl)
        self.assertEqual(hpat_func(), test_impl())

    def test_join_left_seq1(self):
        def test_impl(df1, df2):
            return pd.merge(df1, df2, how='left', on='key')

        hpat_func = hpat.jit(test_impl)
        df1 = pd.DataFrame(
            {'key': [2,3,5,1,2,8], 'A': np.array([4,6,3,9,9,-1], np.float)})
        df2 = pd.DataFrame(
            {'key': [1,2,9,3,2], 'B': np.array([1,7,2,6,5], np.float)})
        h_res = hpat_func(df1, df2)
        res = test_impl(df1, df2)
        np.testing.assert_array_equal(h_res.key.values, res.key.values)
        # converting arrays to sets since order of values can be different
        self.assertEqual(set(h_res.A.values), set(res.A.values))
        self.assertEqual(
            set(h_res.B.dropna().values), set(res.B.dropna().values))

    def test_join_left_seq2(self):
        def test_impl(df1, df2):
            return pd.merge(df1, df2, how='left', on='key')

        hpat_func = hpat.jit(test_impl)
        # test left run where a key is repeated on left but not right side
        df1 = pd.DataFrame(
            {'key': [2,3,5,3,2,8], 'A': np.array([4,6,3,9,9,-1], np.float)})
        df2 = pd.DataFrame(
            {'key': [1,2,9,3,10], 'B': np.array([1,7,2,6,5], np.float)})
        h_res = hpat_func(df1, df2)
        res = test_impl(df1, df2)
        np.testing.assert_array_equal(h_res.key.values, res.key.values)
        # converting arrays to sets since order of values can be different
        self.assertEqual(set(h_res.A.values), set(res.A.values))
        self.assertEqual(
            set(h_res.B.dropna().values), set(res.B.dropna().values))

    def test_join_right_seq1(self):
        def test_impl(df1, df2):
            return pd.merge(df1, df2, how='right', on='key')

        hpat_func = hpat.jit(test_impl)
        df1 = pd.DataFrame(
            {'key': [2,3,5,1,2,8], 'A': np.array([4,6,3,9,9,-1], np.float)})
        df2 = pd.DataFrame(
            {'key': [1,2,9,3,2], 'B': np.array([1,7,2,6,5], np.float)})
        h_res = hpat_func(df1, df2)
        res = test_impl(df1, df2)
        np.testing.assert_array_equal(h_res.key.values, res.key.values)
        # converting arrays to sets since order of values can be different
        self.assertEqual(set(h_res.B.values), set(res.B.values))
        self.assertEqual(
            set(h_res.A.dropna().values), set(res.A.dropna().values))

    def test_join_outer_seq1(self):
        def test_impl(df1, df2):
            return pd.merge(df1, df2, how='outer', on='key')

        hpat_func = hpat.jit(test_impl)
        df1 = pd.DataFrame(
            {'key': [2,3,5,1,2,8], 'A': np.array([4,6,3,9,9,-1], np.float)})
        df2 = pd.DataFrame(
            {'key': [1,2,9,3,2], 'B': np.array([1,7,2,6,5], np.float)})
        h_res = hpat_func(df1, df2)
        res = test_impl(df1, df2)
        np.testing.assert_array_equal(h_res.key.values, res.key.values)
        # converting arrays to sets since order of values can be different
        self.assertEqual(
            set(h_res.B.dropna().values), set(res.B.dropna().values))
        self.assertEqual(
            set(h_res.A.dropna().values), set(res.A.dropna().values))

if __name__ == "__main__":
    unittest.main()
<commit_msg>fix join str test<commit_after>import unittest
import pandas as pd
import numpy as np
import random
import string
import pyarrow.parquet as pq
import numba
import hpat
from hpat import hiframes_sort
from hpat.str_arr_ext import StringArray
from hpat.tests.test_utils import (count_array_REPs, count_parfor_REPs,
                            count_parfor_OneDs, count_array_OneDs, dist_IR_contains,
                            get_start_end)

class TestJoin(unittest.TestCase):
    def test_join1(self):
        def test_impl(n):
            df1 = pd.DataFrame({'key1': np.arange(n)+3, 'A': np.arange(n)+1.0})
            df2 = pd.DataFrame({'key2': 2*np.arange(n)+1, 'B': n+np.arange(n)+1.0})
            df3 = pd.merge(df1, df2, left_on='key1', right_on='key2')
            return df3.B.sum()

        hpat_func = hpat.jit(test_impl)
        n = 11
        self.assertEqual(hpat_func(n), test_impl(n))
        self.assertEqual(count_array_REPs(), 0)
        self.assertEqual(count_parfor_REPs(), 0)
        n = 11111
        self.assertEqual(hpat_func(n), test_impl(n))

    def test_join1_seq(self):
        def test_impl(n):
            df1 = pd.DataFrame({'key1': np.arange(n)+3, 'A': np.arange(n)+1.0})
            df2 = pd.DataFrame({'key2': 2*np.arange(n)+1, 'B': n+np.arange(n)+1.0})
            df3 = df1.merge(df2, left_on='key1', right_on='key2')
            return df3.B

        hpat_func = hpat.jit(test_impl)
        n = 11
        self.assertEqual(hpat_func(n).sum(), test_impl(n).sum())
        self.assertEqual(count_array_OneDs(), 0)
        self.assertEqual(count_parfor_OneDs(), 0)
        n = 11111
        self.assertEqual(hpat_func(n).sum(), test_impl(n).sum())

    def test_join1_seq_str(self):
        def test_impl():
            df1 = pd.DataFrame({'key1': ['foo', 'bar', 'baz']})
            df2 = pd.DataFrame({'key2': ['baz', 'bar', 'baz'], 'B': ['b', 'zzz', 'ss']})
            df3 = pd.merge(df1, df2, left_on='key1', right_on='key2')
            return df3.B

        hpat_func = hpat.jit(test_impl)
        self.assertEqual(set(hpat_func()), set(test_impl()))

    def test_join1_seq_str_na(self):
        # test setting NA in string data column
        def test_impl():
            df1 = pd.DataFrame({'key1': ['foo', 'bar', 'baz']})
            df2 = pd.DataFrame({'key2': ['baz', 'bar', 'baz'], 'B': ['b', 'zzz', 'ss']})
            df3 = df1.merge(df2, left_on='key1', right_on='key2', how='left')
            return df3.B

        hpat_func = hpat.jit(test_impl)
        self.assertEqual(set(hpat_func()), set(test_impl()))

    def test_join_mutil_seq1(self):
        def test_impl(df1, df2):
            return df1.merge(df2, on=['A', 'B'])

        hpat_func = hpat.jit(test_impl)
        df1 = pd.DataFrame({'A': [3,1,1,3,4],
                            'B': [1,2,3,2,3],
                            'C': [7,8,9,4,5]})

        df2 = pd.DataFrame({'A': [2,1,4,4,3],
                            'B': [1,3,2,3,2],
                            'D': [1,2,3,4,8]})

        pd.testing.assert_frame_equal(hpat_func(df1, df2), test_impl(df1, df2))

    def test_join_mutil_parallel1(self):
        def test_impl(A1, B1, C1, A2, B2, D2):
            df1 = pd.DataFrame({'A': A1, 'B': B1, 'C': C1})
            df2 = pd.DataFrame({'A': A2, 'B': B2, 'D': D2})
            df3 = df1.merge(df2, on=['A', 'B'])
            return df3.C.sum() + df3.D.sum()

        hpat_func = hpat.jit(locals={
            'A1:input': 'distributed',
            'B1:input': 'distributed',
            'C1:input': 'distributed',
            'A2:input': 'distributed',
            'B2:input': 'distributed',
            'D2:input': 'distributed',})(test_impl)
        df1 = pd.DataFrame({'A': [3,1,1,3,4],
                            'B': [1,2,3,2,3],
                            'C': [7,8,9,4,5]})

        df2 = pd.DataFrame({'A': [2,1,4,4,3],
                            'B': [1,3,2,3,2],
                            'D': [1,2,3,4,8]})

        start, end = get_start_end(len(df1))
        h_A1 = df1.A.values[start:end]
        h_B1 = df1.B.values[start:end]
        h_C1 = df1.C.values[start:end]
        h_A2 = df2.A.values[start:end]
        h_B2 = df2.B.values[start:end]
        h_D2 = df2.D.values[start:end]
        p_A1 = df1.A.values
        p_B1 = df1.B.values
        p_C1 = df1.C.values
        p_A2 = df2.A.values
        p_B2 = df2.B.values
        p_D2 = df2.D.values
        h_res = hpat_func(h_A1, h_B1, h_C1, h_A2, h_B2, h_D2)
        p_res = test_impl(p_A1, p_B1, p_C1, p_A2, p_B2, p_D2)
        self.assertEqual(h_res, p_res)

    def test_merge_asof_seq1(self):
        def test_impl(df1, df2):
            return pd.merge_asof(df1, df2, on='time')

        hpat_func = hpat.jit(test_impl)
        df1 = pd.DataFrame(
            {'time': pd.DatetimeIndex(
                ['2017-01-03', '2017-01-06', '2017-02-21']), 'B': [4, 5, 6]})
        df2 = pd.DataFrame(
            {'time': pd.DatetimeIndex(
                ['2017-01-01', '2017-01-02', '2017-01-04', '2017-02-23',
                '2017-02-25']), 'A': [2,3,7,8,9]})
        pd.testing.assert_frame_equal(hpat_func(df1, df2), test_impl(df1, df2))

    def test_merge_asof_parallel1(self):
        def test_impl():
            df1 = pd.read_parquet('asof1.pq')
            df2 = pd.read_parquet('asof2.pq')
            df3 = pd.merge_asof(df1, df2, on='time')
            return (df3.A.sum(), df3.time.max(), df3.B.sum())

        hpat_func = hpat.jit(test_impl)
        self.assertEqual(hpat_func(), test_impl())

    def test_join_left_seq1(self):
        def test_impl(df1, df2):
            return pd.merge(df1, df2, how='left', on='key')

        hpat_func = hpat.jit(test_impl)
        df1 = pd.DataFrame(
            {'key': [2,3,5,1,2,8], 'A': np.array([4,6,3,9,9,-1], np.float)})
        df2 = pd.DataFrame(
            {'key': [1,2,9,3,2], 'B': np.array([1,7,2,6,5], np.float)})
        h_res = hpat_func(df1, df2)
        res = test_impl(df1, df2)
        np.testing.assert_array_equal(h_res.key.values, res.key.values)
        # converting arrays to sets since order of values can be different
        self.assertEqual(set(h_res.A.values), set(res.A.values))
        self.assertEqual(
            set(h_res.B.dropna().values), set(res.B.dropna().values))

    def test_join_left_seq2(self):
        def test_impl(df1, df2):
            return pd.merge(df1, df2, how='left', on='key')

        hpat_func = hpat.jit(test_impl)
        # test left run where a key is repeated on left but not right side
        df1 = pd.DataFrame(
            {'key': [2,3,5,3,2,8], 'A': np.array([4,6,3,9,9,-1], np.float)})
        df2 = pd.DataFrame(
            {'key': [1,2,9,3,10], 'B': np.array([1,7,2,6,5], np.float)})
        h_res = hpat_func(df1, df2)
        res = test_impl(df1, df2)
        np.testing.assert_array_equal(h_res.key.values, res.key.values)
        # converting arrays to sets since order of values can be different
        self.assertEqual(set(h_res.A.values), set(res.A.values))
        self.assertEqual(
            set(h_res.B.dropna().values), set(res.B.dropna().values))

    def test_join_right_seq1(self):
        def test_impl(df1, df2):
            return pd.merge(df1, df2, how='right', on='key')

        hpat_func = hpat.jit(test_impl)
        df1 = pd.DataFrame(
            {'key': [2,3,5,1,2,8], 'A': np.array([4,6,3,9,9,-1], np.float)})
        df2 = pd.DataFrame(
            {'key': [1,2,9,3,2], 'B': np.array([1,7,2,6,5], np.float)})
        h_res = hpat_func(df1, df2)
        res = test_impl(df1, df2)
        np.testing.assert_array_equal(h_res.key.values, res.key.values)
        # converting arrays to sets since order of values can be different
        self.assertEqual(set(h_res.B.values), set(res.B.values))
        self.assertEqual(
            set(h_res.A.dropna().values), set(res.A.dropna().values))

    def test_join_outer_seq1(self):
        def test_impl(df1, df2):
            return pd.merge(df1, df2, how='outer', on='key')

        hpat_func = hpat.jit(test_impl)
        df1 = pd.DataFrame(
            {'key': [2,3,5,1,2,8], 'A': np.array([4,6,3,9,9,-1], np.float)})
        df2 = pd.DataFrame(
            {'key': [1,2,9,3,2], 'B': np.array([1,7,2,6,5], np.float)})
        h_res = hpat_func(df1, df2)
        res = test_impl(df1, df2)
        np.testing.assert_array_equal(h_res.key.values, res.key.values)
        # converting arrays to sets since order of values can be different
        self.assertEqual(
            set(h_res.B.dropna().values), set(res.B.dropna().values))
        self.assertEqual(
            set(h_res.A.dropna().values), set(res.A.dropna().values))

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>#!python3

"""
Experiments with a new multi-unit double-auction mechanism.
"""

from random_datasets import randomValuations, randomAuction
from traders import Trader, virtualTraders, virtualTradersWithIndices, winningAndLosingTraders
from operator import itemgetter
from collections import defaultdict

import numpy as np

def newMUDA(buyers:list, numOfItems:int):
    (vbuyers,vsellers) = virtualTraders(buyers)
    # print("vbuyers",vbuyers)
    vbuyers_sorted_by_1 = sorted(vbuyers, key=itemgetter(1), reverse=True)
    print("vbuyers_sorted_by_1",vbuyers_sorted_by_1)
    vbuyers_winning = vbuyers_sorted_by_1[0:numOfItems]
    print("vbuyers_winning",vbuyers_winning)
    mapIndexToQuantity = defaultdict(int)
    for (_, value, index) in vbuyers_winning:
        mapIndexToQuantity[index] += 1

    print("mapIndexToQuantity",mapIndexToQuantity)
    buyers_sorted_by_2 = sorted(buyers, key=lambda t:t.valueOf(numBundles=2), reverse=True)
    print("buyers_sorted_by_2",buyers_sorted_by_2)

    # calculate price for 2 units:
    price_2 = 0
    for b in buyers_sorted_by_2:
        if mapIndexToQuantity.get(b.index,0)==0:
            price_2 = b.valueOf(numBundles=2)
            break
    print("price_2", price_2)

    # calculate price for 1 units:
    price_1 = 0
    for b in vbuyers_sorted_by_1:
        if mapIndexToQuantity.get(b[2],0)==0:
            price_1 = b[1]
            break
    print("price_1", price_1)

    for (index,quantity) in mapIndexToQuantity.items():
        b = buyers[index]
        value_1 = b.valueOf(1)
        utility_1 = value_1 - price_1
        value_2 = b.valueOf(2)
        utility_2 = value_2 - price_2
        print(index,":   ","quantity",quantity,"  value_1",value_1,"utility_1",utility_1,"  value_2",value_2,"utility_2",utility_2)
        if utility_1 < utility_2 and quantity==1:
            print("  WARNING: utility_1<utility_2 but quantity=1")
        if utility_1 > utility_2 and quantity==2:
            print("  WARNING: utility_1>utility_2 but quantity=2")

    return "In construction"

if __name__ == "__main__":
    # seed = 47585  # bad for "<1,<2"
    # seed = 464410 # bad for "!=1,!=2"
    # seed = 50193  # bad for ">0,>0"
    seed = int(np.round(np.random.random()*1000000))
    print("seed",seed)
    np.random.seed(seed)
    numOfTraders = 6
    traders = []
    for i in range(numOfTraders):
        traders.append(Trader.Buyer(randomValuations(
            minNumOfUnits=1, maxNumOfUnits=2, meanValue=10, maxNoiseSize=10, round=True, index=i), index=i))
    print("traders",traders)
    print(newMUDA(traders, numOfItems=5))

<commit_msg>experimental stuff<commit_after>#!python3

"""
Experiments with a new multi-unit double-auction mechanism.

EXPERIMENTAL STUFF
"""

from random_datasets import randomValuations, randomAuction
from traders import Trader, virtualTraders, virtualTradersWithIndices, winningAndLosingTraders
from operator import itemgetter
from collections import defaultdict

import numpy as np

def newMUDA(buyers:list, numOfItems:int):
    (vbuyers,vsellers) = virtualTraders(buyers)
    # print("vbuyers",vbuyers)
    vbuyers_sorted_by_1 = sorted(vbuyers, key=itemgetter(1), reverse=True)
    print("vbuyers_sorted_by_1",vbuyers_sorted_by_1)
    vbuyers_winning = vbuyers_sorted_by_1[0:numOfItems]
    print("vbuyers_winning",vbuyers_winning)
    mapIndexToQuantity = defaultdict(int)
    for (_, value, index) in vbuyers_winning:
        mapIndexToQuantity[index] += 1

    print("mapIndexToQuantity",mapIndexToQuantity)
    buyers_sorted_by_2 = sorted(buyers, key=lambda t:t.valueOf(numBundles=2), reverse=True)
    print("buyers_sorted_by_2")
    for b in buyers_sorted_by_2: print("  ",b)

    # calculate price for 2 units:
    price_2 = 0
    for b in buyers_sorted_by_2:
        if mapIndexToQuantity.get(b.index,0)!=2: #==0:#
            price_2 = b.valueOf(numBundles=2)
            break
    print("price_2", price_2)

    # calculate price for 1 units:
    price_1 = 0
    for b in vbuyers_sorted_by_1:
        if mapIndexToQuantity.get(b[2],0)!=1: #==0: #
            price_1 = b[1]
            break
    print("price_1", price_1)

    for (index,quantity) in mapIndexToQuantity.items():
        b = buyers[index]
        value_1 = b.valueOf(1)
        utility_1 = value_1 - price_1
        value_2 = b.valueOf(2)
        utility_2 = value_2 - price_2
        print(index,":   ","quantity",quantity,"  value_1",value_1,"utility_1",utility_1,"  value_2",value_2,"utility_2",utility_2)
        if utility_1 < utility_2 and quantity==1:
            print("  WARNING: utility_1<utility_2 but quantity=1")
        if utility_1 > utility_2 and quantity==2:
            print("  WARNING: utility_1>utility_2 but quantity=2")

    return "In construction"

if __name__ == "__main__":
    # seed = 47585  # bad for "<1,<2"
    # seed = 115423 # bad for "==0,==0"
    # seed = 353511 # bad for "!=1,!=2"
    seed = int(np.round(np.random.random()*1000000))
    print("seed",seed)
    np.random.seed(seed)
    numOfSellers = 4
    numOfRealBuyers = 6
    traders = []
    for i in range(numOfRealBuyers):
        traders.append(Trader.Buyer(randomValuations(
            minNumOfUnits=1, maxNumOfUnits=2, meanValue=10, maxNoiseSize=10, round=True, index=i), index=i))
    print("traders",traders)
    print(newMUDA(traders, numOfItems=numOfSellers))

<|endoftext|>


<commit_before>"""
This function calculates the binomial proportions of a series of binomial variables
for each level of a given group variable. The dataframe has the following format:

        group     A    B    C
    0      g1   yes   no  yes
    1      g1   yes  NaN  yes
    2      g2    no  NaN  yes
    3      g1    no  yes  NaN
    4      g2    no  yes   no
    5      g2    no  yes  yes
    6      g1    no  yes  yes
    7      g1   yes   no  yes
    8      g2   NaN   no   no
    9      g1   yes   no   no

 ...and produces the following dataframe:

    group   var   count   success   propn
       g1     A       6         4    0.66
       g1     B       5         2    0.40
       g1     C       5         4    0.80
       g2     A       3         0    0.00
       g2     B       4         2    0.50
       g2     C       4         2    0.50

1. NEED TO CHECK that all variables in var list are in the dataframe.
2. NEED TO CHECK that group var is not included in list of columns
3. IF GROUPS TO PLOT IS A LIST AND GROUP VAR NAME IS NONE THEN CHECK
"""

import pkg_resources

try:
    pkg_resources.get_distribution('numpy')
except pkg_resources.DistributionNotFound:
    numpyPresent = False
    print("Error: Numpy package not available.")
else:
    numpyPresent = True
    import numpy as np

try:
    pkg_resources.get_distribution('pandas')
except pkg_resources.DistributionNotFound:
    pandasPresent = False
    print("Error: Pandas package not available.")
else:
    pandasPresent = True
    import pandas as pd

try:
    pkg_resources.get_distribution('scipy')
except pkg_resources.DistributionNotFound:
    scipyPresent = False
    print("Error: Scipy package not available.")
else:
    scipyPresent = True
    from scipy.stats import norm

def phjCalculateBinomialProportions(phjTempDF,
                                    phjColumnsList = None,
                                    phjSuccess = 'yes',
                                    phjMissingValue = 'missing',
                                    phjGroupVarName = None,
                                    phjConfidenceIntervalType = 'normal',
                                    phjConfidenceIntervalLevel = 0.95,
                                    phjPlotResults = True,
                                    phjGroupsToPlotList = 'all',
                                    phjSortProportions = False,
                                    phjGraphTitle = None,
                                    phjPrintResults = False):

    if phjGroupVarName is None:
        # If no group var name is supplied then calculate success for
        # whole columns and return summary dataframe.
        phjPropDF = phjCountSuccesses(x = phjTempDF[phjColumnsList],
                                      phjColumnsList = phjColumnsList)

    else:
        # Copy required columns to new dataframe
        phjTempDF = phjTempDF[phjColumnsList + [phjGroupVarName]].copy()

        # Calculate number of successes for each level of group variable and return summary dataframe
        phjPropDF = phjTempDF.groupby(phjGroupVarName).apply(lambda x: phjCountSuccesses(x,
                                                                                phjColumnsList = phjColumnsList))

    phjPropDF = phjPropDF.reset_index(drop = False)

    phjPropDF['proportion'] = phjPropDF['success'] / phjPropDF['count']

    if phjSortProportions != False:
        if phjSortProportions == 'asc':
            phjPropDF = phjPropDF.sort_values(by = 'proportion',
                                              axis = 0,
                                              ascending = True)

        elif phjSortProportions == 'desc':
            phjPropDF = phjPropDF.sort_values(by = 'proportion',
                                              axis = 0,
                                              ascending = False)

        else:
            print('Option for sorting does not exist.')

    if phjConfidenceIntervalType == 'normal':
        # Calculate standard error
        phjPropDF['stderr'] = ((phjPropDF['proportion'] * (1 - phjPropDF['proportion'])) / phjPropDF['count']).pow(1/2)

        # Calculate reliability coefficient
        phjReliabilityCoefficient = norm.ppf(1 - ((1 - phjConfidenceIntervalLevel)/2))

        # Calculate confidence interval
        phjCIColumnName = str(int(phjConfidenceIntervalLevel *100)) + 'ci'
        phjPropDF[phjCIColumnName] = phjReliabilityCoefficient * phjPropDF['stderr']

        # Calculate uppper and lower CI limits
        phjPropDF['lowCILimit'] = phjPropDF['proportion'] - phjPropDF[phjCIColumnName]
        phjPropDF['uppCILimit'] = phjPropDF['proportion'] + phjPropDF[phjCIColumnName]

    else:
        phjPropDF['stderr'] = np.nan

    if phjPrintResults == True:
        with pd.option_context('display.float_format','{:,.4f}'.format):
            print(phjPropDF)

    if phjPlotResults == True:
        phjPlotProportions(phjTempDF = phjPropDF,
                           phjGroupVarName = phjGroupVarName,
                           phjGroupsToPlotList = 'all',
                           phjCIColumnName = phjCIColumnName,
                           phjGraphTitle = phjGraphTitle)

    return phjPropDF

def phjCountSuccesses(x,
                      phjColumnsList):

    phjSummaryDF = pd.DataFrame(columns = phjColumnsList, index = ['count','success'])

    for var in phjColumnsList:
        phjSummaryDF.loc['count',var] = ((x[var].dropna()) == 'yes').count()
        phjSummaryDF.loc['success',var] = ((x[var].dropna()) == 'yes').sum()

    # Transpose
    phjSummaryDF = phjSummaryDF.T

    # Rename index
    phjSummaryDF.index.name = 'var'

    return phjSummaryDF

def phjPlotProportions(phjTempDF,
                       phjGroupVarName = None,
                       phjGroupsToPlotList = None,
                       phjCIColumnName = None,
                       phjGraphTitle = None):

    # Plot graphs
    # ===========
    # If no group var name is provided then assume plot all rows
    if phjGroupVarName is None:
        if phjGraphTitle is None:
            phjTempDF.plot(x = 'var',
                           y = 'proportion',
                           kind = 'bar',
                           title = 'Proportions',
                           legend = False,
                           yerr = phjCIColumnName)
        else:
            phjTempDF.plot(x = 'var',
                           y = 'proportion',
                           kind = 'bar',
                           title = phjGraphTitle,
                           legend = False,
                           yerr = phjCIColumnName)

    else:
        if phjGroupsToPlotList == 'all':
            phjGroupsToPlotList = phjTempDF[phjGroupVarName].unique()

        for i, phjGroup in phjTempDF.groupby(phjGroupVarName):
            if i in phjGroupsToPlotList:
                if phjGraphTitle is None:
                    phjTempDF.loc[phjTempDF[phjGroupVarName] == i,:].plot(x = 'var',
                                                                          y = 'proportion',
                                                                          kind = 'bar',
                                                                          title = 'Group ' + i,
                                                                          legend = False,
                                                                          yerr = phjCIColumnName)
                else:
                    phjTempDF.loc[phjTempDF[phjGroupVarName] == i,:].plot(x = 'var',
                                                                          y = 'proportion',
                                                                          kind = 'bar',
                                                                          title = phjGraphTitle,
                                                                          legend = False,
                                                                          yerr = phjCIColumnName)

            plt.show()

    return

if __name__ == '__main__':
main()
<commit_msg>Delete phjCalculateBinomialProportions.py<commit_after><|endoftext|>


<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt

import frappe
from frappe.model import rename_field
from frappe.modules import scrub, get_doctype_module

rename_map = {
	"Opportunity": [
		["enquiry_details", "items"]
	],
	"Quotation": [
		["quotation_details", "items"],
		["other_charges", "taxes"]
	],
	"Sales Order": [
		["sales_order_details", "items"],
		["other_charges", "taxes"],
		["packing_details", "packed_items"]
	],
	"Delivery Note": [
		["delivery_note_details", "items"],
		["other_charges", "taxes"],
		["packing_details", "packed_items"]
	],
	"Sales Invoice": [
		["entries", "items"],
		["other_charges", "taxes"],
		["packing_details", "packed_items"],
		["advance_adjustment_details", "advances"]
	],
	"Material Request": [
		["indent_details", "items"]
	],
	"Supplier Quotation": [
		["quotation_items", "items"],
		["other_charges", "taxes"]
	],
	"Purchase Order": [
		["po_details", "items"],
		["other_charges", "taxes"],
		["po_raw_material_details", "supplied_items"]
	],
	"Purchase Receipt": [
		["purchase_receipt_details", "items"],
		["other_charges", "taxes"],
		["pr_raw_material_details", "supplied_items"]
	],
	"Purchase Invoice": [
		["entries", "items"],
		["other_charges", "taxes"],
		["advance_allocation_details", "advances"]
	],
	"Production Order": [
		["production_order_operations", "operations"]
	],
	"BOM": [
		["bom_operations", "operations"],
		["bom_materials", "items"],
		["flat_bom_details", "exploded_items"]
	],
	"Payment Reconciliation": [
		["payment_reconciliation_payments", "payments"],
		["payment_reconciliation_invoices", "invoices"]
	],
	"Sales Taxes and Charges Template": [
		["other_charges", "taxes"],
		["valid_for_territories", "territories"]
	],
	"Purchase Taxes and Charges Template": [
		["other_charges", "taxes"]
	],
	"Shipping Rule": [
		["shipping_rule_conditions", "conditions"],
		["valid_for_territories", "territories"]
	],
	"Price List": [
		["valid_for_territories", "territories"]
	],
	"Appraisal": [
		["appraisal_details", "goals"]
	],
	"Appraisal Template": [
		["kra_sheet", "goals"]
	],
	"Bank Reconciliation": [
		["entries", "journal_entries"]
	],
	"Cost Center": [
		["budget_details", "budgets"]
	],
	"C-Form": [
		["invoice_details", "invoices"]
	],
	"Employee": [
		["employee_leave_approvers", "leave_approvers"],
		["educational_qualification_details", "education"],
		["previous_experience_details", "external_work_history"],
		["experience_in_company_details", "internal_work_history"]
	],
	"Expense Claim": [
		["expense_voucher_details", "expenses"]
	],
	"Fiscal Year": [
		["fiscal_year_companies", "companies"]
	],
	"Holiday List": [
		["holiday_list_details", "holidays"]
	],
	"Installation Note": [
		["installed_item_details", "items"]
	],
	"Item": [
		["item_variants", "variants"],
		["item_reorder", "reorder_levels"],
		["uom_conversion_details", "uoms"],
		["item_supplier_details", "supplier_items"],
		["item_customer_details", "customer_items"],
		["item_tax", "taxes"],
		["item_specification_details", "quality_parameters"],
		["item_website_specifications", "website_specifications"]
	],
	"Item Group": [
		["item_website_specifications", "website_specifications"]
	],
	"Landed Cost Voucher": [
		["landed_cost_purchase_receipts", "purchase_receipts"],
		["landed_cost_items", "items"],
		["landed_cost_taxes_and_charges", "taxes"]
	],
	"Maintenance Schedule": [
		["item_maintenance_detail", "items"],
		["maintenance_schedule_detail", "schedules"]
	],
	"Maintenance Visit": [
		["maintenance_visit_details", "purposes"]
	],
	"Packing Slip": [
		["item_details", "items"]
	],
	"Customer": [
		["party_accounts", "accounts"]
	],
	"Customer Group": [
		["party_accounts", "accounts"]
	],
	"Supplier": [
		["party_accounts", "accounts"]
	],
	"Supplier Type": [
		["party_accounts", "accounts"]
	],
	"Payment Tool": [
		["payment_tool_details", "vouchers"]
	],
	"Production Planning Tool": [
		["pp_so_details", "sales_orders"],
		["pp_details", "items"]
	],
	"Quality Inspection": [
		["qa_specification_details", "readings"]
	],
	"Salary Slip": [
		["earning_details", "earnings"],
		["deduction_details", "deductions"]
	],
	"Salary Structure": [
		["earning_details", "earnings"],
		["deduction_details", "deductions"]
	],
	"Sales BOM": [
		["sales_bom_items", "items"]
	],
	"SMS Settings": [
		["static_parameter_details", "parameters"]
	],
	"Stock Entry": [
		["mtn_details", "items"]
	],
	"Sales Partner": [
		["partner_target_details", "targets"]
	],
	"Sales Person": [
		["target_details", "targets"]
	],
	"Territory": [
		["target_details", "targets"]
	],
	"Time Log Batch": [
		["time_log_batch_details", "time_logs"]
	],
	"Workstation": [
		["workstation_operation_hours", "working_hours"]
	],
	"Payment Reconciliation Payment": [
		["journal_voucher", "journal_entry"],
	],
	"Purchase Invoice Advance": [
		["journal_voucher", "journal_entry"],
	],
	"Sales Invoice Advance": [
		["journal_voucher", "journal_entry"],
	],
	"Journal Entry": [
		["entries", "accounts"]
	],
	"Monthly Distribution": [
		["budget_distribution_details", "percentages"]
	]
}

def execute():
	# rename doctypes
	tables = frappe.db.sql_list("show tables")
	for old_dt, new_dt in [["Journal Voucher Detail", "Journal Entry Account"],
		["Journal Voucher", "Journal Entry"],
		["Budget Distribution Detail", "Monthly Distribution Percentage"],
		["Budget Distribution", "Monthly Distribution"]]:
			if "tab"+new_dt not in tables:
				frappe.rename_doc("DocType", old_dt, new_dt, force=True)

	# reload new child doctypes
	frappe.reload_doc("manufacturing", "doctype", "production_order_operation")
	frappe.reload_doc("manufacturing", "doctype", "workstation_working_hour")
	frappe.reload_doc("stock", "doctype", "item_variant")
	frappe.reload_doc("accounts", "doctype", "party_account")
	frappe.reload_doc("accounts", "doctype", "fiscal_year_company")

	#rename table fieldnames
	for dn in rename_map:
		frappe.reload_doc(get_doctype_module(dn), "doctype", scrub(dn))

	for dt, field_list in rename_map.items():
		for field in field_list:
			rename_field(dt, field[0], field[1])

	# update voucher type
	for old, new in [["Bank Voucher", "Bank Entry"], ["Cash Voucher", "Cash Entry"],
		["Credit Card Voucher", "Credit Card Entry"], ["Contra Voucher", "Contra Entry"],
		["Write Off Voucher", "Write Off Entry"], ["Excise Voucher", "Excise Entry"]]:
			frappe.db.sql("update `tabJournal Entry` set voucher_type=%s where voucher_type=%s", (new, old))
<commit_msg>Rename table fieldnames only if doctype exists<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt

import frappe
from frappe.model import rename_field
from frappe.modules import scrub, get_doctype_module

rename_map = {
	"Opportunity": [
		["enquiry_details", "items"]
	],
	"Quotation": [
		["quotation_details", "items"],
		["other_charges", "taxes"]
	],
	"Sales Order": [
		["sales_order_details", "items"],
		["other_charges", "taxes"],
		["packing_details", "packed_items"]
	],
	"Delivery Note": [
		["delivery_note_details", "items"],
		["other_charges", "taxes"],
		["packing_details", "packed_items"]
	],
	"Sales Invoice": [
		["entries", "items"],
		["other_charges", "taxes"],
		["packing_details", "packed_items"],
		["advance_adjustment_details", "advances"]
	],
	"Material Request": [
		["indent_details", "items"]
	],
	"Supplier Quotation": [
		["quotation_items", "items"],
		["other_charges", "taxes"]
	],
	"Purchase Order": [
		["po_details", "items"],
		["other_charges", "taxes"],
		["po_raw_material_details", "supplied_items"]
	],
	"Purchase Receipt": [
		["purchase_receipt_details", "items"],
		["other_charges", "taxes"],
		["pr_raw_material_details", "supplied_items"]
	],
	"Purchase Invoice": [
		["entries", "items"],
		["other_charges", "taxes"],
		["advance_allocation_details", "advances"]
	],
	"Production Order": [
		["production_order_operations", "operations"]
	],
	"BOM": [
		["bom_operations", "operations"],
		["bom_materials", "items"],
		["flat_bom_details", "exploded_items"]
	],
	"Payment Reconciliation": [
		["payment_reconciliation_payments", "payments"],
		["payment_reconciliation_invoices", "invoices"]
	],
	"Sales Taxes and Charges Template": [
		["other_charges", "taxes"],
		["valid_for_territories", "territories"]
	],
	"Purchase Taxes and Charges Template": [
		["other_charges", "taxes"]
	],
	"Shipping Rule": [
		["shipping_rule_conditions", "conditions"],
		["valid_for_territories", "territories"]
	],
	"Price List": [
		["valid_for_territories", "territories"]
	],
	"Appraisal": [
		["appraisal_details", "goals"]
	],
	"Appraisal Template": [
		["kra_sheet", "goals"]
	],
	"Bank Reconciliation": [
		["entries", "journal_entries"]
	],
	"Cost Center": [
		["budget_details", "budgets"]
	],
	"C-Form": [
		["invoice_details", "invoices"]
	],
	"Employee": [
		["employee_leave_approvers", "leave_approvers"],
		["educational_qualification_details", "education"],
		["previous_experience_details", "external_work_history"],
		["experience_in_company_details", "internal_work_history"]
	],
	"Expense Claim": [
		["expense_voucher_details", "expenses"]
	],
	"Fiscal Year": [
		["fiscal_year_companies", "companies"]
	],
	"Holiday List": [
		["holiday_list_details", "holidays"]
	],
	"Installation Note": [
		["installed_item_details", "items"]
	],
	"Item": [
		["item_variants", "variants"],
		["item_reorder", "reorder_levels"],
		["uom_conversion_details", "uoms"],
		["item_supplier_details", "supplier_items"],
		["item_customer_details", "customer_items"],
		["item_tax", "taxes"],
		["item_specification_details", "quality_parameters"],
		["item_website_specifications", "website_specifications"]
	],
	"Item Group": [
		["item_website_specifications", "website_specifications"]
	],
	"Landed Cost Voucher": [
		["landed_cost_purchase_receipts", "purchase_receipts"],
		["landed_cost_items", "items"],
		["landed_cost_taxes_and_charges", "taxes"]
	],
	"Maintenance Schedule": [
		["item_maintenance_detail", "items"],
		["maintenance_schedule_detail", "schedules"]
	],
	"Maintenance Visit": [
		["maintenance_visit_details", "purposes"]
	],
	"Packing Slip": [
		["item_details", "items"]
	],
	"Customer": [
		["party_accounts", "accounts"]
	],
	"Customer Group": [
		["party_accounts", "accounts"]
	],
	"Supplier": [
		["party_accounts", "accounts"]
	],
	"Supplier Type": [
		["party_accounts", "accounts"]
	],
	"Payment Tool": [
		["payment_tool_details", "vouchers"]
	],
	"Production Planning Tool": [
		["pp_so_details", "sales_orders"],
		["pp_details", "items"]
	],
	"Quality Inspection": [
		["qa_specification_details", "readings"]
	],
	"Salary Slip": [
		["earning_details", "earnings"],
		["deduction_details", "deductions"]
	],
	"Salary Structure": [
		["earning_details", "earnings"],
		["deduction_details", "deductions"]
	],
	"Sales BOM": [
		["sales_bom_items", "items"]
	],
	"SMS Settings": [
		["static_parameter_details", "parameters"]
	],
	"Stock Entry": [
		["mtn_details", "items"]
	],
	"Sales Partner": [
		["partner_target_details", "targets"]
	],
	"Sales Person": [
		["target_details", "targets"]
	],
	"Territory": [
		["target_details", "targets"]
	],
	"Time Log Batch": [
		["time_log_batch_details", "time_logs"]
	],
	"Workstation": [
		["workstation_operation_hours", "working_hours"]
	],
	"Payment Reconciliation Payment": [
		["journal_voucher", "journal_entry"],
	],
	"Purchase Invoice Advance": [
		["journal_voucher", "journal_entry"],
	],
	"Sales Invoice Advance": [
		["journal_voucher", "journal_entry"],
	],
	"Journal Entry": [
		["entries", "accounts"]
	],
	"Monthly Distribution": [
		["budget_distribution_details", "percentages"]
	]
}

def execute():
	# rename doctypes
	tables = frappe.db.sql_list("show tables")
	for old_dt, new_dt in [["Journal Voucher Detail", "Journal Entry Account"],
		["Journal Voucher", "Journal Entry"],
		["Budget Distribution Detail", "Monthly Distribution Percentage"],
		["Budget Distribution", "Monthly Distribution"]]:
			if "tab"+new_dt not in tables:
				frappe.rename_doc("DocType", old_dt, new_dt, force=True)

	# reload new child doctypes
	frappe.reload_doc("manufacturing", "doctype", "production_order_operation")
	frappe.reload_doc("manufacturing", "doctype", "workstation_working_hour")
	frappe.reload_doc("stock", "doctype", "item_variant")
	frappe.reload_doc("accounts", "doctype", "party_account")
	frappe.reload_doc("accounts", "doctype", "fiscal_year_company")

	#rename table fieldnames
	if frappe.db.exists("DocType", dn):
		for dn in rename_map:
			frappe.reload_doc(get_doctype_module(dn), "doctype", scrub(dn))

		for dt, field_list in rename_map.items():
			for field in field_list:
				rename_field(dt, field[0], field[1])

	# update voucher type
	for old, new in [["Bank Voucher", "Bank Entry"], ["Cash Voucher", "Cash Entry"],
		["Credit Card Voucher", "Credit Card Entry"], ["Contra Voucher", "Contra Entry"],
		["Write Off Voucher", "Write Off Entry"], ["Excise Voucher", "Excise Entry"]]:
			frappe.db.sql("update `tabJournal Entry` set voucher_type=%s where voucher_type=%s", (new, old))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
from frappe import _

from datetime import timedelta
from frappe.utils import flt, time_diff_in_hours, get_datetime, getdate, cint, get_datetime_str
from frappe.model.document import Document
from frappe.model.mapper import get_mapped_doc
from erpnext.manufacturing.doctype.workstation.workstation import (check_if_within_operating_hours,
	WorkstationHolidayError)
from erpnext.manufacturing.doctype.manufacturing_settings.manufacturing_settings import get_mins_between_operations

class OverlapError(frappe.ValidationError): pass
class OverProductionLoggedError(frappe.ValidationError): pass

class Timesheet(Document):
	def validate(self):
		self.set_status()
		self.validate_dates()
		self.validate_time_logs()
		self.update_cost()
		self.calculate_total_amounts()

	def calculate_total_amounts(self):
		self.total_hours = 0.0
		self.total_billing_amount = 0.0
		self.total_costing_amount = 0.0

		for d in self.get("time_logs"):
			self.total_hours += flt(d.hours)
			if d.billable:
				self.total_billing_amount += flt(d.billing_amount)
				self.total_costing_amount += flt(d.costing_amount)

	def set_status(self):
		self.status = {
			"0": "Draft",
			"1": "Submitted",
			"2": "Cancelled"
		}[str(self.docstatus or 0)]

		if self.sales_invoice:
			self.status = "Billed"

		if self.salary_slip:
			self.status = "Payslip"

		if self.sales_invoice and self.salary_slip:
			self.status = "Completed"

	def set_dates(self):
		if self.docstatus < 2:
			start_date = min([d.from_time for d in self.time_logs])
			end_date = max([d.to_time for d in self.time_logs])

			if start_date and end_date:
				self.start_date = getdate(start_date)
				self.end_date = getdate(end_date)

	def before_submit(self):
		self.set_dates()

	def before_cancel(self):
		self.set_status()

	def on_cancel(self):
		self.update_production_order(None)
		self.update_task_and_project()

	def on_submit(self):
		self.validate_mandatory_fields()
		self.update_production_order(self.name)
		self.update_task_and_project()

	def validate_mandatory_fields(self):
		if self.production_order:
			production_order = frappe.get_doc("Production Order", self.production_order)
			pending_qty = flt(production_order.qty) - flt(production_order.produced_qty)

		for data in self.time_logs:
			if not data.from_time and not data.to_time:
				frappe.throw(_("Row {0}: From Time and To Time is mandatory.").format(data.idx))

			if not data.activity_type and self.employee:
				frappe.throw(_("Row {0}: Activity Type is mandatory.").format(data.idx))

			if flt(data.hours) == 0.0:
				frappe.throw(_("Row {0}: Hours value must be greater than zero.").format(data.idx))

			if self.production_order and flt(data.completed_qty) == 0:
				frappe.throw(_("Row {0}: Completed Qty must be greater than zero.").format(data.idx))

			if self.production_order and flt(pending_qty) < flt(data.completed_qty):
				frappe.throw(_("Row {0}: Completed Qty cannot be more than {1} for operation {2}").format(data.idx, pending_qty, data.operation),
					OverProductionLoggedError)

	def update_production_order(self, time_sheet):
		if self.production_order:
			pro = frappe.get_doc('Production Order', self.production_order)

			for timesheet in self.time_logs:
				for data in pro.operations:
					if data.name == timesheet.operation_id:
						summary = self.get_actual_timesheet_summary(timesheet.operation_id)
						data.time_sheet = time_sheet
						data.completed_qty = summary.completed_qty
						data.actual_operation_time = summary.mins
						data.actual_start_time = summary.from_time
						data.actual_end_time = summary.to_time

			pro.flags.ignore_validate_update_after_submit = True
			pro.update_operation_status()
			pro.calculate_operating_cost()
			pro.set_actual_dates()
			pro.save()

	def get_actual_timesheet_summary(self, operation_id):
		"""Returns 'Actual Operating Time'. """
		return frappe.db.sql("""select
			sum(tsd.hours*60) as mins, sum(tsd.completed_qty) as completed_qty, min(tsd.from_time) as from_time,
			max(tsd.to_time) as to_time from `tabTimesheet Detail` as tsd, `tabTimesheet` as ts where
			ts.production_order = %s and tsd.operation_id = %s and ts.docstatus=1 and ts.name = tsd.parent""",
			(self.production_order, operation_id), as_dict=1)[0]

	def update_task_and_project(self):
		for data in self.time_logs:
			if data.task:
				task = frappe.get_doc("Task", data.task)
				task.update_time_and_costing()
				task.save()

			elif data.project:
				frappe.get_doc("Project", data.project).update_project()

	def validate_dates(self):
		for data in self.time_logs:
			if time_diff_in_hours(data.to_time, data.from_time) < 0:
				frappe.throw(_("To date cannot be before from date"))

	def validate_time_logs(self):
		for data in self.get('time_logs'):
			self.check_workstation_timings(data)
			self.validate_overlap(data)

	def validate_overlap(self, data):
		if self.production_order:
			self.validate_overlap_for("workstation", data, data.workstation)
		else:
			self.validate_overlap_for("user", data, self.user)
			self.validate_overlap_for("employee", data, self.employee)

	def validate_overlap_for(self, fieldname, args, value):
		if not value: return

		existing = self.get_overlap_for(fieldname, args, value)
		if existing:
			frappe.throw(_("Row {0}: From Time and To Time overlap with existing from and to time").format(args.idx),
				OverlapError)

	def get_overlap_for(self, fieldname, args, value):
		cond = "ts.`{0}`".format(fieldname)
		if fieldname == 'workstation':
			cond = "tsd.`{0}`".format(fieldname)

		existing = frappe.db.sql("""select ts.name as name, tsd.from_time as from_time, tsd.to_time as to_time from
			`tabTimesheet Detail` tsd, `tabTimesheet` ts where {0}=%(val)s and tsd.parent = ts.name and
			(
				(%(from_time)s > tsd.from_time and %(from_time)s < tsd.to_time) or
				(%(to_time)s > tsd.from_time and %(to_time)s < tsd.to_time) or
				(%(from_time)s <= tsd.from_time and %(to_time)s >= tsd.to_time))
			and tsd.name!=%(name)s
			and ts.docstatus < 2""".format(cond),
			{
				"val": value,
				"from_time": args.from_time,
				"to_time": args.to_time,
				"name": args.name or "No Name"
			}, as_dict=True)

		return existing[0] if existing else None

	def check_workstation_timings(self, args):
		"""Checks if **Time Log** is between operating hours of the **Workstation**."""
		if args.workstation and args.from_time and args.to_time:
			check_if_within_operating_hours(args.workstation, args.operation, args.from_time, args.to_time)

	def schedule_for_production_order(self, index):
		for data in self.time_logs:
			if data.idx == index:
				self.move_to_next_day(data) #check for workstation holiday
				self.move_to_next_non_overlapping_slot(data) #check for overlap
				break

	def move_to_next_non_overlapping_slot(self, data):
		overlapping = self.get_overlap_for("workstation", data, data.workstation)
		if overlapping:
			time_sheet = self.get_last_working_slot(overlapping.name, data.workstation)
			data.from_time = get_datetime(time_sheet.to_time) + get_mins_between_operations()
			data.to_time = self.get_to_time(data)
			self.check_workstation_working_day(data)

	def get_last_working_slot(self, time_sheet, workstation):
		return frappe.db.sql(""" select max(from_time) as from_time, max(to_time) as to_time
			from `tabTimesheet Detail` where workstation = %(workstation)s""",
			{'workstation': workstation}, as_dict=True)[0]

	def move_to_next_day(self, data):
		"""Move start and end time one day forward"""
		self.check_workstation_working_day(data)

	def check_workstation_working_day(self, data):
		while True:
			try:
				self.check_workstation_timings(data)
				break
			except WorkstationHolidayError:
				if frappe.message_log: frappe.message_log.pop()
				data.from_time = get_datetime(data.from_time) + timedelta(hours=24)
				data.to_time = self.get_to_time(data)

	def get_to_time(self, data):
		return get_datetime(data.from_time) + timedelta(hours=data.hours)

	def update_cost(self):
		for data in self.time_logs:
			if data.activity_type and (not data.billing_amount or not data.costing_amount):
				rate = get_activity_cost(self.employee, data.activity_type)
				hours =  data.hours or 0
				if rate:
					data.billing_rate = flt(rate.get('billing_rate'))
					data.costing_rate = flt(rate.get('costing_rate'))
					data.billing_amount = data.billing_rate * hours
					data.costing_amount = data.costing_rate * hours

@frappe.whitelist()
def make_sales_invoice(source_name, target=None):
	target = frappe.new_doc("Sales Invoice")

	target.append("timesheets", get_mapped_doc("Timesheet", source_name, {
		"Timesheet": {
			"doctype": "Sales Invoice Timesheet",
			"field_map": {
				"total_billing_amount": "billing_amount",
				"name": "time_sheet"
			},
		}
	}))

	target.run_method("calculate_billing_amount_from_timesheet")

	return target

@frappe.whitelist()
def make_salary_slip(source_name, target_doc=None):
	target = frappe.new_doc("Salary Slip")
	set_missing_values(source_name, target)

	target.append("timesheets", get_mapped_doc("Timesheet", source_name, {
		"Timesheet": {
			"doctype": "Salary Slip Timesheet",
			"field_map": {
				"total_hours": "working_hours",
				"name": "time_sheet"
			},
		}
	}))

	target.run_method("get_emp_and_leave_details")

	return target

def set_missing_values(time_sheet, target):
	doc = frappe.get_doc('Timesheet', time_sheet)
	target.employee = doc.employee
	target.employee_name = doc.employee_name
	target.salary_slip_based_on_timesheet = 1
	target.start_date = doc.start_date
	target.end_date = doc.end_date

@frappe.whitelist()
def get_activity_cost(employee=None, activity_type=None):
	rate = frappe.db.get_values("Activity Cost", {"employee": employee,
		"activity_type": activity_type}, ["costing_rate", "billing_rate"], as_dict=True)
	if not rate:
		rate = frappe.db.get_values("Activity Type", {"activity_type": activity_type},
			["costing_rate", "billing_rate"], as_dict=True)

	return rate[0] if rate else {}

@frappe.whitelist()
def get_employee_list(doctype, txt, searchfield, start, page_len, filters):
	test = frappe.db.sql("""select distinct employee, employee_name
		from `tabSalary Structure` where salary_slip_based_on_timesheet=1
		and employee like %(txt)s limit %(start)s, %(page_len)s""",
		{'txt': "%%%s%%"% txt, '_txt': txt.replace("%", ""), 'start': start, 'page_len': page_len})
	return test<commit_msg>removed test variable<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
from frappe import _

from datetime import timedelta
from frappe.utils import flt, time_diff_in_hours, get_datetime, getdate, cint, get_datetime_str
from frappe.model.document import Document
from frappe.model.mapper import get_mapped_doc
from erpnext.manufacturing.doctype.workstation.workstation import (check_if_within_operating_hours,
	WorkstationHolidayError)
from erpnext.manufacturing.doctype.manufacturing_settings.manufacturing_settings import get_mins_between_operations

class OverlapError(frappe.ValidationError): pass
class OverProductionLoggedError(frappe.ValidationError): pass

class Timesheet(Document):
	def validate(self):
		self.set_status()
		self.validate_dates()
		self.validate_time_logs()
		self.update_cost()
		self.calculate_total_amounts()

	def calculate_total_amounts(self):
		self.total_hours = 0.0
		self.total_billing_amount = 0.0
		self.total_costing_amount = 0.0

		for d in self.get("time_logs"):
			self.total_hours += flt(d.hours)
			if d.billable:
				self.total_billing_amount += flt(d.billing_amount)
				self.total_costing_amount += flt(d.costing_amount)

	def set_status(self):
		self.status = {
			"0": "Draft",
			"1": "Submitted",
			"2": "Cancelled"
		}[str(self.docstatus or 0)]

		if self.sales_invoice:
			self.status = "Billed"

		if self.salary_slip:
			self.status = "Payslip"

		if self.sales_invoice and self.salary_slip:
			self.status = "Completed"

	def set_dates(self):
		if self.docstatus < 2:
			start_date = min([d.from_time for d in self.time_logs])
			end_date = max([d.to_time for d in self.time_logs])

			if start_date and end_date:
				self.start_date = getdate(start_date)
				self.end_date = getdate(end_date)

	def before_submit(self):
		self.set_dates()

	def before_cancel(self):
		self.set_status()

	def on_cancel(self):
		self.update_production_order(None)
		self.update_task_and_project()

	def on_submit(self):
		self.validate_mandatory_fields()
		self.update_production_order(self.name)
		self.update_task_and_project()

	def validate_mandatory_fields(self):
		if self.production_order:
			production_order = frappe.get_doc("Production Order", self.production_order)
			pending_qty = flt(production_order.qty) - flt(production_order.produced_qty)

		for data in self.time_logs:
			if not data.from_time and not data.to_time:
				frappe.throw(_("Row {0}: From Time and To Time is mandatory.").format(data.idx))

			if not data.activity_type and self.employee:
				frappe.throw(_("Row {0}: Activity Type is mandatory.").format(data.idx))

			if flt(data.hours) == 0.0:
				frappe.throw(_("Row {0}: Hours value must be greater than zero.").format(data.idx))

			if self.production_order and flt(data.completed_qty) == 0:
				frappe.throw(_("Row {0}: Completed Qty must be greater than zero.").format(data.idx))

			if self.production_order and flt(pending_qty) < flt(data.completed_qty):
				frappe.throw(_("Row {0}: Completed Qty cannot be more than {1} for operation {2}").format(data.idx, pending_qty, data.operation),
					OverProductionLoggedError)

	def update_production_order(self, time_sheet):
		if self.production_order:
			pro = frappe.get_doc('Production Order', self.production_order)

			for timesheet in self.time_logs:
				for data in pro.operations:
					if data.name == timesheet.operation_id:
						summary = self.get_actual_timesheet_summary(timesheet.operation_id)
						data.time_sheet = time_sheet
						data.completed_qty = summary.completed_qty
						data.actual_operation_time = summary.mins
						data.actual_start_time = summary.from_time
						data.actual_end_time = summary.to_time

			pro.flags.ignore_validate_update_after_submit = True
			pro.update_operation_status()
			pro.calculate_operating_cost()
			pro.set_actual_dates()
			pro.save()

	def get_actual_timesheet_summary(self, operation_id):
		"""Returns 'Actual Operating Time'. """
		return frappe.db.sql("""select
			sum(tsd.hours*60) as mins, sum(tsd.completed_qty) as completed_qty, min(tsd.from_time) as from_time,
			max(tsd.to_time) as to_time from `tabTimesheet Detail` as tsd, `tabTimesheet` as ts where
			ts.production_order = %s and tsd.operation_id = %s and ts.docstatus=1 and ts.name = tsd.parent""",
			(self.production_order, operation_id), as_dict=1)[0]

	def update_task_and_project(self):
		for data in self.time_logs:
			if data.task:
				task = frappe.get_doc("Task", data.task)
				task.update_time_and_costing()
				task.save()

			elif data.project:
				frappe.get_doc("Project", data.project).update_project()

	def validate_dates(self):
		for data in self.time_logs:
			if time_diff_in_hours(data.to_time, data.from_time) < 0:
				frappe.throw(_("To date cannot be before from date"))

	def validate_time_logs(self):
		for data in self.get('time_logs'):
			self.check_workstation_timings(data)
			self.validate_overlap(data)

	def validate_overlap(self, data):
		if self.production_order:
			self.validate_overlap_for("workstation", data, data.workstation)
		else:
			self.validate_overlap_for("user", data, self.user)
			self.validate_overlap_for("employee", data, self.employee)

	def validate_overlap_for(self, fieldname, args, value):
		if not value: return

		existing = self.get_overlap_for(fieldname, args, value)
		if existing:
			frappe.throw(_("Row {0}: From Time and To Time overlap with existing from and to time").format(args.idx),
				OverlapError)

	def get_overlap_for(self, fieldname, args, value):
		cond = "ts.`{0}`".format(fieldname)
		if fieldname == 'workstation':
			cond = "tsd.`{0}`".format(fieldname)

		existing = frappe.db.sql("""select ts.name as name, tsd.from_time as from_time, tsd.to_time as to_time from
			`tabTimesheet Detail` tsd, `tabTimesheet` ts where {0}=%(val)s and tsd.parent = ts.name and
			(
				(%(from_time)s > tsd.from_time and %(from_time)s < tsd.to_time) or
				(%(to_time)s > tsd.from_time and %(to_time)s < tsd.to_time) or
				(%(from_time)s <= tsd.from_time and %(to_time)s >= tsd.to_time))
			and tsd.name!=%(name)s
			and ts.docstatus < 2""".format(cond),
			{
				"val": value,
				"from_time": args.from_time,
				"to_time": args.to_time,
				"name": args.name or "No Name"
			}, as_dict=True)

		return existing[0] if existing else None

	def check_workstation_timings(self, args):
		"""Checks if **Time Log** is between operating hours of the **Workstation**."""
		if args.workstation and args.from_time and args.to_time:
			check_if_within_operating_hours(args.workstation, args.operation, args.from_time, args.to_time)

	def schedule_for_production_order(self, index):
		for data in self.time_logs:
			if data.idx == index:
				self.move_to_next_day(data) #check for workstation holiday
				self.move_to_next_non_overlapping_slot(data) #check for overlap
				break

	def move_to_next_non_overlapping_slot(self, data):
		overlapping = self.get_overlap_for("workstation", data, data.workstation)
		if overlapping:
			time_sheet = self.get_last_working_slot(overlapping.name, data.workstation)
			data.from_time = get_datetime(time_sheet.to_time) + get_mins_between_operations()
			data.to_time = self.get_to_time(data)
			self.check_workstation_working_day(data)

	def get_last_working_slot(self, time_sheet, workstation):
		return frappe.db.sql(""" select max(from_time) as from_time, max(to_time) as to_time
			from `tabTimesheet Detail` where workstation = %(workstation)s""",
			{'workstation': workstation}, as_dict=True)[0]

	def move_to_next_day(self, data):
		"""Move start and end time one day forward"""
		self.check_workstation_working_day(data)

	def check_workstation_working_day(self, data):
		while True:
			try:
				self.check_workstation_timings(data)
				break
			except WorkstationHolidayError:
				if frappe.message_log: frappe.message_log.pop()
				data.from_time = get_datetime(data.from_time) + timedelta(hours=24)
				data.to_time = self.get_to_time(data)

	def get_to_time(self, data):
		return get_datetime(data.from_time) + timedelta(hours=data.hours)

	def update_cost(self):
		for data in self.time_logs:
			if data.activity_type and (not data.billing_amount or not data.costing_amount):
				rate = get_activity_cost(self.employee, data.activity_type)
				hours =  data.hours or 0
				if rate:
					data.billing_rate = flt(rate.get('billing_rate'))
					data.costing_rate = flt(rate.get('costing_rate'))
					data.billing_amount = data.billing_rate * hours
					data.costing_amount = data.costing_rate * hours

@frappe.whitelist()
def make_sales_invoice(source_name, target=None):
	target = frappe.new_doc("Sales Invoice")

	target.append("timesheets", get_mapped_doc("Timesheet", source_name, {
		"Timesheet": {
			"doctype": "Sales Invoice Timesheet",
			"field_map": {
				"total_billing_amount": "billing_amount",
				"name": "time_sheet"
			},
		}
	}))

	target.run_method("calculate_billing_amount_from_timesheet")

	return target

@frappe.whitelist()
def make_salary_slip(source_name, target_doc=None):
	target = frappe.new_doc("Salary Slip")
	set_missing_values(source_name, target)

	target.append("timesheets", get_mapped_doc("Timesheet", source_name, {
		"Timesheet": {
			"doctype": "Salary Slip Timesheet",
			"field_map": {
				"total_hours": "working_hours",
				"name": "time_sheet"
			},
		}
	}))

	target.run_method("get_emp_and_leave_details")

	return target

def set_missing_values(time_sheet, target):
	doc = frappe.get_doc('Timesheet', time_sheet)
	target.employee = doc.employee
	target.employee_name = doc.employee_name
	target.salary_slip_based_on_timesheet = 1
	target.start_date = doc.start_date
	target.end_date = doc.end_date

@frappe.whitelist()
def get_activity_cost(employee=None, activity_type=None):
	rate = frappe.db.get_values("Activity Cost", {"employee": employee,
		"activity_type": activity_type}, ["costing_rate", "billing_rate"], as_dict=True)
	if not rate:
		rate = frappe.db.get_values("Activity Type", {"activity_type": activity_type},
			["costing_rate", "billing_rate"], as_dict=True)

	return rate[0] if rate else {}

@frappe.whitelist()
def get_employee_list(doctype, txt, searchfield, start, page_len, filters):
	return frappe.db.sql("""select distinct employee, employee_name
		from `tabSalary Structure` where salary_slip_based_on_timesheet=1
		and employee like %(txt)s limit %(start)s, %(page_len)s""",
		{'txt': "%%%s%%"% txt, '_txt': txt.replace("%", ""), 'start': start, 'page_len': page_len})<|endoftext|>


<commit_before>#
# Copyright (c) 2015 Juniper Networks, Inc. All rights reserved.
#
from snmp import SnmpSession
import copy, traceback, time, ConfigParser
from vnc_api.vnc_api import VncApi

class DeviceConfig(object):
    __pat = None

    def __init__(self, name, cfg={}, mibs=[], flow_export_source_ip=None):
        self._raw = cfg
        self.name = name
        self.mibs = mibs or SnmpSession.TABLES() #all
        self.flow_export_source_ip = flow_export_source_ip
        self.snmp_name = None

    def set_snmp_name(self, name):
        self.snmp_name = name

    def get_snmp_name(self):
        return self.snmp_name

    def set_mibs(self, mib):
        if mib in SnmpSession.TABLES() and mib not in self.mibs:
            self.mibs.append(mib)

    def get_mibs(self):
        return self.mibs

    def get_flow_export_source_ip(self):
        return self.flow_export_source_ip

    def snmp_cfg(self):
        cfg = copy.copy(self._raw)
        cfg['DestHost'] = self.name
        cfg['Version'] = int(cfg['Version'])
        return cfg

    @classmethod
    def _pat(self):
        if self.__pat is None:
           self.__pat = re.compile(', *| +')
        return self.__pat

    @staticmethod
    def _get_and_remove_key(data_dict, key, default=None):
        val = default
        if key in data_dict:
            val = data_dict[key]
            del data_dict[key]
        return val

    @classmethod
    def _mklist(self, s):
        if isinstance(s, str):
            return self._pat().split(s)
        return s

    @staticmethod
    def fom_file(filename):
        devices = []
        devcfg = ConfigParser.SafeConfigParser()
        devcfg.optionxform = str
        devcfg.read([filename])
        for dev in devcfg.sections():
            nd = dict(devcfg.items(dev))
            mibs = DeviceConfig._mklist(DeviceConfig._get_and_remove_key(nd,
                        'Mibs', []))
            flow_export_source_ip = DeviceConfig._get_and_remove_key(nd,
                    'FlowExportSourceIp')
            devices.append(DeviceConfig(dev, nd, mibs,
                        flow_export_source_ip))
        return devices

    @staticmethod
    def get_vnc(usr, passwd, tenant, api_servers, auth_host=None,
            auth_port=None, auth_protocol=None, notifycb=None):
        for rt in (5, 2, 7, 9, 16, 25):
          for api_server in api_servers:
            srv = api_server.split(':')
            if len(srv) == 2:
                ip, port = srv[0], int(srv[1])
            else:
                ip, port = '127.0.0.1', int(srv[0])
            try:
                vnc = VncApi(usr, passwd, tenant, ip, port,
                        auth_host=auth_host, auth_port=auth_port,
                        auth_protocol=auth_protocol)
                if callable(notifycb):
                    notifycb('api', 'Connected', servers=api_server)
                return vnc
            except Exception as e:
                traceback.print_exc()
                if callable(notifycb):
                    notifycb('api', 'Not connected', servers=api_server,
                            up=False)
                time.sleep(rt)
        raise e

    @staticmethod
    def fom_api_server(api_servers, usr, passwd, tenant, auth_host=None,
            auth_port=None, auth_protocol=None, notifycb=None):
        devices = []

        vnc = DeviceConfig.get_vnc(usr, passwd, tenant, api_servers,
                auth_host=auth_host, auth_port=auth_port,
                auth_protocol=auth_protocol, notifycb=notifycb)
        for x in vnc.physical_routers_list()['physical-routers']:
            pr = vnc.physical_router_read(id=x['uuid'])
            snmp = pr.get_physical_router_snmp_credentials()
            if snmp:
                nd = { 'Version': snmp.get_version() }
                if snmp.get_local_port():
                    nd['RemotePort'] = snmp.get_local_port()
                if snmp.get_retries():
                    nd['Retries'] = snmp.get_retries()
                if snmp.get_timeout():
                    nd['Timeout'] = snmp.get_timeout()
                if nd['Version'] in (1, 2):
                    nd['Community'] = snmp.get_v2_community()
                elif nd['Version'] == 3:
                    if snmp.get_v3_security_name():
                        nd['SecName'] = snmp.get_v3_security_name()
                    if snmp.get_v3_security_level():
                        nd['SecLevel'] = snmp.get_v3_security_level()
                    if snmp.get_v3_security_engine_id():
                        nd['SecEngineId'] = snmp.get_v3_security_engine_id()
                    if snmp.get_v3_context():
                        nd['Context'] = snmp.get_v3_context()
                    if snmp.get_v3_context_engine_id():
                        nd['ContextEngineId'] = snmp.get_v3_context_engine_id()
                    if snmp.get_v3_authentication_protocol():
                        nd['AuthProto'] = snmp.get_v3_authentication_protocol()
                    if snmp.get_v3_authentication_password():
                        nd['AuthPass'] = snmp.get_v3_authentication_password()
                    if snmp.get_v3_privacy_protocol():
                        nd['PrivProto'] = snmp.get_v3_privacy_protocol()
                    if snmp.get_v3_privacy_password():
                        nd['PrivProto'] = snmp.get_v3_privacy_password()
                devices.append(DeviceConfig(
                            pr.get_physical_router_management_ip(),
                            nd, [], pr.get_physical_router_dataplane_ip()))
        return devices
<commit_msg>change the flow export ip to management ip<commit_after>#
# Copyright (c) 2015 Juniper Networks, Inc. All rights reserved.
#
from snmp import SnmpSession
import copy, traceback, time, ConfigParser
from vnc_api.vnc_api import VncApi

class DeviceConfig(object):
    __pat = None

    def __init__(self, name, cfg={}, mibs=[], flow_export_source_ip=None):
        self._raw = cfg
        self.name = name
        self.mibs = mibs or SnmpSession.TABLES() #all
        self.flow_export_source_ip = flow_export_source_ip
        self.snmp_name = None

    def set_snmp_name(self, name):
        self.snmp_name = name

    def get_snmp_name(self):
        return self.snmp_name

    def set_mibs(self, mib):
        if mib in SnmpSession.TABLES() and mib not in self.mibs:
            self.mibs.append(mib)

    def get_mibs(self):
        return self.mibs

    def get_flow_export_source_ip(self):
        return self.flow_export_source_ip

    def snmp_cfg(self):
        cfg = copy.copy(self._raw)
        cfg['DestHost'] = self.name
        cfg['Version'] = int(cfg['Version'])
        return cfg

    @classmethod
    def _pat(self):
        if self.__pat is None:
           self.__pat = re.compile(', *| +')
        return self.__pat

    @staticmethod
    def _get_and_remove_key(data_dict, key, default=None):
        val = default
        if key in data_dict:
            val = data_dict[key]
            del data_dict[key]
        return val

    @classmethod
    def _mklist(self, s):
        if isinstance(s, str):
            return self._pat().split(s)
        return s

    @staticmethod
    def fom_file(filename):
        devices = []
        devcfg = ConfigParser.SafeConfigParser()
        devcfg.optionxform = str
        devcfg.read([filename])
        for dev in devcfg.sections():
            nd = dict(devcfg.items(dev))
            mibs = DeviceConfig._mklist(DeviceConfig._get_and_remove_key(nd,
                        'Mibs', []))
            flow_export_source_ip = DeviceConfig._get_and_remove_key(nd,
                    'FlowExportSourceIp')
            devices.append(DeviceConfig(dev, nd, mibs,
                        flow_export_source_ip))
        return devices

    @staticmethod
    def get_vnc(usr, passwd, tenant, api_servers, auth_host=None,
            auth_port=None, auth_protocol=None, notifycb=None):
        for rt in (5, 2, 7, 9, 16, 25):
          for api_server in api_servers:
            srv = api_server.split(':')
            if len(srv) == 2:
                ip, port = srv[0], int(srv[1])
            else:
                ip, port = '127.0.0.1', int(srv[0])
            try:
                vnc = VncApi(usr, passwd, tenant, ip, port,
                        auth_host=auth_host, auth_port=auth_port,
                        auth_protocol=auth_protocol)
                if callable(notifycb):
                    notifycb('api', 'Connected', servers=api_server)
                return vnc
            except Exception as e:
                traceback.print_exc()
                if callable(notifycb):
                    notifycb('api', 'Not connected', servers=api_server,
                            up=False)
                time.sleep(rt)
        raise e

    @staticmethod
    def fom_api_server(api_servers, usr, passwd, tenant, auth_host=None,
            auth_port=None, auth_protocol=None, notifycb=None):
        devices = []

        vnc = DeviceConfig.get_vnc(usr, passwd, tenant, api_servers,
                auth_host=auth_host, auth_port=auth_port,
                auth_protocol=auth_protocol, notifycb=notifycb)
        for x in vnc.physical_routers_list()['physical-routers']:
            pr = vnc.physical_router_read(id=x['uuid'])
            snmp = pr.get_physical_router_snmp_credentials()
            if snmp:
                nd = { 'Version': snmp.get_version() }
                if snmp.get_local_port():
                    nd['RemotePort'] = snmp.get_local_port()
                if snmp.get_retries():
                    nd['Retries'] = snmp.get_retries()
                if snmp.get_timeout():
                    nd['Timeout'] = snmp.get_timeout()
                if nd['Version'] in (1, 2):
                    nd['Community'] = snmp.get_v2_community()
                elif nd['Version'] == 3:
                    if snmp.get_v3_security_name():
                        nd['SecName'] = snmp.get_v3_security_name()
                    if snmp.get_v3_security_level():
                        nd['SecLevel'] = snmp.get_v3_security_level()
                    if snmp.get_v3_security_engine_id():
                        nd['SecEngineId'] = snmp.get_v3_security_engine_id()
                    if snmp.get_v3_context():
                        nd['Context'] = snmp.get_v3_context()
                    if snmp.get_v3_context_engine_id():
                        nd['ContextEngineId'] = snmp.get_v3_context_engine_id()
                    if snmp.get_v3_authentication_protocol():
                        nd['AuthProto'] = snmp.get_v3_authentication_protocol()
                    if snmp.get_v3_authentication_password():
                        nd['AuthPass'] = snmp.get_v3_authentication_password()
                    if snmp.get_v3_privacy_protocol():
                        nd['PrivProto'] = snmp.get_v3_privacy_protocol()
                    if snmp.get_v3_privacy_password():
                        nd['PrivProto'] = snmp.get_v3_privacy_password()
                devices.append(DeviceConfig(
                            pr.get_physical_router_management_ip(),
                            nd, [], pr.get_physical_router_management_ip()))
        return devices
<|endoftext|>