<commit_before># from hypothesis import composite
from hypothesis import given
from hypothesis.strategies import (
    text, integers, lists, composite, uuids, sampled_from, none, one_of,
    permutations)

RESOURCES = [
    'unit',
    'organization',
    'department',
    'ontologyword',
    'ontologytree',
]

def int_keys(draw):
    return draw(lists(
        integers(min_value=1, max_value=1000000),
        min_size=1, max_size=3, unique=True))

def uuid_keys(draw):
    return draw(lists(uuids(), min_size=2, max_size=3))

def translated_field(draw, name, allow_none=True, languages=['fi', 'sv', 'en']):
    result = {}
    for lang in languages:
        if allow_none:
            val = draw(one_of(text(), none()))
        else:
            val = draw(text())
        result['name_{}'.format(lang)] = val
    return result

VIEWPOINTS = ['00', '11', '12', '13', '21', '22', '23', '31', '32', '33', '41',
              '51', '52', '61']

VIEWPOINT_STATES = ['green', 'red', 'unknown']

def accessibility_viewpoints(draw):
    return ','.join([
        '{}:{}'.format(key, draw(sampled_from(VIEWPOINT_STATES))) for key in VIEWPOINTS])

PROVIDER_TYPES = ["CONTRACT_SCHOOL",
                  "OTHER_PRODUCTION_METHOD",
                  "PAYMENT_COMMITMENT",
                  "PURCHASED_SERVICE",
                  "SELF_PRODUCED",
                  "SUPPORTED_OPERATIONS",
                  "UNKNOWN_PRODUCTION_METHOD"]

@composite
def make_source(draw):
    return {'id': draw(text()), 'source': draw(text())}

def unit_maker(draw, resource_ids):
    def make_unit(uid):
        result = {
            'id': uid,
            'org_id': str(draw(sampled_from(resource_ids['organization']))),
            'dept_id': str(draw(sampled_from(resource_ids['department']))),
            'ontologyword_ids': draw(permutations(resource_ids['ontologyword'])),
            'ontologytree_ids': draw(permutations(resource_ids['ontologytree'])),
            'accessibility_viewpoints': accessibility_viewpoints(draw),
            'sources': draw(lists(make_source(), min_size=0, max_size=2)),
            'provider_type': draw(sampled_from(PROVIDER_TYPES)),
            'accessibility_email': draw(one_of(text(), none()))  # TODO: map elsewhere?
        }
        result.update(translated_field(draw, 'name', allow_none=False))
        return result
    return make_unit

def organization_maker(*args):
    return lambda x: {'id': str(x)}

def department_maker(draw, resource_ids):
    def make_department(did):
        return {
            'id': str(did),
            'hierarchy_level': 0,
            'org_id': str(draw(sampled_from(resource_ids['organization'])))
        }
    return make_department

def ontologyword_maker(*args):
    return lambda x: {'id': x}

def ontologytree_maker(*args):
    return lambda x: {'id': x}

make_resource = {}
for r in RESOURCES:
    make_resource[r] = locals()['{}_maker'.format(r)]

@composite
def closed_object_set(draw):
    ids = {
        'unit': int_keys(draw),
        'organization': uuid_keys(draw),
        'department': uuid_keys(draw),
        'ontologyword': int_keys(draw),
        'ontologytree': int_keys(draw)
    }
    resources = {}
    for key, identifiers in ids.items():
        resources[key] = list(map(make_resource[key](draw, ids), ids[key]))
    return resources

@given(closed_object_set())
def test_something(cos):
    assert True
<commit_msg>Make id range smaller<commit_after># from hypothesis import composite
from hypothesis import given
from hypothesis.strategies import (
    text, integers, lists, composite, uuids, sampled_from, none, one_of,
    permutations)

RESOURCES = [
    'unit',
    'organization',
    'department',
    'ontologyword',
    'ontologytree',
]

def int_keys(draw):
    return draw(lists(
        integers(min_value=1, max_value=15),
        min_size=1, max_size=3, unique=True))

def uuid_keys(draw):
    return draw(lists(uuids(), min_size=2, max_size=3))

def translated_field(draw, name, allow_none=True, languages=['fi', 'sv', 'en']):
    result = {}
    for lang in languages:
        if allow_none:
            val = draw(one_of(text(), none()))
        else:
            val = draw(text())
        result['name_{}'.format(lang)] = val
    return result

VIEWPOINTS = ['00', '11', '12', '13', '21', '22', '23', '31', '32', '33', '41',
              '51', '52', '61']

VIEWPOINT_STATES = ['green', 'red', 'unknown']

def accessibility_viewpoints(draw):
    return ','.join([
        '{}:{}'.format(key, draw(sampled_from(VIEWPOINT_STATES))) for key in VIEWPOINTS])

PROVIDER_TYPES = ["CONTRACT_SCHOOL",
                  "OTHER_PRODUCTION_METHOD",
                  "PAYMENT_COMMITMENT",
                  "PURCHASED_SERVICE",
                  "SELF_PRODUCED",
                  "SUPPORTED_OPERATIONS",
                  "UNKNOWN_PRODUCTION_METHOD"]

@composite
def make_source(draw):
    return {'id': draw(text()), 'source': draw(text())}

def unit_maker(draw, resource_ids):
    def make_unit(uid):
        result = {
            'id': uid,
            'org_id': str(draw(sampled_from(resource_ids['organization']))),
            'dept_id': str(draw(sampled_from(resource_ids['department']))),
            'ontologyword_ids': draw(permutations(resource_ids['ontologyword'])),
            'ontologytree_ids': draw(permutations(resource_ids['ontologytree'])),
            'accessibility_viewpoints': accessibility_viewpoints(draw),
            'sources': draw(lists(make_source(), min_size=0, max_size=2)),
            'provider_type': draw(sampled_from(PROVIDER_TYPES)),
            'accessibility_email': draw(one_of(text(), none()))  # TODO: map to another field
        }
        result.update(translated_field(draw, 'name', allow_none=False))
        return result
    return make_unit

def organization_maker(*args):
    return lambda x: {'id': str(x)}

def department_maker(draw, resource_ids):
    def make_department(did):
        return {
            'id': str(did),
            'hierarchy_level': 0,
            'org_id': str(draw(sampled_from(resource_ids['organization'])))
        }
    return make_department

def ontologyword_maker(*args):
    return lambda x: {'id': x}

def ontologytree_maker(*args):
    return lambda x: {'id': x}

make_resource = {}
for r in RESOURCES:
    make_resource[r] = locals()['{}_maker'.format(r)]

@composite
def closed_object_set(draw):
    ids = {
        'unit': int_keys(draw),
        'organization': uuid_keys(draw),
        'department': uuid_keys(draw),
        'ontologyword': int_keys(draw),
        'ontologytree': int_keys(draw)
    }
    resources = {}
    for key, identifiers in ids.items():
        resources[key] = list(map(make_resource[key](draw, ids), ids[key]))
    return resources

@given(closed_object_set())
def test_something(cos):
    assert True
<|endoftext|>


<commit_before>from pygfnn.network import Model, make_connections
from pygfnn.oscillator import Zparam
from pygfnn.gfnn import GFNN

import numpy as np

# load onset signal
# odf = np.loadtxt('sampleOnsets/rumba.onset.txt')
odf = np.loadtxt('sampleOnsets/bossa.onset.txt')
fs_odf = odf[0]
odf = odf[1:]*0.5   # this factor is relevant (but annoying)
t_odf = np.arange(0, odf.size)
t_odf = t_odf/fs_odf

# create a pair of GFNNs
params1 = Zparam(0, -1.0, -0.25, 0, 0, .5)
params2 = Zparam(0.03, 1.0, -1.0, 0, 0, .5)

internal_strength = 0.6
internal_stdev = 0.5

center_freq = 2.0
half_range =  2
oscs_per_octave = 64

layer1 = GFNN(params1,
              fc=center_freq,
              octaves_per_side=half_range,
              oscs_per_octave=oscs_per_octave)

layer2 = GFNN(params2,
              fc=center_freq,
              octaves_per_side=half_range,
              oscs_per_octave=oscs_per_octave)

# add internal connectivities
rels = [1./3., 1./2., 1., 2., 3.]
layer1.connect_internally(relations=rels,
                          internal_strength=internal_strength,
                          internal_stdev=internal_stdev,
                          complex_kernel=True)

layer2.connect_internally(relations=rels,
                          internal_strength=internal_strength,
                          internal_stdev=internal_stdev,
                          complex_kernel=True)

# create the model
net = Model()
net.add_layer(layer1)
net.add_layer(layer2, visible=False)

# add inter-layer connections
affConnStrength = 0.75;
affConnStdDev = 0.5;
effConnStrength = -0.75;
effConnStdDev = 4.0;

affConn = affConnStrength * make_connections (layer1,
                                              layer2,
                                              harmonics=[1],
                                              stdev=affConnStdDev,
                                              complex_kernel=False,
                                              self_connect=True,
                                              conn_type='gauss')

effConn = effConnStrength * make_connections (layer2,
                                              layer1,
                                              harmonics=[1],
                                              stdev=effConnStdDev,
                                              complex_kernel=True,
                                              self_connect=False,
                                              conn_type='gauss')

net.connect_layers(layer1, layer2, affConn)
net.connect_layers(layer2, layer1, effConn)

# run the model
net.run(odf, t_odf, 1.0/fs_odf)

# visualize outputs
TF = layer2.TF
f = layer1.f
T = 1.0/f

plot_onset_signal = False
plot_internal_conns = False
plot_tf_output = True

if plot_onset_signal:
    import matplotlib.pyplot as plt
    plt.plot(t_odf, odf)
    plt.show()

if plot_internal_conns:
    from pygfnn.vis import plot_connections
    plot_connections(effConn, layer1.f, layer2.f)

if plot_tf_output:
    # from pygfnn.vis import tf_simple
    # tf_simple(TF, t_odf, T, odf, np.abs)
    # tf_simple(TF, t_odf, T, None, np.abs)
    from pygfnn.vis import tf_detail
    tf_detail(TF, t_odf, T, np.max(t_odf), odf, np.abs)

<commit_msg>timing execution<commit_after>from pygfnn.network import Model, make_connections
from pygfnn.oscillator import Zparam
from pygfnn.gfnn import GFNN
from pygfnn.utils import timed

import numpy as np

# load onset signal
# odf = np.loadtxt('sampleOnsets/rumba.onset.txt')
odf = np.loadtxt('sampleOnsets/bossa.onset.txt')
fs_odf = odf[0]
odf = odf[1:]*0.5   # this factor is relevant (but annoying)
t_odf = np.arange(0, odf.size)
t_odf = t_odf/fs_odf

# create a pair of GFNNs
params1 = Zparam(0, -1.0, -0.25, 0, 0, .5)
params2 = Zparam(0.03, 1.0, -1.0, 0, 0, .5)

internal_strength = 0.6
internal_stdev = 0.5

center_freq = 2.0
half_range =  2
oscs_per_octave = 64

layer1 = GFNN(params1,
              fc=center_freq,
              octaves_per_side=half_range,
              oscs_per_octave=oscs_per_octave)

layer2 = GFNN(params2,
              fc=center_freq,
              octaves_per_side=half_range,
              oscs_per_octave=oscs_per_octave)

# add internal connectivities
rels = [1./3., 1./2., 1., 2., 3.]
layer1.connect_internally(relations=rels,
                          internal_strength=internal_strength,
                          internal_stdev=internal_stdev,
                          complex_kernel=True)

layer2.connect_internally(relations=rels,
                          internal_strength=internal_strength,
                          internal_stdev=internal_stdev,
                          complex_kernel=True)

# create the model
net = Model()
net.add_layer(layer1)
net.add_layer(layer2, visible=False)

# add inter-layer connections
affConnStrength = 0.75;
affConnStdDev = 0.5;
effConnStrength = -0.75;
effConnStdDev = 4.0;

affConn = affConnStrength * make_connections (layer1,
                                              layer2,
                                              harmonics=[1],
                                              stdev=affConnStdDev,
                                              complex_kernel=False,
                                              self_connect=True,
                                              conn_type='gauss')

effConn = effConnStrength * make_connections (layer2,
                                              layer1,
                                              harmonics=[1],
                                              stdev=effConnStdDev,
                                              complex_kernel=True,
                                              self_connect=False,
                                              conn_type='gauss')

net.connect_layers(layer1, layer2, affConn)
net.connect_layers(layer2, layer1, effConn)

# run the model
@timed
def run():
  net.run(odf, t_odf, 1.0/fs_odf)

run()

# visualize outputs
TF = layer2.TF
f = layer2.f
T = 1.0/f

plot_onset_signal = False
plot_internal_conns = False
plot_tf_output = True

if plot_onset_signal:
    import matplotlib.pyplot as plt
    plt.plot(t_odf, odf)
    plt.show()

if plot_internal_conns:
    from pygfnn.vis import plot_connections
    plot_connections(effConn, layer1.f, layer2.f)

if plot_tf_output:
    # from pygfnn.vis import tf_simple
    # tf_simple(TF, t_odf, T, odf, np.abs)
    # tf_simple(TF, t_odf, T, None, np.abs)
    from pygfnn.vis import tf_detail
    tf_detail(TF, t_odf, T, np.max(t_odf), odf, np.abs)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from selenium.webdriver.firefox.webdriver import WebDriver
from selenium.webdriver.common.action_chains import ActionChains
import time, unittest

def is_alert_present(wd):
    try:
        wd.switch_to_alert().text
        return True
    except:
        return False

class test_add_group(unittest.TestCase):
    def setUp(self):
        self.wd = WebDriver()
        self.wd.implicitly_wait(60)

    def test_test_add_group(self):
        success = True
        wd = self.wd
        wd.get("http://localhost/addressbook/group.php")
        wd.find_element_by_name("user").click()
        wd.find_element_by_name("user").clear()
        wd.find_element_by_name("user").send_keys("admin")
        wd.find_element_by_id("LoginForm").click()
        wd.find_element_by_name("pass").click()
        wd.find_element_by_name("pass").clear()
        wd.find_element_by_name("pass").send_keys("secret")
        wd.find_element_by_xpath("//form[@id='LoginForm']/input[3]").click()
        wd.find_element_by_link_text("grupy").click()
        wd.find_element_by_name("new").click()
        wd.find_element_by_name("group_name").click()
        wd.find_element_by_name("group_name").clear()
        wd.find_element_by_name("group_name").send_keys("test")
        wd.find_element_by_name("group_header").click()
        wd.find_element_by_name("group_header").clear()
        wd.find_element_by_name("group_header").send_keys("test")
        wd.find_element_by_name("group_footer").click()
        wd.find_element_by_name("group_footer").clear()
        wd.find_element_by_name("group_footer").send_keys("test")
        wd.find_element_by_name("submit").click()
        wd.find_element_by_link_text("group page").click()
        wd.find_element_by_link_text("Wyloguj się").click()
        self.assertTrue(success)

    def tearDown(self):
        self.wd.quit()

if __name__ == '__main__':
    unittest.main()
<commit_msg>test_add<commit_after># -*- coding: utf-8 -*-
from selenium.webdriver.firefox.webdriver import WebDriver
from selenium.webdriver.common.action_chains import ActionChains
import time, unittest

def is_alert_present(wd):
    try:
        wd.switch_to_alert().text
        return True
    except:
        return False

class test_add_group(unittest.TestCase):
    def setUp(self):
        self.wd = WebDriver()
        self.wd.implicitly_wait(60)

    def test_test_add_group(self):
        success = True
        wd = self.wd
        wd.get("http://localhost/addressbook/group.php")
        wd.find_element_by_name("user").click()
        wd.find_element_by_name("user").clear()
        wd.find_element_by_name("user").send_keys("admin")
        wd.find_element_by_name("pass").click()
        wd.find_element_by_name("pass").clear()
        wd.find_element_by_name("pass").send_keys("secret")
        wd.find_element_by_xpath("//form[@id='LoginForm']/input[3]").click()
        wd.find_element_by_link_text("grupy").click()
        wd.find_element_by_name("new").click()
        wd.find_element_by_name("group_name").click()
        wd.find_element_by_name("group_name").clear()
        wd.find_element_by_name("group_name").send_keys("test python")
        wd.find_element_by_name("group_header").click()
        wd.find_element_by_name("group_header").clear()
        wd.find_element_by_name("group_header").send_keys("test")
        wd.find_element_by_name("group_footer").click()
        wd.find_element_by_name("group_footer").clear()
        wd.find_element_by_name("group_footer").send_keys("test")
        wd.find_element_by_xpath("//div[@id='content']/form").click()
        wd.find_element_by_name("submit").click()
        wd.find_element_by_link_text("group page").click()
        wd.find_element_by_link_text("Wyloguj się").click()
        self.assertTrue(success)

    def tearDown(self):
        self.wd.quit()

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# See LICENSE file for copyright and license details

''' Test 'generator' module. '''

import unittest
import ast
import misc
import generator
import table

class TestGenerator(unittest.TestCase):
    ''' Test 'Generator' class. '''

    def test_1(self):
        ''' Generate some code. '''
        ast_ = ast.Module(
            import_list=['stdio', 'ogre3d'],
            declaration_sequence=[
                ast.FunctionDeclaration(
                    name='f1',
                    interface=ast.FunctionInterface(parameter_list=[]),
                    body=[
                        ast.VariableDeclaration(
                            expression=ast.FunctionCall(
                                expression=ast.Identifier('plus'),
                                argument_list=[
                                    ast.Number(1),
                                    ast.Number(2),
                                ]
                            ),
                            name='a',
                            type=ast.Identifier('int'),
                        ),
                    ]
                ),
                ast.FunctionDeclaration(
                    name='f2',
                    interface=ast.FunctionInterface(parameter_list=[]),
                    body=[
                        ast.VariableDeclaration(
                            expression=ast.FunctionCall(
                                expression=ast.Identifier('x'),
                                argument_list=[]
                            ),
                            name='a',
                            type=ast.Identifier('int'),
                        ),
                    ]
                ),
                ast.FunctionDeclaration(
                    name='main',
                    interface=ast.FunctionInterface(parameter_list=[]),
                    body=[],
                ),
            ]
        )
        gen = generator.Generator(
            table=table.Table.from_ast(ast_),
        )
        # print('\n' + my_pretty_print(gen))
        real_output = gen.generate()
        expected_output = (
            '// import: stdio\n'
            '// import: ogre3d\n'
            '\n'
            'void f1(void);\n'
            'void f2(void);\n'
            'void main(void);\n'
            '\n'
            'void f1(void) {\n'
            '  int tmp_0;\n'
            '  int a;\n'
            '\n'
            '  plus(&tmp_0, 1, 2);\n'
            '  a = tmp_0;\n'
            '}\n'
            '\n'
            'void f2(void) {\n'
            '  int tmp_0;\n'
            '  int a;\n'
            '\n'
            '  x(&tmp_0);\n'
            '  a = tmp_0;\n'
            '}\n'
            '\n'
            'void main(void) {\n'
            '\n'
            '}\n'
            '\n'
        )
        misc.assert_equal(self, expected_output, real_output)

    def test_nested_func_calls(self):
        ''' Generate nested function calls. '''
        ast_ = ast.Module(
            import_list=[],
            declaration_sequence=[
                ast.FunctionDeclaration(
                    name='main',
                    interface=ast.FunctionInterface(parameter_list=[]),
                    body=[
                        ast.FunctionCall(
                            expression=ast.Identifier('a'),
                            argument_list=[
                                ast.FunctionCall(
                                    expression=ast.Identifier('b'),
                                    argument_list=[]
                                ),
                            ]
                        ),
                    ]
                )
            ]
        )
        gen = generator.Generator(
            table=table.Table.from_ast(ast_),
        )
        real_output = gen.generate()
        expected_output = (
            '\n'
            'void main(void);\n'
            '\n'
            'void main(void) {\n'
            '  int tmp_0;\n'
            '  int tmp_1;\n'
            '\n'
            '  b(&tmp_1);\n'
            '  a(&tmp_0, tmp_1);\n'
            '}\n'
            '\n'
        )
        misc.assert_equal(self, expected_output, real_output)

    def test_2(self):
        ''' Generate some simple code. '''
        ast_ = ast.Module(
            declaration_sequence=[
                ast.FunctionDeclaration(
                    name='f1',
                    interface=ast.FunctionInterface(
                        parameter_list=[
                            ast.Parameter(
                                name='a',
                                type=ast.Identifier('int')
                            ),
                            ast.Parameter(
                                name='b',
                                type=ast.Identifier('int')
                            ),
                        ],
                        return_type=ast.Identifier('int'),
                    ),
                    body=[]
                ),
            ]
        )
        gen = generator.Generator(
            table=table.Table.from_ast(ast_),
        )
        # print('\n' + my_pretty_print(gen))
        real_output = gen.generate()
        expected_output = (
            '\n'
            'void f1(int* __result, int a, int b);\n'
            '\n'
            'void f1(int* __result, int a, int b) {\n'
            '\n'
            '}\n'
            '\n'
        )
        misc.assert_equal(self, expected_output, real_output)

    def test_if(self):
        ''' Test if. '''
        func = table.Function(
            name='main',
            interface=ast.FunctionInterface(
                return_type=None,
                parameter_list=[],
            ),
            constant_list=[],
            variable_list=[
                table.Variable(type='int', name='tmp_0'),
                table.Variable(type='int', name='tmp_1'),
            ],
            block_list=[
                [
                    table.IfStatement(
                        expression_id=table.LinkToFunctionCall(id=0),
                        if_branch_id=1,
                    ),
                ],
                [
                    table.FunctionCallStatement(
                        expression_id=table.LinkToFunctionCall(id=1)),
                    table.FunctionCallStatement(
                        expression_id=table.LinkToFunctionCall(id=2)),
                ],
            ],
            expression_list=[
                table.FunctionCallExpression(
                    result_id=table.LinkToVariable(id=0),
                    name='f1',
                    argument_id_list=[],
                ),
                table.FunctionCallExpression(
                    result_id=table.LinkToVariable(id=1),
                    name='f2',
                    argument_id_list=[],
                ),
                table.FunctionCallExpression(
                    result_id=table.LinkToVariable(id=1),
                    name='f3',
                    argument_id_list=[],
                ),
            ]
        )
        table_ = table.Table(
            declaration_list=[func],
            identifier_list=[],
            import_list=[],
        )

        gen = generator.Generator(table=table_)
        real_output = gen.generate()
        expected_output = (
            '\n'
            'void main(void);\n'
            '\n'
            'void main(void) {\n'
            '  int tmp_0;\n'
            '  int tmp_1;\n'
            '\n'
            '  f1(&tmp_0);\n'
            '  if (tmp_0) {\n'
            '    f2(&tmp_1);\n'
            '    f3(&tmp_1);\n'
            '  }\n'
            '}\n'
            '\n'
        )
        misc.assert_equal(self, expected_output, real_output)

    def test_return_integer_constant(self):
        ''' Test returning integer constant. '''
        func = table.Function(
            name='main',
            interface=ast.FunctionInterface(
                return_type=None,
                parameter_list=[],
            ),
            constant_list=[
                table.Constant(type="int", value=1),
            ],
            variable_list=[],
            block_list=[
                [
                    table.ReturnStatement(
                        expression_id=table.LinkToNumberConstant(id=0),
                    ),
                ],
            ],
            expression_list=[],
        )
        table_ = table.Table(
            declaration_list=[func],
            identifier_list=[],
            import_list=[],
        )

        gen = generator.Generator(table=table_)
        real_output = gen.generate()
        expected_output = (
            '\n'
            'void main(void);\n'
            '\n'
            'void main(void) {\n'
            '\n'
            '  *__result = 1;\n'
            '  return;\n'
            '}\n'
            '\n'
        )
        misc.assert_equal(self, expected_output, real_output)

    def test_return_function_call_result(self):
        ''' Test returning function call result. '''
        func = table.Function(
            name='main',
            interface=ast.FunctionInterface(
                return_type=None,
                parameter_list=[],
            ),
            constant_list=[],
            variable_list=[
                table.Variable(type='int', name='tmp_0'),
            ],
            block_list=[
                [
                    table.ReturnStatement(
                        expression_id=table.LinkToFunctionCall(id=0)),
                ],
            ],
            expression_list=[
                table.FunctionCallExpression(
                    result_id=table.LinkToVariable(id=0),
                    name='f',
                    argument_id_list=[],
                ),
            ],
        )
        table_ = table.Table(
            declaration_list=[func],
            identifier_list=[],
            import_list=[],
        )

        gen = generator.Generator(table=table_)
        real_output = gen.generate()
        expected_output = (
            '\n'
            'void main(void);\n'
            '\n'
            'void main(void) {\n'
            '  int tmp_0;\n'
            '\n'
            '  f(&tmp_0);\n'
            '  *__result = tmp_0;\n'
            '  return;\n'
            '}\n'
            '\n'
        )
        misc.assert_equal(self, expected_output, real_output)

# vim: set tabstop=4 shiftwidth=4 softtabstop=4 expandtab:
<commit_msg>test_generator.py: Removed functional test from file with unit tests<commit_after># -*- coding: utf-8 -*-
# See LICENSE file for copyright and license details

''' Test 'generator' module. '''

import unittest
import ast
import misc
import generator
import table

class TestGenerator(unittest.TestCase):
    ''' Test 'Generator' class. '''

    def test_nested_func_calls(self):
        ''' Generate nested function calls. '''
        ast_ = ast.Module(
            import_list=[],
            declaration_sequence=[
                ast.FunctionDeclaration(
                    name='main',
                    interface=ast.FunctionInterface(parameter_list=[]),
                    body=[
                        ast.FunctionCall(
                            expression=ast.Identifier('a'),
                            argument_list=[
                                ast.FunctionCall(
                                    expression=ast.Identifier('b'),
                                    argument_list=[]
                                ),
                            ]
                        ),
                    ]
                )
            ]
        )
        gen = generator.Generator(
            table=table.Table.from_ast(ast_),
        )
        real_output = gen.generate()
        expected_output = (
            '\n'
            'void main(void);\n'
            '\n'
            'void main(void) {\n'
            '  int tmp_0;\n'
            '  int tmp_1;\n'
            '\n'
            '  b(&tmp_1);\n'
            '  a(&tmp_0, tmp_1);\n'
            '}\n'
            '\n'
        )
        misc.assert_equal(self, expected_output, real_output)

    def test_2(self):
        ''' Generate some simple code. '''
        ast_ = ast.Module(
            declaration_sequence=[
                ast.FunctionDeclaration(
                    name='f1',
                    interface=ast.FunctionInterface(
                        parameter_list=[
                            ast.Parameter(
                                name='a',
                                type=ast.Identifier('int')
                            ),
                            ast.Parameter(
                                name='b',
                                type=ast.Identifier('int')
                            ),
                        ],
                        return_type=ast.Identifier('int'),
                    ),
                    body=[]
                ),
            ]
        )
        gen = generator.Generator(
            table=table.Table.from_ast(ast_),
        )
        # print('\n' + my_pretty_print(gen))
        real_output = gen.generate()
        expected_output = (
            '\n'
            'void f1(int* __result, int a, int b);\n'
            '\n'
            'void f1(int* __result, int a, int b) {\n'
            '\n'
            '}\n'
            '\n'
        )
        misc.assert_equal(self, expected_output, real_output)

    def test_if(self):
        ''' Test if. '''
        func = table.Function(
            name='main',
            interface=ast.FunctionInterface(
                return_type=None,
                parameter_list=[],
            ),
            constant_list=[],
            variable_list=[
                table.Variable(type='int', name='tmp_0'),
                table.Variable(type='int', name='tmp_1'),
            ],
            block_list=[
                [
                    table.IfStatement(
                        expression_id=table.LinkToFunctionCall(id=0),
                        if_branch_id=1,
                    ),
                ],
                [
                    table.FunctionCallStatement(
                        expression_id=table.LinkToFunctionCall(id=1)),
                    table.FunctionCallStatement(
                        expression_id=table.LinkToFunctionCall(id=2)),
                ],
            ],
            expression_list=[
                table.FunctionCallExpression(
                    result_id=table.LinkToVariable(id=0),
                    name='f1',
                    argument_id_list=[],
                ),
                table.FunctionCallExpression(
                    result_id=table.LinkToVariable(id=1),
                    name='f2',
                    argument_id_list=[],
                ),
                table.FunctionCallExpression(
                    result_id=table.LinkToVariable(id=1),
                    name='f3',
                    argument_id_list=[],
                ),
            ]
        )
        table_ = table.Table(
            declaration_list=[func],
            identifier_list=[],
            import_list=[],
        )

        gen = generator.Generator(table=table_)
        real_output = gen.generate()
        expected_output = (
            '\n'
            'void main(void);\n'
            '\n'
            'void main(void) {\n'
            '  int tmp_0;\n'
            '  int tmp_1;\n'
            '\n'
            '  f1(&tmp_0);\n'
            '  if (tmp_0) {\n'
            '    f2(&tmp_1);\n'
            '    f3(&tmp_1);\n'
            '  }\n'
            '}\n'
            '\n'
        )
        misc.assert_equal(self, expected_output, real_output)

    def test_return_integer_constant(self):
        ''' Test returning integer constant. '''
        func = table.Function(
            name='main',
            interface=ast.FunctionInterface(
                return_type=None,
                parameter_list=[],
            ),
            constant_list=[
                table.Constant(type="int", value=1),
            ],
            variable_list=[],
            block_list=[
                [
                    table.ReturnStatement(
                        expression_id=table.LinkToNumberConstant(id=0),
                    ),
                ],
            ],
            expression_list=[],
        )
        table_ = table.Table(
            declaration_list=[func],
            identifier_list=[],
            import_list=[],
        )

        gen = generator.Generator(table=table_)
        real_output = gen.generate()
        expected_output = (
            '\n'
            'void main(void);\n'
            '\n'
            'void main(void) {\n'
            '\n'
            '  *__result = 1;\n'
            '  return;\n'
            '}\n'
            '\n'
        )
        misc.assert_equal(self, expected_output, real_output)

    def test_return_function_call_result(self):
        ''' Test returning function call result. '''
        func = table.Function(
            name='main',
            interface=ast.FunctionInterface(
                return_type=None,
                parameter_list=[],
            ),
            constant_list=[],
            variable_list=[
                table.Variable(type='int', name='tmp_0'),
            ],
            block_list=[
                [
                    table.ReturnStatement(
                        expression_id=table.LinkToFunctionCall(id=0)),
                ],
            ],
            expression_list=[
                table.FunctionCallExpression(
                    result_id=table.LinkToVariable(id=0),
                    name='f',
                    argument_id_list=[],
                ),
            ],
        )
        table_ = table.Table(
            declaration_list=[func],
            identifier_list=[],
            import_list=[],
        )

        gen = generator.Generator(table=table_)
        real_output = gen.generate()
        expected_output = (
            '\n'
            'void main(void);\n'
            '\n'
            'void main(void) {\n'
            '  int tmp_0;\n'
            '\n'
            '  f(&tmp_0);\n'
            '  *__result = tmp_0;\n'
            '  return;\n'
            '}\n'
            '\n'
        )
        misc.assert_equal(self, expected_output, real_output)

# vim: set tabstop=4 shiftwidth=4 softtabstop=4 expandtab:
<|endoftext|>


<commit_before>#
# This file is part of LiteX-Boards.
#
# Copyright (c) 2019 Antti Lukats <antti.lukats@gmail.com>
# Copyright (c) 2019 Florent Kermarrec <florent@enjoy-digital.fr>
# SPDX-License-Identifier: BSD-2-Clause

from litex.build.generic_platform import *
from litex.build.altera import AlteraPlatform
from litex.build.altera.programmer import USBBlaster

# IOs ----------------------------------------------------------------------------------------------

_io = [
    # Clk / Rst
    ("clk12", 0, Pins("G21"),  IOStandard("3.3-V LVTTL")),
    ("clk25", 0, Pins("AA12"), IOStandard("3.3-V LVTTL")),
    ("cpu_reset", 0, Pins("V15"), IOStandard("3.3-V LVTTL")),

    # Leds
    ("user_led", 0, Pins("C18"), IOStandard("3.3-V LVTTL")),
    ("user_led", 1, Pins("D19"), IOStandard("3.3-V LVTTL")),
    ("user_led", 2, Pins("C19"), IOStandard("3.3-V LVTTL")),
    ("user_led", 3, Pins("C17"), IOStandard("3.3-V LVTTL")),
    ("user_led", 4, Pins("D18"), IOStandard("3.3-V LVTTL")),

    # Switches
    ("sw", 0, Pins("U10"), IOStandard("3.3-V LVTTL")),
    ("sw", 1, Pins("U11"), IOStandard("3.3-V LVTTL")),
    ("sw", 2, Pins("V11"), IOStandard("3.3-V LVTTL")),
    ("sw", 3, Pins("T10"), IOStandard("3.3-V LVTTL")),
    ("sw", 4, Pins("T11"), IOStandard("3.3-V LVTTL")),

    # Serial
    ("serial", 0,
        Subsignal("tx", Pins("B21"), IOStandard("3.3-V LVTTL")),
        Subsignal("rx", Pins("C20"), IOStandard("3.3-V LVTTL")),
    ),

    # SDR SDRAM
    ("sdram_clock", 0, Pins("AA3"), IOStandard("3.3-V LVTTL")),
    ("sdram", 0,
        Subsignal("a", Pins(
            "V5   Y3 W6  Y4 AB5 AB6 AA6 AA7",
            "AB7 AA5 V6 AA8 AB8")),
        Subsignal("ba",    Pins("Y6 V7")),
        Subsignal("cs_n",  Pins("W7")),
        Subsignal("cke",   Pins("AA4")),
        Subsignal("ras_n", Pins("V8")),
        Subsignal("cas_n", Pins("Y7")),
        Subsignal("we_n",  Pins("W8")),
        Subsignal("dq", Pins(
            "AB16 Y17 AA16 AA19 AB18 AA20 AB19 AB20",
            "Y13  Y15 AA13 AB15 AB13 AA15 AA14 AB14")),
        Subsignal("dm", Pins("Y14 W13")),
        IOStandard("3.3-V LVTTL")
    ),

    # ECPS
    ("epcs", 0,
        Subsignal("data0", Pins("K1")),
        Subsignal("dclk",  Pins("K2")),
        Subsignal("ncs0",  Pins("E2")),
        Subsignal("asd0",  Pins("D1")),
        IOStandard("3.3-V LVTTL")
    ),

    # HyperRAM
    ("hyperram", 0,
        Subsignal("clk",   Pins("T16")),
        Subsignal("rst_n", Pins("U12")),
        Subsignal("dq",    Pins("T15 W17 U14 R15 R14 V16 U16 U17")),
        Subsignal("cs_n",  Pins("V13")),
        Subsignal("rwds",  Pins("U13")),
        IOStandard("3.3-V LVTTL")
    ),

    # GPIO Leds
    ("gpio_leds", 0,
        Pins("AB10 AA10 AA9 Y10 W10 U9 U8 U7"),
        IOStandard("3.3-V LVTTL")
    ),

    # MII Ethernet
    ("eth_clocks", 0,
        Subsignal("tx", Pins("U21")),
        Subsignal("rx", Pins("V22")),
        IOStandard("3.3-V LVTTL"),
    ),
    ("eth", 0,
        Subsignal("rst_n",   Pins("R19")),
        Subsignal("mdio",    Pins("AA21")),
        Subsignal("mdc",     Pins("AA22")),
        Subsignal("rx_dv",   Pins("W21")),
        Subsignal("rx_er",   Pins("V21")),
        Subsignal("rx_data", Pins("W22 W20 Y21 Y22")),
        Subsignal("tx_en",   Pins("T18")),
        Subsignal("tx_data", Pins("T17 U20 U19 T20")),
        Subsignal("col",     Pins("T19")),
        Subsignal("crs",     Pins("R20")),
        IOStandard("3.3-V LVTTL"),
    ),
    ("eth_clocks", 1,
        Subsignal("tx", Pins("N16")),
        Subsignal("rx", Pins("V22")),
        IOStandard("3.3-V LVTTL"),
    ),
    ("eth", 1,
        Subsignal("rst_n",   Pins("M21")),
        Subsignal("mdio",    Pins("N20")),
        Subsignal("mdc",     Pins("N18")),
        Subsignal("rx_dv",   Pins("R18")),
        Subsignal("rx_er",   Pins("P17")),
        Subsignal("rx_data", Pins("M20 M19 M16 N19")),
        Subsignal("tx_en",   Pins("R22")),
        Subsignal("tx_data", Pins("R21 N21 M22 N22")),
        Subsignal("col",     Pins("P21")),
        Subsignal("crs",     Pins("P22")),
        IOStandard("3.3-V LVTTL"),
    ),
]

# Platform -----------------------------------------------------------------------------------------

class Platform(AlteraPlatform):
    default_clk_name   = "clk12"
    default_clk_period = 1e9/12e6

    def __init__(self):
        AlteraPlatform.__init__(self, "10CL055YU484A7G", _io)

    def create_programmer(self):
        return USBBlaster()

    def do_finalize(self, fragment):
        AlteraPlatform.do_finalize(self, fragment)
        self.add_period_constraint(self.lookup_request("clk12", loose=True), 1e9/12e6)
        self.add_period_constraint(self.lookup_request("clk25", loose=True), 1e9/25e6)
<commit_msg>Use Arrow-USB-Blaster for trenz c10lprefkit<commit_after>#
# This file is part of LiteX-Boards.
#
# Copyright (c) 2019 Antti Lukats <antti.lukats@gmail.com>
# Copyright (c) 2019 Florent Kermarrec <florent@enjoy-digital.fr>
# SPDX-License-Identifier: BSD-2-Clause

from litex.build.generic_platform import *
from litex.build.altera import AlteraPlatform
from litex.build.altera.programmer import USBBlaster

# IOs ----------------------------------------------------------------------------------------------

_io = [
    # Clk / Rst
    ("clk12", 0, Pins("G21"),  IOStandard("3.3-V LVTTL")),
    ("clk25", 0, Pins("AA12"), IOStandard("3.3-V LVTTL")),
    ("cpu_reset", 0, Pins("V15"), IOStandard("3.3-V LVTTL")),

    # Leds
    ("user_led", 0, Pins("C18"), IOStandard("3.3-V LVTTL")),
    ("user_led", 1, Pins("D19"), IOStandard("3.3-V LVTTL")),
    ("user_led", 2, Pins("C19"), IOStandard("3.3-V LVTTL")),
    ("user_led", 3, Pins("C17"), IOStandard("3.3-V LVTTL")),
    ("user_led", 4, Pins("D18"), IOStandard("3.3-V LVTTL")),

    # Switches
    ("sw", 0, Pins("U10"), IOStandard("3.3-V LVTTL")),
    ("sw", 1, Pins("U11"), IOStandard("3.3-V LVTTL")),
    ("sw", 2, Pins("V11"), IOStandard("3.3-V LVTTL")),
    ("sw", 3, Pins("T10"), IOStandard("3.3-V LVTTL")),
    ("sw", 4, Pins("T11"), IOStandard("3.3-V LVTTL")),

    # Serial
    ("serial", 0,
        Subsignal("tx", Pins("B21"), IOStandard("3.3-V LVTTL")),
        Subsignal("rx", Pins("C20"), IOStandard("3.3-V LVTTL")),
    ),

    # SDR SDRAM
    ("sdram_clock", 0, Pins("AA3"), IOStandard("3.3-V LVTTL")),
    ("sdram", 0,
        Subsignal("a", Pins(
            "V5   Y3 W6  Y4 AB5 AB6 AA6 AA7",
            "AB7 AA5 V6 AA8 AB8")),
        Subsignal("ba",    Pins("Y6 V7")),
        Subsignal("cs_n",  Pins("W7")),
        Subsignal("cke",   Pins("AA4")),
        Subsignal("ras_n", Pins("V8")),
        Subsignal("cas_n", Pins("Y7")),
        Subsignal("we_n",  Pins("W8")),
        Subsignal("dq", Pins(
            "AB16 Y17 AA16 AA19 AB18 AA20 AB19 AB20",
            "Y13  Y15 AA13 AB15 AB13 AA15 AA14 AB14")),
        Subsignal("dm", Pins("Y14 W13")),
        IOStandard("3.3-V LVTTL")
    ),

    # ECPS
    ("epcs", 0,
        Subsignal("data0", Pins("K1")),
        Subsignal("dclk",  Pins("K2")),
        Subsignal("ncs0",  Pins("E2")),
        Subsignal("asd0",  Pins("D1")),
        IOStandard("3.3-V LVTTL")
    ),

    # HyperRAM
    ("hyperram", 0,
        Subsignal("clk",   Pins("T16")),
        Subsignal("rst_n", Pins("U12")),
        Subsignal("dq",    Pins("T15 W17 U14 R15 R14 V16 U16 U17")),
        Subsignal("cs_n",  Pins("V13")),
        Subsignal("rwds",  Pins("U13")),
        IOStandard("3.3-V LVTTL")
    ),

    # GPIO Leds
    ("gpio_leds", 0,
        Pins("AB10 AA10 AA9 Y10 W10 U9 U8 U7"),
        IOStandard("3.3-V LVTTL")
    ),

    # MII Ethernet
    ("eth_clocks", 0,
        Subsignal("tx", Pins("U21")),
        Subsignal("rx", Pins("V22")),
        IOStandard("3.3-V LVTTL"),
    ),
    ("eth", 0,
        Subsignal("rst_n",   Pins("R19")),
        Subsignal("mdio",    Pins("AA21")),
        Subsignal("mdc",     Pins("AA22")),
        Subsignal("rx_dv",   Pins("W21")),
        Subsignal("rx_er",   Pins("V21")),
        Subsignal("rx_data", Pins("W22 W20 Y21 Y22")),
        Subsignal("tx_en",   Pins("T18")),
        Subsignal("tx_data", Pins("T17 U20 U19 T20")),
        Subsignal("col",     Pins("T19")),
        Subsignal("crs",     Pins("R20")),
        IOStandard("3.3-V LVTTL"),
    ),
    ("eth_clocks", 1,
        Subsignal("tx", Pins("N16")),
        Subsignal("rx", Pins("V22")),
        IOStandard("3.3-V LVTTL"),
    ),
    ("eth", 1,
        Subsignal("rst_n",   Pins("M21")),
        Subsignal("mdio",    Pins("N20")),
        Subsignal("mdc",     Pins("N18")),
        Subsignal("rx_dv",   Pins("R18")),
        Subsignal("rx_er",   Pins("P17")),
        Subsignal("rx_data", Pins("M20 M19 M16 N19")),
        Subsignal("tx_en",   Pins("R22")),
        Subsignal("tx_data", Pins("R21 N21 M22 N22")),
        Subsignal("col",     Pins("P21")),
        Subsignal("crs",     Pins("P22")),
        IOStandard("3.3-V LVTTL"),
    ),
]

# Platform -----------------------------------------------------------------------------------------

class Platform(AlteraPlatform):
    default_clk_name   = "clk12"
    default_clk_period = 1e9/12e6

    def __init__(self):
        AlteraPlatform.__init__(self, "10CL055YU484A7G", _io)

    def create_programmer(self):
        return USBBlaster(cable_name="Arrow-USB-Blaster")

    def do_finalize(self, fragment):
        AlteraPlatform.do_finalize(self, fragment)
        self.add_period_constraint(self.lookup_request("clk12", loose=True), 1e9/12e6)
        self.add_period_constraint(self.lookup_request("clk25", loose=True), 1e9/25e6)
<|endoftext|>


<commit_before>import errno
import imp
import json
import logging
import multiprocessing
import os
import pickle
import re
import requests
import socket
import subprocess
import sys
import tempfile
from string import Template

from django.conf import settings

from loomengine.utils.connection import Connection
import loomengine.utils.version
import loomengine.utils.cloud

PLAYBOOKS_PATH = os.path.join(imp.find_module('loomengine')[1], 'playbooks')
GCLOUD_CREATE_WORKER_PLAYBOOK = os.path.join(PLAYBOOKS_PATH, 'gcloud_create_worker.yml')
GCLOUD_RUN_TASK_PLAYBOOK = os.path.join(PLAYBOOKS_PATH, 'gcloud_run_task.yml')
GCE_PY_PATH = os.path.join(imp.find_module('loomengine')[1], 'utils', 'gce.py')

logger = logging.getLogger(__name__)

class CloudTaskManager:

    @classmethod
    def run(cls, task_run):
        from api.models.task_runs import TaskRunAttempt
        task_run_attempt = TaskRunAttempt.create_from_task_run(task_run)

        # Don't want to block while waiting for VM to come up, so start another process to finish the rest of the steps.
        logger.debug("Launching CloudTaskManager as a separate process.")

        requested_resources = {
            'cores': task_run_attempt.task_run.step_run.resources.cores,
            'memory': task_run_attempt.task_run.step_run.resources.memory,
            'disk_size': task_run_attempt.task_run.step_run.resources.disk_size,
        }
        environment = {
            'docker_image': task_run_attempt.task_definition.environment.docker_image,
        }

        hostname = socket.gethostname()
        worker_name = cls.create_worker_name(hostname, task_run_attempt)
        task_run_attempt_id = task_run_attempt.id.hex
        worker_log_file = task_run_attempt.get_worker_log_file()

        task_run_attempt.status = task_run_attempt.STATUSES.PROVISIONING_HOST
        task_run_attempt.save()

        process = multiprocessing.Process(target=CloudTaskManager._try_run, args=(task_run_attempt_id, requested_resources, environment, worker_name, worker_log_file))
        process.start()

    @classmethod
    def _try_run(cls, task_run_attempt_id, requested_resources, environment, worker_name, worker_log_file):
        try:
            cls._run(task_run_attempt_id, requested_resources, environment, worker_name, worker_log_file)
        except Exception as e:
            logger.exception('Failed to create task run attempt %s: %s' % (task_run_attempt_id, str(e)))

    @classmethod
    def _run(cls, task_run_attempt_id, requested_resources, environment, worker_name, worker_log_file):
        from api.models.task_runs import TaskRunAttempt

        logger.debug("CloudTaskManager separate process started.")
        logger.debug("task_run_attempt: %s" % task_run_attempt_id)

        connection = Connection(settings.MASTER_URL_FOR_SERVER)

        """Create a VM, deploy Docker and Loom, and pass command to task runner."""
        if settings.WORKER_TYPE != 'GOOGLE_CLOUD':
            raise CloudTaskManagerError('Unsupported cloud type: ' + settings.WORKER_TYPE)
        # TODO: Support other cloud providers. For now, assume GCE.
        instance_type = CloudTaskManager._get_cheapest_instance_type(cores=requested_resources['cores'], memory=requested_resources['memory'])

        scratch_disk_name = worker_name+'-disk'
        scratch_disk_device_path = '/dev/disk/by-id/google-'+scratch_disk_name
        if requested_resources.get('disk_size') is not None:
            scratch_disk_size_gb = requested_resources['disk_size']
        else:
            scratch_disk_size_gb = settings.WORKER_SCRATCH_DISK_SIZE

        playbook_vars = {
            'boot_disk_type': settings.WORKER_BOOT_DISK_TYPE,
            'boot_disk_size_gb': settings.WORKER_BOOT_DISK_SIZE,
            'docker_full_name': settings.DOCKER_FULL_NAME,
            'docker_tag': settings.DOCKER_TAG,
            'gce_email': settings.GCE_EMAIL,
            'gce_credential': settings.GCE_PEM_FILE_PATH,
            'gce_ssh_key_file': settings.GCE_SSH_KEY_FILE,
            'instance_name': worker_name,
            'instance_image': settings.WORKER_VM_IMAGE,
            'instance_type': instance_type,
            'log_level': settings.LOG_LEVEL,
            'master_url': settings.MASTER_URL_FOR_WORKER,
            'network': settings.WORKER_NETWORK,
            'remote_user': 'loom',
            'scratch_disk_name': scratch_disk_name,
            'scratch_disk_device_path': scratch_disk_device_path,
            'scratch_disk_mount_point': settings.WORKER_SCRATCH_DISK_MOUNT_POINT,
            'scratch_disk_type': settings.WORKER_SCRATCH_DISK_TYPE,
            'scratch_disk_size_gb': scratch_disk_size_gb,
            'subnetwork': settings.WORKER_CUSTOM_SUBNET,
            'tags': settings.WORKER_TAGS,
            'task_run_attempt_id': task_run_attempt_id,
            'task_run_docker_image': environment['docker_image'],
            'use_internal_ip': settings.WORKER_USES_SERVER_INTERNAL_IP,
            'worker_log_file': worker_log_file,
            'zone': settings.WORKER_LOCATION,
        }
        logger.debug('Starting worker VM using playbook vars: %s' % playbook_vars)

        try:
            with open(os.path.join(settings.LOGS_DIR, 'loom_ansible.log'), 'a', 0) as ansible_logfile:
                cls._run_playbook(GCLOUD_CREATE_WORKER_PLAYBOOK, playbook_vars, logfile=ansible_logfile)
        except Exception as e:
            logger.exception('Failed to provision host.')
            connection.post_task_run_attempt_error({
                'message': 'Failed to provision host',
                'detail': str(e)
            })
            connection.update_task_run_attempt(
                task_run_attempt_id,
                {
                    'status': TaskRunAttempt.STATUSES.FINISHED,
                })
            raise e

        try:
            with open(os.path.join(settings.LOGS_DIR, 'loom_ansible.log'), 'a', 0) as ansible_logfile:
                connection.update_task_run_attempt(
                    task_run_attempt_id,
                    {
                        'status': TaskRunAttempt.STATUSES.LAUNCHING_MONITOR,
                    })
                cls._run_playbook(GCLOUD_RUN_TASK_PLAYBOOK, playbook_vars, logfile=ansible_logfile)
        except Exception as e:
            logger.exception('Failed to launch monitor process on worker: %s')
            connection.post_task_run_attempt_error({
                'message': 'Failed to launch monitor process on worker',
                'detail': str(e)
            })
            connection.update_task_run_attempt(
                task_run_attempt_id,
                {
                    'status': TaskRunAttempt.STATUSES.FINISHED,
                })
            raise e

        logger.debug("CloudTaskManager process done.")
        ansible_logfile.close()

    @classmethod
    def _run_playbook(cls, playbook, playbook_vars, logfile=None):
        """ Runs a playbook by passing it a dict of vars on the command line."""
        ansible_env = os.environ.copy()
        ansible_env['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
        ansible_env['INVENTORY_IP_TYPE'] = 'internal'       # Tell gce.py to use internal IP for ansible_ssh_host
        playbook_vars_json_string = json.dumps(playbook_vars)
        cmd = ['ansible-playbook', '-vvv', '--key-file', os.path.expanduser(settings.GCE_SSH_KEY_FILE), '-i', GCE_PY_PATH, playbook, '--extra-vars', playbook_vars_json_string]
        returncode = subprocess.call(cmd, env=ansible_env, stderr=subprocess.STDOUT, stdout=logfile)
        if not returncode == 0:
            raise Exception('Nonzero returncode %s for command: %s' % (returncode, ' '.join(cmd)))

    @classmethod
    def _get_cheapest_instance_type(cls, cores, memory):
        """ Determine the cheapest instance type given a minimum number of cores and minimum amount of RAM (in GB). """

        if settings.WORKER_TYPE != 'GOOGLE_CLOUD': #TODO: support other cloud providers
            raise CloudTaskManagerError('Not a recognized cloud provider: ' + settings.WORKER_TYPE)
        else:
            pricelist = CloudTaskManager._get_gcloud_pricelist()

            # Filter out preemptible, shared-CPU, and non-US instance types
            us_instance_types = {k:v for k,v in pricelist.items()\
                if k.startswith('CP-COMPUTEENGINE-VMIMAGE-') and not k.endswith('-PREEMPTIBLE') and 'us' in v and v['cores'] != 'shared'}

            # Convert to array and add keys (instance type names) as type names
            price_array = []
            for key in us_instance_types:
                value = us_instance_types[key]
                value.update({'name':key.replace('CP-COMPUTEENGINE-VMIMAGE-', '').lower()})
                price_array.append(value)

            # Sort by price in US
            price_array.sort(None, lambda x: x['us'])

            # Look for an instance type that satisfies requested cores and memory; first will be cheapest
            for instance_type in price_array:
                if int(instance_type['cores']) >= int(cores) and float(instance_type['memory']) >= float(memory):
                    return instance_type['name']

            # No instance type found that can fulfill requested cores and memory
            raise CloudTaskManagerError('No instance type found with at least %d cores and %f GB of RAM.' % (cores, memory))

    @classmethod
    def _get_gcloud_pricelist(cls):
        """ Retrieve latest pricelist from Google Cloud, or use cached copy if not reachable. """
        try:
            r = requests.get('http://cloudpricingcalculator.appspot.com/static/data/pricelist.json')
            content = json.loads(r.content)
        except ConnectionError:
            logger.warning("Couldn't get updated pricelist from http://cloudpricingcalculator.appspot.com/static/data/pricelist.json. Falling back to cached copy, but prices may be out of date.")
            with open('pricelist.json') as infile:
                content = json.load(infile)

        #logger.debug('Using pricelist ' + content['version'] + ', updated ' + content['updated'])
        pricelist = content['gcp_price_list']
        return pricelist

    @classmethod
    def delete_instance_by_name(cls, instance_name):
        """ Delete the instance that ran a task. """
        # Don't want to block while waiting for VM to be deleted, so start another process to finish the rest of the steps.
        process = multiprocessing.Process(target=CloudTaskManager._delete_instance, args=(instance_name))
        process.start()

    @classmethod
    def delete_instance_by_task_run(cls, host_name, task_run):
        """ Delete the instance that ran a task. """
        instance_name = cls.create_worker_name(host_name, task_run)
        cls.delete_instance_by_name(instance_name)

    @classmethod
    def _delete_instance(cls, instance_name):
        if settings.WORKER_TYPE != 'GOOGLE_CLOUD':
            raise CloudTaskManagerError('Unsupported cloud type: ' + settings.WORKER_TYPE)
        # TODO: Support other cloud providers. For now, assume GCE.
        zone = settings.WORKER_LOCATION
        s = Template(
"""---
- hosts: localhost
  connection: local
  tasks:
  - name: Delete an instance.
    gce: name=$instance_name zone=$zone state=deleted
""")
        playbook = s.substitute(locals())
        cls._run_playbook_string(playbook)
        # TODO: Delete scratch disk

    @classmethod
    def create_worker_name(cls, hostname, task_run_attempt):
        """ Create a name for the worker instance. Since hostname, workflow name, and step name can easily be duplicated,
        we do this in two steps to ensure that at least 4 characters of the location ID are part of the name.
        Also, worker scratch disks are named by appending '-disk' to the instance name, and disk names are max 63 characters,
        so leave 5 characters for the '-disk' suffix.
        """
        task_run = task_run_attempt.task_run
        #workflow_name = task_run.workflow_name
        step_name = task_run.step_run.template.name
        attempt_id = task_run_attempt.id.hex
        name_base = '-'.join([hostname, step_name])
        sanitized_name_base = cls.sanitize_instance_name_base(name_base)
        sanitized_name_base = sanitized_name_base[:53]      # leave 10 characters at the end for location id and -disk suffix

        instance_name = '-'.join([sanitized_name_base, attempt_id])
        sanitized_instance_name = cls.sanitize_instance_name(instance_name)
        sanitized_instance_name = sanitized_instance_name[:58]      # leave 5 characters for -disk suffix
        return sanitized_instance_name

    @classmethod
    def sanitize_instance_name_base(cls, name):
        """ Instance names must start with a lowercase letter. All following characters must be a dash, lowercase letter, or digit. """
        name = str(name).lower()                # make all letters lowercase
        name = re.sub(r'[^-a-z0-9]', '', name)  # remove invalid characters
        name = re.sub(r'^[^a-z]+', '', name)    # remove non-lowercase letters from the beginning
        return name

    @classmethod
    def sanitize_instance_name(cls, name):
        """ Instance names must start with a lowercase letter. All following characters must be a dash, lowercase letter, or digit. Last character cannot be a dash.
        Instance names must be 1-63 characters long.
        """
        name = str(name).lower()                # make all letters lowercase
        name = re.sub(r'[^-a-z0-9]', '', name)  # remove invalid characters
        name = re.sub(r'^[^a-z]+', '', name)    # remove non-lowercase letters from the beginning
        name = re.sub(r'-+$', '', name)         # remove dashes from the end
        name = name[:63]                        # truncate if too long
        if len(name) < 1:
            raise CloudTaskManagerError('Cannot create an instance name from %s' % name)

        sanitized_name = name
        return sanitized_name

class CloudTaskManagerError(Exception):
    pass
<commit_msg>fix bug saving error msgs with post_task_run_attempt_error<commit_after>import errno
import imp
import json
import logging
import multiprocessing
import os
import pickle
import re
import requests
import socket
import subprocess
import sys
import tempfile
from string import Template

from django.conf import settings

from loomengine.utils.connection import Connection
import loomengine.utils.version
import loomengine.utils.cloud

PLAYBOOKS_PATH = os.path.join(imp.find_module('loomengine')[1], 'playbooks')
GCLOUD_CREATE_WORKER_PLAYBOOK = os.path.join(PLAYBOOKS_PATH, 'gcloud_create_worker.yml')
GCLOUD_RUN_TASK_PLAYBOOK = os.path.join(PLAYBOOKS_PATH, 'gcloud_run_task.yml')
GCE_PY_PATH = os.path.join(imp.find_module('loomengine')[1], 'utils', 'gce.py')

logger = logging.getLogger(__name__)

class CloudTaskManager:

    @classmethod
    def run(cls, task_run):
        from api.models.task_runs import TaskRunAttempt
        task_run_attempt = TaskRunAttempt.create_from_task_run(task_run)

        # Don't want to block while waiting for VM to come up, so start another process to finish the rest of the steps.
        logger.debug("Launching CloudTaskManager as a separate process.")

        requested_resources = {
            'cores': task_run_attempt.task_run.step_run.resources.cores,
            'memory': task_run_attempt.task_run.step_run.resources.memory,
            'disk_size': task_run_attempt.task_run.step_run.resources.disk_size,
        }
        environment = {
            'docker_image': task_run_attempt.task_definition.environment.docker_image,
        }

        hostname = socket.gethostname()
        worker_name = cls.create_worker_name(hostname, task_run_attempt)
        task_run_attempt_id = task_run_attempt.id.hex
        worker_log_file = task_run_attempt.get_worker_log_file()

        task_run_attempt.status = task_run_attempt.STATUSES.PROVISIONING_HOST
        task_run_attempt.save()

        process = multiprocessing.Process(target=CloudTaskManager._try_run, args=(task_run_attempt_id, requested_resources, environment, worker_name, worker_log_file))
        process.start()

    @classmethod
    def _try_run(cls, task_run_attempt_id, requested_resources, environment, worker_name, worker_log_file):
        try:
            cls._run(task_run_attempt_id, requested_resources, environment, worker_name, worker_log_file)
        except Exception as e:
            logger.exception('Failed to create task run attempt %s: %s' % (task_run_attempt_id, str(e)))

    @classmethod
    def _run(cls, task_run_attempt_id, requested_resources, environment, worker_name, worker_log_file):
        from api.models.task_runs import TaskRunAttempt

        logger.debug("CloudTaskManager separate process started.")
        logger.debug("task_run_attempt: %s" % task_run_attempt_id)

        connection = Connection(settings.MASTER_URL_FOR_SERVER)

        """Create a VM, deploy Docker and Loom, and pass command to task runner."""
        if settings.WORKER_TYPE != 'GOOGLE_CLOUD':
            raise CloudTaskManagerError('Unsupported cloud type: ' + settings.WORKER_TYPE)
        # TODO: Support other cloud providers. For now, assume GCE.
        instance_type = CloudTaskManager._get_cheapest_instance_type(cores=requested_resources['cores'], memory=requested_resources['memory'])

        scratch_disk_name = worker_name+'-disk'
        scratch_disk_device_path = '/dev/disk/by-id/google-'+scratch_disk_name
        if requested_resources.get('disk_size') is not None:
            scratch_disk_size_gb = requested_resources['disk_size']
        else:
            scratch_disk_size_gb = settings.WORKER_SCRATCH_DISK_SIZE

        playbook_vars = {
            'boot_disk_type': settings.WORKER_BOOT_DISK_TYPE,
            'boot_disk_size_gb': settings.WORKER_BOOT_DISK_SIZE,
            'docker_full_name': settings.DOCKER_FULL_NAME,
            'docker_tag': settings.DOCKER_TAG,
            'gce_email': settings.GCE_EMAIL,
            'gce_credential': settings.GCE_PEM_FILE_PATH,
            'gce_ssh_key_file': settings.GCE_SSH_KEY_FILE,
            'instance_name': worker_name,
            'instance_image': settings.WORKER_VM_IMAGE,
            'instance_type': instance_type,
            'log_level': settings.LOG_LEVEL,
            'master_url': settings.MASTER_URL_FOR_WORKER,
            'network': settings.WORKER_NETWORK,
            'remote_user': 'loom',
            'scratch_disk_name': scratch_disk_name,
            'scratch_disk_device_path': scratch_disk_device_path,
            'scratch_disk_mount_point': settings.WORKER_SCRATCH_DISK_MOUNT_POINT,
            'scratch_disk_type': settings.WORKER_SCRATCH_DISK_TYPE,
            'scratch_disk_size_gb': scratch_disk_size_gb,
            'subnetwork': settings.WORKER_CUSTOM_SUBNET,
            'tags': settings.WORKER_TAGS,
            'task_run_attempt_id': task_run_attempt_id,
            'task_run_docker_image': environment['docker_image'],
            'use_internal_ip': settings.WORKER_USES_SERVER_INTERNAL_IP,
            'worker_log_file': worker_log_file,
            'zone': settings.WORKER_LOCATION,
        }
        logger.debug('Starting worker VM using playbook vars: %s' % playbook_vars)

        try:
            with open(os.path.join(settings.LOGS_DIR, 'loom_ansible.log'), 'a', 0) as ansible_logfile:
                cls._run_playbook(GCLOUD_CREATE_WORKER_PLAYBOOK, playbook_vars, logfile=ansible_logfile)
        except Exception as e:
            logger.exception('Failed to provision host.')
            connection.post_task_run_attempt_error(
                task_run_attempt_id,
                {
                    'message': 'Failed to provision host',
                    'detail': str(e)
                })
            connection.update_task_run_attempt(
                task_run_attempt_id,
                {
                    'status': TaskRunAttempt.STATUSES.FINISHED,
                })
            raise e

        try:
            with open(os.path.join(settings.LOGS_DIR, 'loom_ansible.log'), 'a', 0) as ansible_logfile:
                connection.update_task_run_attempt(
                    task_run_attempt_id,
                    {
                        'status': TaskRunAttempt.STATUSES.LAUNCHING_MONITOR,
                    })
                cls._run_playbook(GCLOUD_RUN_TASK_PLAYBOOK, playbook_vars, logfile=ansible_logfile)
        except Exception as e:
            logger.exception('Failed to launch monitor process on worker: %s')
            connection.post_task_run_attempt_error(
                task_run_attempt_id,
                {
                    'message': 'Failed to launch monitor process on worker',
                    'detail': str(e)
                })
            connection.update_task_run_attempt(
                task_run_attempt_id,
                {
                    'status': TaskRunAttempt.STATUSES.FINISHED,
                })
            raise e

        logger.debug("CloudTaskManager process done.")
        ansible_logfile.close()

    @classmethod
    def _run_playbook(cls, playbook, playbook_vars, logfile=None):
        """ Runs a playbook by passing it a dict of vars on the command line."""
        ansible_env = os.environ.copy()
        ansible_env['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
        ansible_env['INVENTORY_IP_TYPE'] = 'internal'       # Tell gce.py to use internal IP for ansible_ssh_host
        playbook_vars_json_string = json.dumps(playbook_vars)
        cmd = ['ansible-playbook', '-vvv', '--key-file', os.path.expanduser(settings.GCE_SSH_KEY_FILE), '-i', GCE_PY_PATH, playbook, '--extra-vars', playbook_vars_json_string]
        returncode = subprocess.call(cmd, env=ansible_env, stderr=subprocess.STDOUT, stdout=logfile)
        if not returncode == 0:
            raise Exception('Nonzero returncode %s for command: %s' % (returncode, ' '.join(cmd)))

    @classmethod
    def _get_cheapest_instance_type(cls, cores, memory):
        """ Determine the cheapest instance type given a minimum number of cores and minimum amount of RAM (in GB). """

        if settings.WORKER_TYPE != 'GOOGLE_CLOUD': #TODO: support other cloud providers
            raise CloudTaskManagerError('Not a recognized cloud provider: ' + settings.WORKER_TYPE)
        else:
            pricelist = CloudTaskManager._get_gcloud_pricelist()

            # Filter out preemptible, shared-CPU, and non-US instance types
            us_instance_types = {k:v for k,v in pricelist.items()\
                if k.startswith('CP-COMPUTEENGINE-VMIMAGE-') and not k.endswith('-PREEMPTIBLE') and 'us' in v and v['cores'] != 'shared'}

            # Convert to array and add keys (instance type names) as type names
            price_array = []
            for key in us_instance_types:
                value = us_instance_types[key]
                value.update({'name':key.replace('CP-COMPUTEENGINE-VMIMAGE-', '').lower()})
                price_array.append(value)

            # Sort by price in US
            price_array.sort(None, lambda x: x['us'])

            # Look for an instance type that satisfies requested cores and memory; first will be cheapest
            for instance_type in price_array:
                if int(instance_type['cores']) >= int(cores) and float(instance_type['memory']) >= float(memory):
                    return instance_type['name']

            # No instance type found that can fulfill requested cores and memory
            raise CloudTaskManagerError('No instance type found with at least %d cores and %f GB of RAM.' % (cores, memory))

    @classmethod
    def _get_gcloud_pricelist(cls):
        """ Retrieve latest pricelist from Google Cloud, or use cached copy if not reachable. """
        try:
            r = requests.get('http://cloudpricingcalculator.appspot.com/static/data/pricelist.json')
            content = json.loads(r.content)
        except ConnectionError:
            logger.warning("Couldn't get updated pricelist from http://cloudpricingcalculator.appspot.com/static/data/pricelist.json. Falling back to cached copy, but prices may be out of date.")
            with open('pricelist.json') as infile:
                content = json.load(infile)

        #logger.debug('Using pricelist ' + content['version'] + ', updated ' + content['updated'])
        pricelist = content['gcp_price_list']
        return pricelist

    @classmethod
    def delete_instance_by_name(cls, instance_name):
        """ Delete the instance that ran a task. """
        # Don't want to block while waiting for VM to be deleted, so start another process to finish the rest of the steps.
        process = multiprocessing.Process(target=CloudTaskManager._delete_instance, args=(instance_name))
        process.start()

    @classmethod
    def delete_instance_by_task_run(cls, host_name, task_run):
        """ Delete the instance that ran a task. """
        instance_name = cls.create_worker_name(host_name, task_run)
        cls.delete_instance_by_name(instance_name)

    @classmethod
    def _delete_instance(cls, instance_name):
        if settings.WORKER_TYPE != 'GOOGLE_CLOUD':
            raise CloudTaskManagerError('Unsupported cloud type: ' + settings.WORKER_TYPE)
        # TODO: Support other cloud providers. For now, assume GCE.
        zone = settings.WORKER_LOCATION
        s = Template(
"""---
- hosts: localhost
  connection: local
  tasks:
  - name: Delete an instance.
    gce: name=$instance_name zone=$zone state=deleted
""")
        playbook = s.substitute(locals())
        cls._run_playbook_string(playbook)
        # TODO: Delete scratch disk

    @classmethod
    def create_worker_name(cls, hostname, task_run_attempt):
        """ Create a name for the worker instance. Since hostname, workflow name, and step name can easily be duplicated,
        we do this in two steps to ensure that at least 4 characters of the location ID are part of the name.
        Also, worker scratch disks are named by appending '-disk' to the instance name, and disk names are max 63 characters,
        so leave 5 characters for the '-disk' suffix.
        """
        task_run = task_run_attempt.task_run
        #workflow_name = task_run.workflow_name
        step_name = task_run.step_run.template.name
        attempt_id = task_run_attempt.id.hex
        name_base = '-'.join([hostname, step_name])
        sanitized_name_base = cls.sanitize_instance_name_base(name_base)
        sanitized_name_base = sanitized_name_base[:53]      # leave 10 characters at the end for location id and -disk suffix

        instance_name = '-'.join([sanitized_name_base, attempt_id])
        sanitized_instance_name = cls.sanitize_instance_name(instance_name)
        sanitized_instance_name = sanitized_instance_name[:58]      # leave 5 characters for -disk suffix
        return sanitized_instance_name

    @classmethod
    def sanitize_instance_name_base(cls, name):
        """ Instance names must start with a lowercase letter. All following characters must be a dash, lowercase letter, or digit. """
        name = str(name).lower()                # make all letters lowercase
        name = re.sub(r'[^-a-z0-9]', '', name)  # remove invalid characters
        name = re.sub(r'^[^a-z]+', '', name)    # remove non-lowercase letters from the beginning
        return name

    @classmethod
    def sanitize_instance_name(cls, name):
        """ Instance names must start with a lowercase letter. All following characters must be a dash, lowercase letter, or digit. Last character cannot be a dash.
        Instance names must be 1-63 characters long.
        """
        name = str(name).lower()                # make all letters lowercase
        name = re.sub(r'[^-a-z0-9]', '', name)  # remove invalid characters
        name = re.sub(r'^[^a-z]+', '', name)    # remove non-lowercase letters from the beginning
        name = re.sub(r'-+$', '', name)         # remove dashes from the end
        name = name[:63]                        # truncate if too long
        if len(name) < 1:
            raise CloudTaskManagerError('Cannot create an instance name from %s' % name)

        sanitized_name = name
        return sanitized_name

class CloudTaskManagerError(Exception):
    pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2016 Antonio Espinosa - <antonio.espinosa@tecnativa.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo.tests.common import TransactionCase

class TestMailgun(TransactionCase):
    def mail_send(self):
        mail = self.env['mail.mail'].create({
            'subject': 'Test subject',
            'email_from': 'from@example.com',
            'email_to': self.recipient,
            'body_html': '
This is a test message
',
        })
        mail.send()
        # Search tracking created
        tracking_email = self.env['mail.tracking.email'].search([
            ('mail_id', '=', mail.id),
        ])
        return mail, tracking_email

    def setUp(self):
        super(TestMailgun, self).setUp()
        self.recipient = u'to@example.com'
        self.mail, self.tracking_email = self.mail_send()
        self.api_key = u'key-12345678901234567890123456789012'
        self.token = u'f1349299097a51b9a7d886fcb5c2735b426ba200ada6e9e149'
        self.timestamp = u'1471021089'
        self.signature = ('4fb6d4dbbe10ce5d620265dcd7a3c0b8'
                          'ca0dede1433103891bc1ae4086e9d5b2')
        self.env['ir.config_parameter'].set_param(
            'mailgun.apikey', self.api_key)
        self.event = {
            'Message-Id': u'<xxx.xxx.xxx-openerp-xxx-res.partner@test_db>',
            'X-Mailgun-Sid': u'WyIwNjgxZSIsICJ0b0BleGFtcGxlLmNvbSIsICI3MG'
                             'I0MWYiXQ==',
            'token': self.token,
            'timestamp': self.timestamp,
            'signature': self.signature,
            'domain': u'example.com',
            'message-headers': u'[]',
            'recipient': self.recipient,
            'odoo_db': self.env.cr.dbname,
            'tracking_email_id': u'%s' % self.tracking_email.id
        }
        self.metadata = {
            'ip': '127.0.0.1',
            'user_agent': False,
            'os_family': False,
            'ua_family': False,
        }

    def event_search(self, event_type):
        event = self.env['mail.tracking.event'].search([
            ('tracking_email_id', '=', self.tracking_email.id),
            ('event_type', '=', event_type),
        ])
        self.assertTrue(event)
        return event

    def test_no_api_key(self):
        self.env['ir.config_parameter'].set_param('mailgun.apikey', '')
        self.test_event_delivered()

    def test_bad_signature(self):
        self.event.update({
            'event': u'delivered',
            'signature': u'bad_signature',
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('ERROR: Signature', response)

    def test_bad_event_type(self):
        self.event.update({
            'event': u'bad_event',
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('ERROR: Event type not supported', response)

    def test_bad_db(self):
        self.event.update({
            'event': u'delivered',
            'odoo_db': u'bad_db',
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('ERROR: Invalid DB', response)

    def test_bad_ts(self):
        timestamp = u'7a'  # Now time will be used instead
        signature = ('06cc05680f6e8110e59b41152b2d1c0f'
                     '1045d755ef2880ff922344325c89a6d4')
        self.event.update({
            'event': u'delivered',
            'timestamp': timestamp,
            'signature': signature,
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('OK', response)

    def test_tracking_not_found(self):
        self.event.update({
            'event': u'delivered',
            'tracking_email_id': u'bad_id',
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('ERROR: Tracking not found', response)

    # https://documentation.mailgun.com/user_manual.html#tracking-deliveries
    def test_event_delivered(self):
        self.event.update({
            'event': u'delivered',
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('OK', response)
        event = self.event_search('delivered')
        self.assertEqual(event.timestamp, float(self.timestamp))
        self.assertEqual(event.recipient, self.recipient)

    # https://documentation.mailgun.com/user_manual.html#tracking-opens
    def test_event_opened(self):
        ip = u'127.0.0.1'
        user_agent = u'Odoo Test/8.0 Gecko Firefox/11.0'
        os_family = u'Linux'
        ua_family = u'Firefox'
        ua_type = u'browser'
        self.event.update({
            'event': u'opened',
            'city': u'Mountain View',
            'country': u'US',
            'region': u'CA',
            'client-name': ua_family,
            'client-os': os_family,
            'client-type': ua_type,
            'device-type': u'desktop',
            'ip': ip,
            'user-agent': user_agent,
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('OK', response)
        event = self.event_search('open')
        self.assertEqual(event.timestamp, float(self.timestamp))
        self.assertEqual(event.recipient, self.recipient)
        self.assertEqual(event.ip, ip)
        self.assertEqual(event.user_agent, user_agent)
        self.assertEqual(event.os_family, os_family)
        self.assertEqual(event.ua_family, ua_family)
        self.assertEqual(event.ua_type, ua_type)
        self.assertEqual(event.mobile, False)
        self.assertEqual(event.user_country_id.code, 'US')

    # https://documentation.mailgun.com/user_manual.html#tracking-clicks
    def test_event_clicked(self):
        ip = u'127.0.0.1'
        user_agent = u'Odoo Test/8.0 Gecko Firefox/11.0'
        os_family = u'Linux'
        ua_family = u'Firefox'
        ua_type = u'browser'
        url = u'https://odoo-community.org'
        self.event.update({
            'event': u'clicked',
            'city': u'Mountain View',
            'country': u'US',
            'region': u'CA',
            'client-name': ua_family,
            'client-os': os_family,
            'client-type': ua_type,
            'device-type': u'tablet',
            'ip': ip,
            'user-agent': user_agent,
            'url': url,
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata, event_type='click')
        self.assertEqual('OK', response)
        event = self.event_search('click')
        self.assertEqual(event.timestamp, float(self.timestamp))
        self.assertEqual(event.recipient, self.recipient)
        self.assertEqual(event.ip, ip)
        self.assertEqual(event.user_agent, user_agent)
        self.assertEqual(event.os_family, os_family)
        self.assertEqual(event.ua_family, ua_family)
        self.assertEqual(event.ua_type, ua_type)
        self.assertEqual(event.mobile, True)
        self.assertEqual(event.url, url)

    # https://documentation.mailgun.com/user_manual.html#tracking-unsubscribes
    def test_event_unsubscribed(self):
        ip = u'127.0.0.1'
        user_agent = u'Odoo Test/8.0 Gecko Firefox/11.0'
        os_family = u'Linux'
        ua_family = u'Firefox'
        ua_type = u'browser'
        self.event.update({
            'event': u'unsubscribed',
            'city': u'Mountain View',
            'country': u'US',
            'region': u'CA',
            'client-name': ua_family,
            'client-os': os_family,
            'client-type': ua_type,
            'device-type': u'mobile',
            'ip': ip,
            'user-agent': user_agent,
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('OK', response)
        event = self.event_search('unsub')
        self.assertEqual(event.timestamp, float(self.timestamp))
        self.assertEqual(event.recipient, self.recipient)
        self.assertEqual(event.ip, ip)
        self.assertEqual(event.user_agent, user_agent)
        self.assertEqual(event.os_family, os_family)
        self.assertEqual(event.ua_family, ua_family)
        self.assertEqual(event.ua_type, ua_type)
        self.assertEqual(event.mobile, True)

    # https://documentation.mailgun.com/
    #   user_manual.html#tracking-spam-complaints
    def test_event_complained(self):
        self.event.update({
            'event': u'complained',
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('OK', response)
        event = self.event_search('spam')
        self.assertEqual(event.timestamp, float(self.timestamp))
        self.assertEqual(event.recipient, self.recipient)
        self.assertEqual(event.error_type, 'spam')

    # https://documentation.mailgun.com/user_manual.html#tracking-bounces
    def test_event_bounced(self):
        code = u'550'
        error = (u"5.1.1 The email account does not exist.\n"
                 "5.1.1 double-checking the recipient's email address")
        notification = u"Please, check recipient's email address"
        self.event.update({
            'event': u'bounced',
            'code': code,
            'error': error,
            'notification': notification,
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('OK', response)
        event = self.event_search('hard_bounce')
        self.assertEqual(event.timestamp, float(self.timestamp))
        self.assertEqual(event.recipient, self.recipient)
        self.assertEqual(event.error_type, code)
        self.assertEqual(event.error_description, error)
        self.assertEqual(event.error_details, notification)

    # https://documentation.mailgun.com/user_manual.html#tracking-failures
    def test_event_dropped(self):
        reason = u'hardfail'
        code = u'605'
        description = u'Not delivering to previously bounced address'
        self.event.update({
            'event': u'dropped',
            'reason': reason,
            'code': code,
            'description': description,
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('OK', response)
        event = self.event_search('reject')
        self.assertEqual(event.timestamp, float(self.timestamp))
        self.assertEqual(event.recipient, self.recipient)
        self.assertEqual(event.error_type, reason)
        self.assertEqual(event.error_description, code)
        self.assertEqual(event.error_details, description)
<commit_msg>[IMP] mute logger on tests error responses so Travis doesn't fail<commit_after># -*- coding: utf-8 -*-
# Copyright 2016 Antonio Espinosa - <antonio.espinosa@tecnativa.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo.tools import mute_logger
from odoo.tests.common import TransactionCase

class TestMailgun(TransactionCase):
    def mail_send(self):
        mail = self.env['mail.mail'].create({
            'subject': 'Test subject',
            'email_from': 'from@example.com',
            'email_to': self.recipient,
            'body_html': '
This is a test message
',
        })
        mail.send()
        # Search tracking created
        tracking_email = self.env['mail.tracking.email'].search([
            ('mail_id', '=', mail.id),
        ])
        return mail, tracking_email

    def setUp(self):
        super(TestMailgun, self).setUp()
        self.recipient = u'to@example.com'
        self.mail, self.tracking_email = self.mail_send()
        self.api_key = u'key-12345678901234567890123456789012'
        self.token = u'f1349299097a51b9a7d886fcb5c2735b426ba200ada6e9e149'
        self.timestamp = u'1471021089'
        self.signature = ('4fb6d4dbbe10ce5d620265dcd7a3c0b8'
                          'ca0dede1433103891bc1ae4086e9d5b2')
        self.env['ir.config_parameter'].set_param(
            'mailgun.apikey', self.api_key)
        self.event = {
            'Message-Id': u'<xxx.xxx.xxx-openerp-xxx-res.partner@test_db>',
            'X-Mailgun-Sid': u'WyIwNjgxZSIsICJ0b0BleGFtcGxlLmNvbSIsICI3MG'
                             'I0MWYiXQ==',
            'token': self.token,
            'timestamp': self.timestamp,
            'signature': self.signature,
            'domain': u'example.com',
            'message-headers': u'[]',
            'recipient': self.recipient,
            'odoo_db': self.env.cr.dbname,
            'tracking_email_id': u'%s' % self.tracking_email.id
        }
        self.metadata = {
            'ip': '127.0.0.1',
            'user_agent': False,
            'os_family': False,
            'ua_family': False,
        }

    def event_search(self, event_type):
        event = self.env['mail.tracking.event'].search([
            ('tracking_email_id', '=', self.tracking_email.id),
            ('event_type', '=', event_type),
        ])
        self.assertTrue(event)
        return event

    def test_no_api_key(self):
        self.env['ir.config_parameter'].set_param('mailgun.apikey', '')
        self.test_event_delivered()

    @mute_logger('odoo.addons.mail_tracking_mailgun.models'
                 '.mail_tracking_email')
    def test_bad_signature(self):
        self.event.update({
            'event': u'delivered',
            'signature': u'bad_signature',
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('ERROR: Signature', response)

    @mute_logger('odoo.addons.mail_tracking_mailgun.models'
                 '.mail_tracking_email')
    def test_bad_event_type(self):
        self.event.update({
            'event': u'bad_event',
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('ERROR: Event type not supported', response)

    @mute_logger('odoo.addons.mail_tracking_mailgun.models'
                 '.mail_tracking_email')
    def test_bad_db(self):
        self.event.update({
            'event': u'delivered',
            'odoo_db': u'bad_db',
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('ERROR: Invalid DB', response)

    def test_bad_ts(self):
        timestamp = u'7a'  # Now time will be used instead
        signature = ('06cc05680f6e8110e59b41152b2d1c0f'
                     '1045d755ef2880ff922344325c89a6d4')
        self.event.update({
            'event': u'delivered',
            'timestamp': timestamp,
            'signature': signature,
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('OK', response)

    @mute_logger('odoo.addons.mail_tracking_mailgun.models'
                 '.mail_tracking_email')
    def test_tracking_not_found(self):
        self.event.update({
            'event': u'delivered',
            'tracking_email_id': u'bad_id',
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('ERROR: Tracking not found', response)

    # https://documentation.mailgun.com/user_manual.html#tracking-deliveries
    def test_event_delivered(self):
        self.event.update({
            'event': u'delivered',
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('OK', response)
        event = self.event_search('delivered')
        self.assertEqual(event.timestamp, float(self.timestamp))
        self.assertEqual(event.recipient, self.recipient)

    # https://documentation.mailgun.com/user_manual.html#tracking-opens
    def test_event_opened(self):
        ip = u'127.0.0.1'
        user_agent = u'Odoo Test/8.0 Gecko Firefox/11.0'
        os_family = u'Linux'
        ua_family = u'Firefox'
        ua_type = u'browser'
        self.event.update({
            'event': u'opened',
            'city': u'Mountain View',
            'country': u'US',
            'region': u'CA',
            'client-name': ua_family,
            'client-os': os_family,
            'client-type': ua_type,
            'device-type': u'desktop',
            'ip': ip,
            'user-agent': user_agent,
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('OK', response)
        event = self.event_search('open')
        self.assertEqual(event.timestamp, float(self.timestamp))
        self.assertEqual(event.recipient, self.recipient)
        self.assertEqual(event.ip, ip)
        self.assertEqual(event.user_agent, user_agent)
        self.assertEqual(event.os_family, os_family)
        self.assertEqual(event.ua_family, ua_family)
        self.assertEqual(event.ua_type, ua_type)
        self.assertEqual(event.mobile, False)
        self.assertEqual(event.user_country_id.code, 'US')

    # https://documentation.mailgun.com/user_manual.html#tracking-clicks
    def test_event_clicked(self):
        ip = u'127.0.0.1'
        user_agent = u'Odoo Test/8.0 Gecko Firefox/11.0'
        os_family = u'Linux'
        ua_family = u'Firefox'
        ua_type = u'browser'
        url = u'https://odoo-community.org'
        self.event.update({
            'event': u'clicked',
            'city': u'Mountain View',
            'country': u'US',
            'region': u'CA',
            'client-name': ua_family,
            'client-os': os_family,
            'client-type': ua_type,
            'device-type': u'tablet',
            'ip': ip,
            'user-agent': user_agent,
            'url': url,
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata, event_type='click')
        self.assertEqual('OK', response)
        event = self.event_search('click')
        self.assertEqual(event.timestamp, float(self.timestamp))
        self.assertEqual(event.recipient, self.recipient)
        self.assertEqual(event.ip, ip)
        self.assertEqual(event.user_agent, user_agent)
        self.assertEqual(event.os_family, os_family)
        self.assertEqual(event.ua_family, ua_family)
        self.assertEqual(event.ua_type, ua_type)
        self.assertEqual(event.mobile, True)
        self.assertEqual(event.url, url)

    # https://documentation.mailgun.com/user_manual.html#tracking-unsubscribes
    def test_event_unsubscribed(self):
        ip = u'127.0.0.1'
        user_agent = u'Odoo Test/8.0 Gecko Firefox/11.0'
        os_family = u'Linux'
        ua_family = u'Firefox'
        ua_type = u'browser'
        self.event.update({
            'event': u'unsubscribed',
            'city': u'Mountain View',
            'country': u'US',
            'region': u'CA',
            'client-name': ua_family,
            'client-os': os_family,
            'client-type': ua_type,
            'device-type': u'mobile',
            'ip': ip,
            'user-agent': user_agent,
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('OK', response)
        event = self.event_search('unsub')
        self.assertEqual(event.timestamp, float(self.timestamp))
        self.assertEqual(event.recipient, self.recipient)
        self.assertEqual(event.ip, ip)
        self.assertEqual(event.user_agent, user_agent)
        self.assertEqual(event.os_family, os_family)
        self.assertEqual(event.ua_family, ua_family)
        self.assertEqual(event.ua_type, ua_type)
        self.assertEqual(event.mobile, True)

    # https://documentation.mailgun.com/
    #   user_manual.html#tracking-spam-complaints
    def test_event_complained(self):
        self.event.update({
            'event': u'complained',
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('OK', response)
        event = self.event_search('spam')
        self.assertEqual(event.timestamp, float(self.timestamp))
        self.assertEqual(event.recipient, self.recipient)
        self.assertEqual(event.error_type, 'spam')

    # https://documentation.mailgun.com/user_manual.html#tracking-bounces
    def test_event_bounced(self):
        code = u'550'
        error = (u"5.1.1 The email account does not exist.\n"
                 "5.1.1 double-checking the recipient's email address")
        notification = u"Please, check recipient's email address"
        self.event.update({
            'event': u'bounced',
            'code': code,
            'error': error,
            'notification': notification,
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('OK', response)
        event = self.event_search('hard_bounce')
        self.assertEqual(event.timestamp, float(self.timestamp))
        self.assertEqual(event.recipient, self.recipient)
        self.assertEqual(event.error_type, code)
        self.assertEqual(event.error_description, error)
        self.assertEqual(event.error_details, notification)

    # https://documentation.mailgun.com/user_manual.html#tracking-failures
    def test_event_dropped(self):
        reason = u'hardfail'
        code = u'605'
        description = u'Not delivering to previously bounced address'
        self.event.update({
            'event': u'dropped',
            'reason': reason,
            'code': code,
            'description': description,
        })
        response = self.env['mail.tracking.email'].event_process(
            None, self.event, self.metadata)
        self.assertEqual('OK', response)
        event = self.event_search('reject')
        self.assertEqual(event.timestamp, float(self.timestamp))
        self.assertEqual(event.recipient, self.recipient)
        self.assertEqual(event.error_type, reason)
        self.assertEqual(event.error_description, code)
        self.assertEqual(event.error_details, description)
<|endoftext|>