<commit_before>"""bsw - Build Static Website

A simple static website generator operating on the current directory and
building static websites from pages and templates.

Your project folder should be structured as follows:
    pages/
    templates/
        base.html

An example layout would be:
    pages/
        about/
            index.html
        index.html
    templates/
        static/
            site_logo.png
        includes/
            social_media_links.html
        base.html
        blog.html
    static/
        images/
            blog_header_20160101.png

This example has a base template (base.html), a blog template (blog.html),
an includes file with social media links (social_media_links.html),
and default site page (index.html) and an about page (about/index.html).

Static files from both the site statics (static/) and template statics
(templates/static/) folders are combined into the build/ folder
(automatically created when the site is first generated).

The structure in pages/ is preserved and cloned to the build/ folder.
"""

import argparse
import os

try:
    # Python 3
    import http.server
    import socketserver
except ImportError:
    # Python 2
    import SimpleHTTPServer
    import SocketServer

import sys

from . import files
from . import pages
from . import templates

OUT_DIR = os.path.abspath(os.path.join(".", "build"))

def serve_content():
    """Serve rendered content on SimpleHTTPServer. This helps with absolute
    file paths such as /static/css/main.css, which wouldn't work
    properly if opened directly in from file:///"""
    PORT = 8000
    os.chdir(OUT_DIR)

    # Allow us to quickly kill and restart server without waiting for TCP
    # socket to close down completely
    try:
        # Python 3
        handler = http.server.SimpleHTTPRequestHandler
        socketserver.TCPServer.allow_reuse_address = 1
        httpd = socketserver.TCPServer(("", PORT), handler)
    except NameError:
        # Python 2
        handler = SimpleHTTPServer.SimpleHTTPRequestHandler
        SocketServer.TCPServer.allow_reuse_address = 1
        httpd = SocketServer.TCPServer(("", PORT), handler)

    print("Serving content at localhost: " + str(PORT))
    try:
        httpd.serve_forever()
    except KeyboardInterrupt:
        print("Exiting")
        sys.exit()

def build_static_web(clean_build_path):
    file_manager = files.FileManager(OUT_DIR)

    # Check that required paths (e.g. base template) exist before starting
    try:
        file_manager.check_required_paths()
    except IOError as ex:
        print("Error: {0}".format(ex))
        print("Did you remember to create the 'templates' folder and "
              "'base.html' template?")
        sys.exit(1)

    if (clean_build_path):
        print("Cleaning build path...")
        file_manager.clean_build_path()
    file_manager.create_out_dir()

    print("Loading pages...")
    site_pages = pages.collect_pages()
    for page in site_pages:
        page.load_and_parse()

    # Check that any explicit templates exist before rendering
    try:
        templates.check_templates_exist(site_pages)
    except IOError as ex:
        print("Error: {0}".format(ex))
        sys.exit(1)

    print("Rendering pages...")
    for page in site_pages:
        page.render()
    print("Writing pages...")
    pages.write_pages(site_pages, OUT_DIR)

    print("Copying site and template assets...")
    file_manager.copy_template_assets()
    file_manager.copy_site_assets()

    print("Static site build complete.")

def main():
    parser = argparse.ArgumentParser(
        description="bsw - build static website")
    parser.add_argument("-C", "--clean", action="store_true",
                        help="remove existing build folder before building")
    parser.add_argument("-s", "--serve", action="store_true",
                        help="serve content after build (default port 8000)")
    parser.add_argument("--init",
                        help="initialise a new bsw site as the specified path")
    parser.add_argument("--template",
                        help="built-in template to use with --init")
    args = parser.parse_args()

    if (args.clean and args.init):
        print("Error: Please specify either --clean or --init")
        sys.exit(1)

    if args.init:
        file_manager = files.FileManager(OUT_DIR)
        template = args.template if args.template else "default"
        file_manager.init_with_template(template, args.init)

    clean_build_path = False
    if args.clean:
        clean_build_path = True

    build_static_web(clean_build_path)

    if args.serve:
        serve_content()
<commit_msg>Changes to command line args<commit_after>"""bsw - Build Static Website

A simple static website generator operating on the current directory and
building static websites from pages and templates.

Your project folder should be structured as follows:
    pages/
    templates/
        base.html

An example layout would be:
    pages/
        about/
            index.html
        index.html
    templates/
        static/
            site_logo.png
        includes/
            social_media_links.html
        base.html
        blog.html
    static/
        images/
            blog_header_20160101.png

This example has a base template (base.html), a blog template (blog.html),
an includes file with social media links (social_media_links.html),
and default site page (index.html) and an about page (about/index.html).

Static files from both the site statics (static/) and template statics
(templates/static/) folders are combined into the build/ folder
(automatically created when the site is first generated).

The structure in pages/ is preserved and cloned to the build/ folder.
"""

import argparse
import os

try:
    # Python 3
    import http.server
    import socketserver
except ImportError:
    # Python 2
    import SimpleHTTPServer
    import SocketServer

import sys

from . import files
from . import pages
from . import templates

OUT_DIR = os.path.abspath(os.path.join(".", "build"))

def serve_content():
    """Serve rendered content on SimpleHTTPServer. This helps with absolute
    file paths such as /static/css/main.css, which wouldn't work
    properly if opened directly in from file:///"""
    PORT = 8000
    os.chdir(OUT_DIR)

    # Allow us to quickly kill and restart server without waiting for TCP
    # socket to close down completely
    try:
        # Python 3
        handler = http.server.SimpleHTTPRequestHandler
        socketserver.TCPServer.allow_reuse_address = 1
        httpd = socketserver.TCPServer(("", PORT), handler)
    except NameError:
        # Python 2
        handler = SimpleHTTPServer.SimpleHTTPRequestHandler
        SocketServer.TCPServer.allow_reuse_address = 1
        httpd = SocketServer.TCPServer(("", PORT), handler)

    print("Serving content at localhost: " + str(PORT))
    try:
        httpd.serve_forever()
    except KeyboardInterrupt:
        print("Exiting")
        sys.exit()

def build_static_web(clean_build_path):
    file_manager = files.FileManager(OUT_DIR)

    # Check that required paths (e.g. base template) exist before starting
    try:
        file_manager.check_required_paths()
    except IOError as ex:
        print("Error: {0}".format(ex))
        print("Did you remember to initialise the site with --init?")
        sys.exit(1)

    if (clean_build_path):
        print("Cleaning build path...")
        file_manager.clean_build_path()
    file_manager.create_out_dir()

    print("Loading pages...")
    site_pages = pages.collect_pages()
    for page in site_pages:
        page.load_and_parse()

    # Check that any explicit templates exist before rendering
    try:
        templates.check_templates_exist(site_pages)
    except IOError as ex:
        print("Error: {0}".format(ex))
        sys.exit(1)

    print("Rendering pages...")
    for page in site_pages:
        page.render()
    print("Writing pages...")
    pages.write_pages(site_pages, OUT_DIR)

    print("Copying site and template assets...")
    file_manager.copy_template_assets()
    file_manager.copy_site_assets()

    print("Static site build complete.")

def main():
    parser = argparse.ArgumentParser(
        description="bsw - build static website")
    parser.add_argument("-C", "--clean", action="store_true",
                        help="remove existing build folder before building")
    parser.add_argument("-s", "--serve", action="store_true",
                        help="serve content after build (default port 8000)")
    parser.add_argument("--init",
                        help="initialise a new bsw site as the specified path",
                        nargs="?",
                        const=".",
                        metavar="PATH")
    parser.add_argument("--template",
                        help="built-in template to use with --init")
    args = parser.parse_args()

    if (args.clean and args.init):
        print("Error: Please specify either --clean or --init")
        sys.exit(1)

    if (args.serve and args.init):
        print("Error: Please specify either --serve or --init")
        sys.exit(1)

    if (args.template and not args.init):
        print("Error: --template can only be used with --init")
        sys.exit(1)

    if args.init:
        file_manager = files.FileManager(OUT_DIR)
        template = args.template if args.template else "default"
        file_manager.init_with_template(template, args.init)
        print("bsw site initialised")
        sys.exit(0)

    clean_build_path = False
    if args.clean:
        clean_build_path = True

    build_static_web(clean_build_path)

    if args.serve:
        serve_content()
<|endoftext|>


<commit_before># stdlib
from typing import Any
from typing import Tuple as TypeTuple

# third party
import pytest
import torch as th

# syft absolute
import syft as sy
from syft import Plan
from syft import make_plan
from syft import serialize
from syft.core.common.uid import UID
from syft.core.io.address import Address
from syft.core.node.common.action.common import Action
from syft.core.node.common.action.function_or_constructor_action import (
    RunFunctionOrConstructorAction,
)
from syft.core.node.common.action.garbage_collect_object_action import (
    GarbageCollectObjectAction,
)
from syft.core.node.common.action.get_enum_attribute_action import EnumAttributeAction
from syft.core.node.common.action.get_object_action import GetObjectAction
from syft.core.node.common.action.get_or_set_property_action import (
    GetOrSetPropertyAction,
)
from syft.core.node.common.action.get_or_set_property_action import PropertyActions
from syft.core.node.common.action.get_or_set_static_attribute_action import (
    GetSetStaticAttributeAction,
)
from syft.core.node.common.action.get_or_set_static_attribute_action import (
    StaticAttributeAction,
)
from syft.core.node.common.action.run_class_method_action import RunClassMethodAction
from syft.core.node.common.action.save_object_action import SaveObjectAction
from syft.core.plan.plan_builder import ROOT_CLIENT
from syft.core.store.storeable_object import StorableObject
from syft.lib.python.list import List

def test_plan_serialization(client: sy.VirtualMachineClient) -> None:

    # cumbersome way to get a pointer as input for our actions,
    # there is probably a better/shorter way
    t = th.tensor([1, 2, 3])
    tensor_pointer = t.send(client)

    # define actions
    a1 = GetObjectAction(
        id_at_location=UID(), address=Address(), reply_to=Address(), msg_id=UID()
    )
    a2 = RunFunctionOrConstructorAction(
        path="torch.Tensor.add",
        args=tuple(),
        kwargs={},
        id_at_location=UID(),
        address=Address(),
        msg_id=UID(),
    )

    a3 = RunClassMethodAction(
        path="torch.Tensor.add",
        _self=tensor_pointer,
        args=[],
        kwargs={},
        id_at_location=UID(),
        address=Address(),
        msg_id=UID(),
    )

    a4 = GarbageCollectObjectAction(id_at_location=UID(), address=Address())
    a5 = EnumAttributeAction(path="", id_at_location=UID(), address=Address())

    a6 = GetOrSetPropertyAction(
        path="",
        _self=tensor_pointer,
        id_at_location=UID(),
        address=Address(),
        args=[],
        kwargs={},
        action=PropertyActions.GET,
        map_to_dyn=False,
    )
    a7 = GetSetStaticAttributeAction(
        path="",
        id_at_location=UID(),
        address=Address(),
        action=StaticAttributeAction.GET,
    )
    a8 = SaveObjectAction(obj=StorableObject(id=UID(), data=t), address=Address())

    # define plan
    plan = Plan([a1, a2, a3, a4, a5, a6, a7, a8])

    # serialize / deserialize
    blob = serialize(plan)
    plan_reconstructed = sy.deserialize(blob=blob)

    # test
    assert isinstance(plan_reconstructed, Plan)
    assert all(isinstance(a, Action) for a in plan_reconstructed.actions)

def test_plan_execution(client: sy.VirtualMachineClient) -> None:
    tensor_pointer1 = th.tensor([1, 2, 3]).send(client)
    tensor_pointer2 = th.tensor([4, 5, 6]).send(client)
    tensor_pointer3 = th.tensor([7, 8, 9]).send(client)

    result_tensor_pointer1 = th.tensor([0, 0, 0]).send(client)
    result_tensor_pointer2 = th.tensor([0, 0, 0]).send(client)

    result1_uid = result_tensor_pointer1.id_at_location
    result2_uid = result_tensor_pointer2.id_at_location

    a1 = RunClassMethodAction(
        path="torch.Tensor.add",
        _self=tensor_pointer1,
        args=[tensor_pointer2],
        kwargs={},
        id_at_location=result1_uid,
        address=Address(),
        msg_id=UID(),
    )

    a2 = RunClassMethodAction(
        path="torch.Tensor.add",
        _self=result_tensor_pointer1,
        args=[tensor_pointer3],
        kwargs={},
        id_at_location=result2_uid,
        address=Address(),
        msg_id=UID(),
    )

    plan = Plan([a1, a2])

    plan_pointer = plan.send(client)

    plan_pointer()

    expected_tensor1 = th.tensor([5, 7, 9])
    expected_tensor2 = th.tensor([12, 15, 18])

    assert all(expected_tensor1 == result_tensor_pointer1.get())
    assert all(expected_tensor2 == result_tensor_pointer2.get())

def test_plan_batched_execution(client: sy.VirtualMachineClient) -> None:
    # placeholders for our input
    input_tensor_pointer1 = th.tensor([0, 0]).send(client)
    input_tensor_pointer2 = th.tensor([0, 0]).send(client)

    # tensors in our model
    model_tensor_pointer1 = th.tensor([1, 2]).send(client)
    model_tensor_pointer2 = th.tensor([3, 4]).send(client)

    # placeholders for intermediate results
    result_tensor_pointer1 = th.tensor([0, 0]).send(client)
    result_tensor_pointer2 = th.tensor([0, 0]).send(client)
    result_tensor_pointer3 = th.tensor([0, 0]).send(client)

    # define plan
    a1 = RunClassMethodAction(
        path="torch.Tensor.mul",
        _self=input_tensor_pointer1,
        args=[model_tensor_pointer1],
        kwargs={},
        id_at_location=result_tensor_pointer1.id_at_location,
        address=Address(),
        msg_id=UID(),
    )

    a2 = RunClassMethodAction(
        path="torch.Tensor.add",
        _self=result_tensor_pointer1,
        args=[model_tensor_pointer2],
        kwargs={},
        id_at_location=result_tensor_pointer2.id_at_location,
        address=Address(),
        msg_id=UID(),
    )

    a3 = RunFunctionOrConstructorAction(
        path="torch.eq",
        args=[result_tensor_pointer2, input_tensor_pointer2],
        kwargs={},
        id_at_location=result_tensor_pointer3.id_at_location,
        address=Address(),
        msg_id=UID(),
    )

    plan = Plan(
        [a1, a2, a3], inputs={"x": input_tensor_pointer1, "y": input_tensor_pointer2}
    )
    plan_pointer = plan.send(client)

    # Test
    # x is random input, y is the expected model(x)
    x_batches = [(th.tensor([1, 1]) + i).send(client) for i in range(2)]
    y_batches = [
        ((th.tensor([1, 1]) + i) * th.tensor([1, 2]) + th.tensor([3, 4])).send(client)
        for i in range(2)
    ]

    for x, y in zip(x_batches, y_batches):
        plan_pointer(x=x, y=y)

        # checks if (model(x) == y) == [True, True]
        assert all(result_tensor_pointer3.get(delete_obj=False))

@pytest.mark.xfail
def test_make_plan(client: sy.VirtualMachineClient) -> None:
    @make_plan
    def add_plan(inp=th.zeros((3))) -> th.Tensor:  # type: ignore
        return inp + inp

    input_tensor = th.tensor([1, 2, 3])
    plan_pointer = add_plan.send(client)
    res = plan_pointer(inp=input_tensor)
    assert th.equal(res.get()[0], th.tensor([2, 4, 6]))

@pytest.mark.xfail
def test_plan_deterministic_bytes(root_client: sy.VirtualMachineClient) -> None:
    # TODO: https://github.com/OpenMined/PySyft/issues/5292
    @make_plan
    def add_plan(inp=th.zeros((3))) -> th.Tensor:  # type: ignore
        return inp + inp

    @make_plan
    def add_plan2(inp=th.zeros((3))) -> th.Tensor:  # type: ignore
        return inp + inp

    plan_pointer = add_plan.send(root_client)
    plan2_pointer = add_plan2.send(root_client)

    plan1 = serialize(plan_pointer, to_bytes=True)
    plan2 = serialize(plan2_pointer, to_bytes=True)

    assert plan1 == plan2

@pytest.mark.skip(reason="Hypothesis: serde never truly worked")
def test_make_plan2(root_client: sy.VirtualMachineClient) -> None:
    @make_plan
    def mul_plan(inp=th.zeros((3)), inp2=th.zeros((3))) -> th.Tensor:  # type: ignore
        return inp * inp2

    t1, t2 = th.tensor([1, 2, 3]), th.tensor([1, 2, 3])
    plan_pointer = mul_plan.send(root_client)
    res = plan_pointer(inp=t1, inp2=t2)
    assert th.equal(res.get()[0], th.tensor([1, 4, 9]))

def test_make_plan_error_no_kwargs() -> None:
    def assertRaises(exc, obj, methodname, *args) -> None:  # type: ignore
        with pytest.raises(exc) as e_info:
            getattr(obj, methodname)(*args)
        assert str(e_info) != ""

    def test_define_plan():  # type: ignore
        @make_plan
        def add_plan(inp):  # type: ignore
            return inp + inp

    # we can only define a plan with *kwargs* when using the @make_plan decorator, not with args
    assertRaises(ValueError, test_define_plan, "__call__")

@pytest.mark.skip(reason="Hypothesis: serde never truly worked")
@pytest.mark.slow
def test_mlp_plan(client: sy.VirtualMachineClient) -> None:
    class MLP(sy.Module):
        def __init__(self, torch_ref):  # type: ignore
            super().__init__(torch_ref=torch_ref)
            self.l1 = self.torch_ref.nn.Linear(784, 100)
            self.a1 = self.torch_ref.nn.ReLU()
            self.l2 = self.torch_ref.nn.Linear(100, 10)

        def forward(self, x):  # type: ignore
            x_reshaped = x.view(-1, 28 * 28)
            l1_out = self.a1(self.l1(x_reshaped))
            l2_out = self.l2(l1_out)
            return l2_out

    def set_params(model, params):  # type: ignore
        """happens outside of plan"""
        for p, p_new in zip(model.parameters(), params):
            p.data = p_new.data

    def cross_entropy_loss(logits, targets, batch_size):  # type: ignore
        norm_logits = logits - logits.max()
        log_probs = norm_logits - norm_logits.exp().sum(dim=1, keepdim=True).log()
        return -(targets * log_probs).sum() / batch_size

    def sgd_step(model, lr=0.1):  # type: ignore
        with ROOT_CLIENT.torch.no_grad():
            for p in model.parameters():
                p.data = p.data - lr * p.grad
                # Todo: fix this
                p.grad = th.zeros_like(p.grad.get())

    local_model = MLP(th)  # type: ignore

    @make_plan
    def train(  # type: ignore
        xs=th.rand([64 * 3, 1, 28, 28]),
        ys=th.randint(0, 10, [64 * 3, 10]),
        params=List(local_model.parameters()),
    ):

        model = local_model.send(ROOT_CLIENT)
        set_params(model, params)
        for i in range(2):
            indices = th.tensor(range(64 * i, 64 * (i + 1)))
            x, y = xs.index_select(0, indices), ys.index_select(0, indices)
            out = model(x)
            loss = cross_entropy_loss(out, y, 64)
            loss.backward()
            sgd_step(model)

        return model.parameters()

    train_ptr = train.send(client)

    old_params = local_model.parameters().copy()

    res_ptr = train_ptr(
        xs=th.rand([64 * 3, 1, 28, 28]),
        ys=th.randint(0, 10, [64 * 3, 10]),
        params=local_model.parameters(),
    )

    (new_params,) = res_ptr.get()

    assert not (old_params[0] == new_params[0]).all()

def test_check_placeholder() -> None:
    def test_define_plan() -> None:
        @make_plan
        def add_plan(inp: Any) -> Any:
            return inp + inp

    def assertRaises(
        exc: Exception, obj: object, methodname: str, *args: TypeTuple[Any, ...]
    ) -> None:
        with pytest.raises(exc) as e_info:
            getattr(obj, methodname)(*args)
        assert str(e_info) != ""

    assertRaises(ValueError, test_define_plan, "__call__")  # type: ignore
<commit_msg>remove xfail for plan<commit_after># stdlib
from typing import Any
from typing import Tuple as TypeTuple

# third party
import pytest
import torch as th

# syft absolute
import syft as sy
from syft import Plan
from syft import make_plan
from syft import serialize
from syft.core.common.uid import UID
from syft.core.io.address import Address
from syft.core.node.common.action.common import Action
from syft.core.node.common.action.function_or_constructor_action import (
    RunFunctionOrConstructorAction,
)
from syft.core.node.common.action.garbage_collect_object_action import (
    GarbageCollectObjectAction,
)
from syft.core.node.common.action.get_enum_attribute_action import EnumAttributeAction
from syft.core.node.common.action.get_object_action import GetObjectAction
from syft.core.node.common.action.get_or_set_property_action import (
    GetOrSetPropertyAction,
)
from syft.core.node.common.action.get_or_set_property_action import PropertyActions
from syft.core.node.common.action.get_or_set_static_attribute_action import (
    GetSetStaticAttributeAction,
)
from syft.core.node.common.action.get_or_set_static_attribute_action import (
    StaticAttributeAction,
)
from syft.core.node.common.action.run_class_method_action import RunClassMethodAction
from syft.core.node.common.action.save_object_action import SaveObjectAction
from syft.core.plan.plan_builder import ROOT_CLIENT
from syft.core.store.storeable_object import StorableObject
from syft.lib.python.list import List

def test_plan_serialization(client: sy.VirtualMachineClient) -> None:

    # cumbersome way to get a pointer as input for our actions,
    # there is probably a better/shorter way
    t = th.tensor([1, 2, 3])
    tensor_pointer = t.send(client)

    # define actions
    a1 = GetObjectAction(
        id_at_location=UID(), address=Address(), reply_to=Address(), msg_id=UID()
    )
    a2 = RunFunctionOrConstructorAction(
        path="torch.Tensor.add",
        args=tuple(),
        kwargs={},
        id_at_location=UID(),
        address=Address(),
        msg_id=UID(),
    )

    a3 = RunClassMethodAction(
        path="torch.Tensor.add",
        _self=tensor_pointer,
        args=[],
        kwargs={},
        id_at_location=UID(),
        address=Address(),
        msg_id=UID(),
    )

    a4 = GarbageCollectObjectAction(id_at_location=UID(), address=Address())
    a5 = EnumAttributeAction(path="", id_at_location=UID(), address=Address())

    a6 = GetOrSetPropertyAction(
        path="",
        _self=tensor_pointer,
        id_at_location=UID(),
        address=Address(),
        args=[],
        kwargs={},
        action=PropertyActions.GET,
        map_to_dyn=False,
    )
    a7 = GetSetStaticAttributeAction(
        path="",
        id_at_location=UID(),
        address=Address(),
        action=StaticAttributeAction.GET,
    )
    a8 = SaveObjectAction(obj=StorableObject(id=UID(), data=t), address=Address())

    # define plan
    plan = Plan([a1, a2, a3, a4, a5, a6, a7, a8])

    # serialize / deserialize
    blob = serialize(plan)
    plan_reconstructed = sy.deserialize(blob=blob)

    # test
    assert isinstance(plan_reconstructed, Plan)
    assert all(isinstance(a, Action) for a in plan_reconstructed.actions)

def test_plan_execution(client: sy.VirtualMachineClient) -> None:
    tensor_pointer1 = th.tensor([1, 2, 3]).send(client)
    tensor_pointer2 = th.tensor([4, 5, 6]).send(client)
    tensor_pointer3 = th.tensor([7, 8, 9]).send(client)

    result_tensor_pointer1 = th.tensor([0, 0, 0]).send(client)
    result_tensor_pointer2 = th.tensor([0, 0, 0]).send(client)

    result1_uid = result_tensor_pointer1.id_at_location
    result2_uid = result_tensor_pointer2.id_at_location

    a1 = RunClassMethodAction(
        path="torch.Tensor.add",
        _self=tensor_pointer1,
        args=[tensor_pointer2],
        kwargs={},
        id_at_location=result1_uid,
        address=Address(),
        msg_id=UID(),
    )

    a2 = RunClassMethodAction(
        path="torch.Tensor.add",
        _self=result_tensor_pointer1,
        args=[tensor_pointer3],
        kwargs={},
        id_at_location=result2_uid,
        address=Address(),
        msg_id=UID(),
    )

    plan = Plan([a1, a2])

    plan_pointer = plan.send(client)

    plan_pointer()

    expected_tensor1 = th.tensor([5, 7, 9])
    expected_tensor2 = th.tensor([12, 15, 18])

    assert all(expected_tensor1 == result_tensor_pointer1.get())
    assert all(expected_tensor2 == result_tensor_pointer2.get())

def test_plan_batched_execution(client: sy.VirtualMachineClient) -> None:
    # placeholders for our input
    input_tensor_pointer1 = th.tensor([0, 0]).send(client)
    input_tensor_pointer2 = th.tensor([0, 0]).send(client)

    # tensors in our model
    model_tensor_pointer1 = th.tensor([1, 2]).send(client)
    model_tensor_pointer2 = th.tensor([3, 4]).send(client)

    # placeholders for intermediate results
    result_tensor_pointer1 = th.tensor([0, 0]).send(client)
    result_tensor_pointer2 = th.tensor([0, 0]).send(client)
    result_tensor_pointer3 = th.tensor([0, 0]).send(client)

    # define plan
    a1 = RunClassMethodAction(
        path="torch.Tensor.mul",
        _self=input_tensor_pointer1,
        args=[model_tensor_pointer1],
        kwargs={},
        id_at_location=result_tensor_pointer1.id_at_location,
        address=Address(),
        msg_id=UID(),
    )

    a2 = RunClassMethodAction(
        path="torch.Tensor.add",
        _self=result_tensor_pointer1,
        args=[model_tensor_pointer2],
        kwargs={},
        id_at_location=result_tensor_pointer2.id_at_location,
        address=Address(),
        msg_id=UID(),
    )

    a3 = RunFunctionOrConstructorAction(
        path="torch.eq",
        args=[result_tensor_pointer2, input_tensor_pointer2],
        kwargs={},
        id_at_location=result_tensor_pointer3.id_at_location,
        address=Address(),
        msg_id=UID(),
    )

    plan = Plan(
        [a1, a2, a3], inputs={"x": input_tensor_pointer1, "y": input_tensor_pointer2}
    )
    plan_pointer = plan.send(client)

    # Test
    # x is random input, y is the expected model(x)
    x_batches = [(th.tensor([1, 1]) + i).send(client) for i in range(2)]
    y_batches = [
        ((th.tensor([1, 1]) + i) * th.tensor([1, 2]) + th.tensor([3, 4])).send(client)
        for i in range(2)
    ]

    for x, y in zip(x_batches, y_batches):
        plan_pointer(x=x, y=y)

        # checks if (model(x) == y) == [True, True]
        assert all(result_tensor_pointer3.get(delete_obj=False))

def test_make_plan(client: sy.VirtualMachineClient) -> None:
    @make_plan
    def add_plan(inp=th.zeros((3))) -> th.Tensor:  # type: ignore
        return inp + inp

    input_tensor = th.tensor([1, 2, 3])
    plan_pointer = add_plan.send(client)
    res = plan_pointer(inp=input_tensor)
    assert th.equal(res.get()[0], th.tensor([2, 4, 6]))

@pytest.mark.xfail
def test_plan_deterministic_bytes(root_client: sy.VirtualMachineClient) -> None:
    # TODO: https://github.com/OpenMined/PySyft/issues/5292
    @make_plan
    def add_plan(inp=th.zeros((3))) -> th.Tensor:  # type: ignore
        return inp + inp

    @make_plan
    def add_plan2(inp=th.zeros((3))) -> th.Tensor:  # type: ignore
        return inp + inp

    plan_pointer = add_plan.send(root_client)
    plan2_pointer = add_plan2.send(root_client)

    plan1 = serialize(plan_pointer, to_bytes=True)
    plan2 = serialize(plan2_pointer, to_bytes=True)

    assert plan1 == plan2

def test_make_plan2(root_client: sy.VirtualMachineClient) -> None:
    @make_plan
    def mul_plan(inp=th.zeros((3)), inp2=th.zeros((3))) -> th.Tensor:  # type: ignore
        return inp * inp2

    t1, t2 = th.tensor([1, 2, 3]), th.tensor([1, 2, 3])
    plan_pointer = mul_plan.send(root_client)
    res = plan_pointer(inp=t1, inp2=t2)
    assert th.equal(res.get()[0], th.tensor([1, 4, 9]))

def test_make_plan_error_no_kwargs() -> None:
    def assertRaises(exc, obj, methodname, *args) -> None:  # type: ignore
        with pytest.raises(exc) as e_info:
            getattr(obj, methodname)(*args)
        assert str(e_info) != ""

    def test_define_plan():  # type: ignore
        @make_plan
        def add_plan(inp):  # type: ignore
            return inp + inp

    # we can only define a plan with *kwargs* when using the @make_plan decorator, not with args
    assertRaises(ValueError, test_define_plan, "__call__")

@pytest.mark.skip(reason="Hypothesis: serde never truly worked")
@pytest.mark.slow
def test_mlp_plan(client: sy.VirtualMachineClient) -> None:
    class MLP(sy.Module):
        def __init__(self, torch_ref):  # type: ignore
            super().__init__(torch_ref=torch_ref)
            self.l1 = self.torch_ref.nn.Linear(784, 100)
            self.a1 = self.torch_ref.nn.ReLU()
            self.l2 = self.torch_ref.nn.Linear(100, 10)

        def forward(self, x):  # type: ignore
            x_reshaped = x.view(-1, 28 * 28)
            l1_out = self.a1(self.l1(x_reshaped))
            l2_out = self.l2(l1_out)
            return l2_out

    def set_params(model, params):  # type: ignore
        """happens outside of plan"""
        for p, p_new in zip(model.parameters(), params):
            p.data = p_new.data

    def cross_entropy_loss(logits, targets, batch_size):  # type: ignore
        norm_logits = logits - logits.max()
        log_probs = norm_logits - norm_logits.exp().sum(dim=1, keepdim=True).log()
        return -(targets * log_probs).sum() / batch_size

    def sgd_step(model, lr=0.1):  # type: ignore
        with ROOT_CLIENT.torch.no_grad():
            for p in model.parameters():
                p.data = p.data - lr * p.grad
                # Todo: fix this
                p.grad = th.zeros_like(p.grad.get())

    local_model = MLP(th)  # type: ignore

    @make_plan
    def train(  # type: ignore
        xs=th.rand([64 * 3, 1, 28, 28]),
        ys=th.randint(0, 10, [64 * 3, 10]),
        params=List(local_model.parameters()),
    ):

        model = local_model.send(ROOT_CLIENT)
        set_params(model, params)
        for i in range(2):
            indices = th.tensor(range(64 * i, 64 * (i + 1)))
            x, y = xs.index_select(0, indices), ys.index_select(0, indices)
            out = model(x)
            loss = cross_entropy_loss(out, y, 64)
            loss.backward()
            sgd_step(model)

        return model.parameters()

    train_ptr = train.send(client)

    old_params = local_model.parameters().copy()

    res_ptr = train_ptr(
        xs=th.rand([64 * 3, 1, 28, 28]),
        ys=th.randint(0, 10, [64 * 3, 10]),
        params=local_model.parameters(),
    )

    (new_params,) = res_ptr.get()

    assert not (old_params[0] == new_params[0]).all()

def test_check_placeholder() -> None:
    def test_define_plan() -> None:
        @make_plan
        def add_plan(inp: Any) -> Any:
            return inp + inp

    def assertRaises(
        exc: Exception, obj: object, methodname: str, *args: TypeTuple[Any, ...]
    ) -> None:
        with pytest.raises(exc) as e_info:
            getattr(obj, methodname)(*args)
        assert str(e_info) != ""

    assertRaises(ValueError, test_define_plan, "__call__")  # type: ignore
<|endoftext|>


<commit_before>from __future__ import print_function
import os
from os.path import join
from datetime import datetime
from dataactcore.aws.s3UrlHandler import s3UrlHandler
from dataactcore.models.domainModels import TASLookup
from dataactcore.models.stagingModels import AwardFinancial
from dataactcore.models.validationModels import RuleSql
from dataactcore.config import CONFIG_BROKER
from dataactvalidator.filestreaming.sqlLoader import SQLLoader
from dataactvalidator.filestreaming.schemaLoader import SchemaLoader
from dataactvalidator.filestreaming.loadFile import loadDomainValues
from dataactvalidator.scripts.loadTas import loadTas
from tests.integration.baseTestValidator import BaseTestValidator
import unittest

class FileTypeTests(BaseTestValidator):

    @classmethod
    def setUpClass(cls):
        """Set up class-wide resources."""
        super(FileTypeTests, cls).setUpClass()
        #TODO: refactor into a pytest fixture

        user = cls.userId
        # TODO: get rid of this flag once we're using a tempdb for test fixtures
        force_tas_load = False

        print("Uploading files")
        # Upload needed files to S3
        s3FileNameValid = cls.uploadFile("appropValid.csv", user)
        s3FileNameMixed = cls.uploadFile("appropMixed.csv", user)
        s3FileNameProgramValid = cls.uploadFile("programActivityValid.csv", user)
        s3FileNameProgramMixed = cls.uploadFile("programActivityMixed.csv", user)
        s3FileNameAwardFinValid = cls.uploadFile("awardFinancialValid.csv", user)
        s3FileNameAwardFinMixed = cls.uploadFile("awardFinancialMixed.csv", user)
        s3FileNameAwardValid = cls.uploadFile("awardValid.csv", user)
        s3FileNameAwardMixed = cls.uploadFile("awardMixed.csv", user)
        s3FileNameAwardMixedDelimiter = cls.uploadFile("awardMixedDelimiter.csv", user)
        s3FileNameCrossAwardFin = cls.uploadFile("cross_file_C.csv", user)
        s3FileNameCrossAward = cls.uploadFile("cross_file_D2.csv", user)
        s3FileNameCrossApprop = cls.uploadFile("cross_file_A.csv", user)
        s3FileNameCrossPgmAct = cls.uploadFile("cross_file_B.csv", user)
        s3FileNameAppropValidShortcols = cls.uploadFile("appropValidShortcols.csv", user)
        s3FileNameProgramMixedShortcols = cls.uploadFile("programActivityMixedShortcols.csv", user)
        s3FileNameAwardFinMixedShortcols = cls.uploadFile("awardFinancialMixedShortcols.csv", user)
        s3FileNameAwardValidShortcols = cls.uploadFile("awardValidShortcols.csv", user)

        # Create submissions and get IDs back
        submissionIDs = {}
        for i in range(0, 16):
            if i == 7:
               # Award financial mixed will be second quarter
                submissionIDs[i] = cls.insertSubmission(cls.jobTracker, user, datetime(2015,3,15))
            else:
                submissionIDs[i] = cls.insertSubmission(cls.jobTracker, user)

        # Create jobs
        jobDb = cls.jobTracker
        statusReady = str(jobDb.getJobStatusId("ready"))
        jobTypeCsv = str(jobDb.getJobTypeId("csv_record_validation"))
        jobTypeValidation = str(jobDb.getJobTypeId("validation"))
        jobInfoList = {
            "valid": [statusReady, jobTypeCsv, str(submissionIDs[1]), s3FileNameValid, 1],
            "mixed": [statusReady, jobTypeCsv, str(submissionIDs[2]), s3FileNameMixed, 1],
            "programValid": [statusReady, jobTypeCsv, str(submissionIDs[4]), s3FileNameProgramValid, 2],
            "programMixed": [statusReady, jobTypeCsv, str(submissionIDs[5]), s3FileNameProgramMixed, 2],
            "awardFinValid": [statusReady, jobTypeCsv, str(submissionIDs[6]), s3FileNameAwardFinValid, 3],
            "awardFinMixed": [statusReady, jobTypeCsv, str(submissionIDs[7]), s3FileNameAwardFinMixed, 3],
            "awardValid": [statusReady, jobTypeCsv, str(submissionIDs[8]), s3FileNameAwardValid, 4],
            "awardMixed": [statusReady, jobTypeCsv, str(submissionIDs[9]), s3FileNameAwardMixed, 4],
            "awardMixedDelimiter": [statusReady, jobTypeCsv, str(submissionIDs[10]), s3FileNameAwardMixedDelimiter, 4],
            "crossApprop": [statusReady, jobTypeCsv, str(submissionIDs[11]), s3FileNameCrossApprop, 1],
            "crossPgmAct": [statusReady, jobTypeCsv, str(submissionIDs[11]), s3FileNameCrossPgmAct, 2],
            "crossAwardFin": [statusReady, jobTypeCsv, str(submissionIDs[11]), s3FileNameCrossAwardFin, 3],
            "crossAward": [statusReady, jobTypeCsv, str(submissionIDs[11]), s3FileNameCrossAward, 4],
            "crossFile": [statusReady, jobTypeValidation, str(submissionIDs[11]), None, None],
            "appropValidShortcols": [statusReady, jobTypeCsv, str(submissionIDs[12]), s3FileNameAppropValidShortcols, 1],
            "programMixedShortcols": [statusReady, jobTypeCsv, str(submissionIDs[13]), s3FileNameProgramMixedShortcols, 2],
            "awardFinMixedShortcols": [statusReady, jobTypeCsv, str(submissionIDs[14]), s3FileNameAwardFinMixedShortcols, 3],
            "awardValidShortcols": [statusReady, jobTypeCsv, str(submissionIDs[15]), s3FileNameAwardValidShortcols, 4]
        }

        jobIdDict = {}
        for key in jobInfoList:
            jobInfo = jobInfoList[key]  # Done this way to be compatible with python 2 and 3
            jobInfo.append(jobDb.session)
            job = cls.addJob(*jobInfo)
            jobId = job.job_id
            jobIdDict[key] = jobId
            print("".join([str(key),": ",str(cls.jobTracker.getSubmissionId(jobId)), ", "]), end = "")

        # Load fields and rules
        FileTypeTests.load_definitions(cls.interfaces, force_tas_load)

        cls.jobIdDict = jobIdDict

    @staticmethod
    def load_definitions(interfaces, force_tas_load):
        """Load file definitions."""
        SchemaLoader.loadAllFromPath(join(CONFIG_BROKER["path"],"dataactvalidator","config"))
        SQLLoader.loadSql("sqlRules.csv")

        # @todo: here is a stop-gap solution; we've added new rules which are
        # filtering out previously usable test data. Temporarily remove these
        # rules
        to_delete = interfaces.validationDb.session.query(RuleSql).filter(
            RuleSql.rule_label.in_(('A30', 'A32', 'A33', 'B20', 'C18', 'C19')))
        for rule in to_delete:
            interfaces.validationDb.session.delete(rule)
        interfaces.validationDb.session.commit()

        # Load domain values tables
        loadDomainValues(
            join(CONFIG_BROKER["path"], "dataactvalidator", "config"),
            join(CONFIG_BROKER["path"], os.path.join("tests", "integration", "data"), "sf_133.csv"),
            join(CONFIG_BROKER["path"], os.path.join("tests", "integration", "data"), "program_activity.csv"))
        if (interfaces.validationDb.session.query(TASLookup).count() == 0
                or force_tas_load):
            # TAS table is empty, load it
            loadTas(tasFile=os.path.join(CONFIG_BROKER["path"], "tests", "integration", "data", "all_tas_betc.csv"), dropIdx=False)

    def test_approp_valid(self):
        """Test valid job."""
        jobId = self.jobIdDict["valid"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_approp_valid_shortcol(self):
        """Test valid approp job with short colnames."""
        jobId = self.jobIdDict["appropValidShortcols"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_approp_mixed(self):
        """Test mixed job with some rows failing."""
        jobId = self.jobIdDict["mixed"]
        self.passed = self.run_test(
            jobId, 200, "finished", 8212, 4, "complete", 39, True, 8, 841)

    def test_program_valid(self):
        """Test valid job."""
        jobId = self.jobIdDict["programValid"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_program_mixed(self):
        """Test mixed job with some rows failing."""
        jobId = self.jobIdDict["programMixed"]
        self.passed = self.run_test(
        jobId, 200, "finished", 11390, 4, "complete", 81, True, 29, 10853)

    def test_program_mixed_shortcols(self):
        """Test object class/program activity job with some rows failing & short colnames."""
        jobId = self.jobIdDict["programMixedShortcols"]
        self.passed = self.run_test(
            jobId, 200, "finished", 11390, 4, "complete", 81, True, 29, 10853)

    def test_award_fin_valid(self):
        """Test valid job."""
        jobId = self.jobIdDict["awardFinValid"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 11, "complete", 0, False)

    def test_award_fin_mixed(self):
        """Test mixed job with some rows failing."""
        jobId = self.jobIdDict["awardFinMixed"]
        self.passed = self.run_test(
        jobId, 200, "finished", 7332, 7, "complete", 45, True, 32, 9795)

        # Test that whitespace is converted to null
        rowThree = self.interfaces.validationDb.session.query(AwardFinancial).filter(AwardFinancial.parent_award_id == "ZZZZ").filter(AwardFinancial.submission_id == self.interfaces.jobDb.getSubmissionId(jobId)).first()
        self.assertIsNone(rowThree.agency_identifier)
        self.assertIsNone(rowThree.piid)
        # And commas removed for numeric
        rowThirteen = self.interfaces.validationDb.session.query(AwardFinancial).filter(AwardFinancial.parent_award_id == "YYYY").filter(AwardFinancial.submission_id == self.interfaces.jobDb.getSubmissionId(jobId)).first()
        self.assertEqual(rowThirteen.deobligations_recov_by_awa_cpe,26000)

    def test_award_fin_mixed_shortcols(self):
        """Test award financial job with some rows failing & short colnames."""
        jobId = self.jobIdDict["awardFinMixedShortcols"]
        self.passed = self.run_test(
            jobId, 200, "finished", 7332, 7, "complete", 45, True, 34, 11737)

    def test_award_valid(self):
        """Test valid job."""
        jobId = self.jobIdDict["awardValid"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_award_valid_shortcols(self):
        """Test valid award (financial assistance) job with short colnames."""
        jobId = self.jobIdDict["awardValidShortcols"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_award_mixed(self):
        """Test mixed job with some rows failing."""
        jobId = self.jobIdDict["awardMixed"]
        self.passed = self.run_test(
            jobId, 200, "finished", 1139, 8, "complete", 17, True, 24, 2085)

    def test_award_mixed_delimiter(self):
        """Test mixed job with mixed delimiter"""
        jobId = self.jobIdDict["awardMixedDelimiter"]
        self.passed = self.run_test(
            jobId, 400, "invalid", False, False, "header_error", 0, False)

    def test_cross_file(self):
        crossId = self.jobIdDict["crossFile"]
        # Run jobs for A, B, C, and D2, then cross file validation job
        # Note: test files used for cross validation use the short column names
        # as a way to ensure those are handled correctly by the validator
        awardFinResponse = self.validateJob(self.jobIdDict["crossAwardFin"],self.useThreads)
        self.assertEqual(awardFinResponse.status_code, 200, msg=str(awardFinResponse.json))
        awardResponse = self.validateJob(self.jobIdDict["crossAward"],self.useThreads)
        self.assertEqual(awardResponse.status_code, 200, msg=str(awardResponse.json))
        appropResponse = self.validateJob(self.jobIdDict["crossApprop"], self.useThreads)
        self.assertEqual(appropResponse.status_code, 200, msg=str(appropResponse.json))
        pgmActResponse = self.validateJob(self.jobIdDict["crossPgmAct"], self.useThreads)
        self.assertEqual(pgmActResponse.status_code, 200, msg=str(pgmActResponse.json))
        crossFileResponse = self.validateJob(crossId, self.useThreads)
        self.assertEqual(crossFileResponse.status_code, 200, msg=str(crossFileResponse.json))

        # Check number of cross file validation errors in DB for this job
        self.assertEqual(self.interfaces.errorDb.checkNumberOfErrorsByJobId(crossId, self.interfaces.validationDb, "fatal"), 0)
        self.assertEqual(self.interfaces.errorDb.checkNumberOfErrorsByJobId(crossId, self.interfaces.validationDb, "warning"), 5)
        # Check cross file job complete
        self.waitOnJob(self.interfaces.jobDb, crossId, "finished", self.useThreads)
        # Check that cross file validation report exists and is the right size
        jobTracker = self.interfaces.jobDb

        submissionId = jobTracker.getSubmissionId(crossId)
        sizePathPairs = [
            (89, self.interfaces.errorDb.getCrossReportName(submissionId, "appropriations", "program_activity")),
            (89, self.interfaces.errorDb.getCrossReportName(submissionId, "award_financial", "award")),
            (2348, self.interfaces.errorDb.getCrossWarningReportName(submissionId, "appropriations", "program_activity")),
            (424, self.interfaces.errorDb.getCrossWarningReportName(submissionId, "award_financial", "award")),
        ]

        for size, path in sizePathPairs:
            if self.local:
                self.assertFileSizeAppxy(size, path)
            else:
                self.assertGreater(
                    s3UrlHandler.getFileSize("errors/" + path), size - 5)
                self.assertLess(
                    s3UrlHandler.getFileSize("errors/" + path), size + 5)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Added B19 and C19 to ignore list<commit_after>from __future__ import print_function
import os
from os.path import join
from datetime import datetime
from dataactcore.aws.s3UrlHandler import s3UrlHandler
from dataactcore.models.domainModels import TASLookup
from dataactcore.models.stagingModels import AwardFinancial
from dataactcore.models.validationModels import RuleSql
from dataactcore.config import CONFIG_BROKER
from dataactvalidator.filestreaming.sqlLoader import SQLLoader
from dataactvalidator.filestreaming.schemaLoader import SchemaLoader
from dataactvalidator.filestreaming.loadFile import loadDomainValues
from dataactvalidator.scripts.loadTas import loadTas
from tests.integration.baseTestValidator import BaseTestValidator
import unittest

class FileTypeTests(BaseTestValidator):

    @classmethod
    def setUpClass(cls):
        """Set up class-wide resources."""
        super(FileTypeTests, cls).setUpClass()
        #TODO: refactor into a pytest fixture

        user = cls.userId
        # TODO: get rid of this flag once we're using a tempdb for test fixtures
        force_tas_load = False

        print("Uploading files")
        # Upload needed files to S3
        s3FileNameValid = cls.uploadFile("appropValid.csv", user)
        s3FileNameMixed = cls.uploadFile("appropMixed.csv", user)
        s3FileNameProgramValid = cls.uploadFile("programActivityValid.csv", user)
        s3FileNameProgramMixed = cls.uploadFile("programActivityMixed.csv", user)
        s3FileNameAwardFinValid = cls.uploadFile("awardFinancialValid.csv", user)
        s3FileNameAwardFinMixed = cls.uploadFile("awardFinancialMixed.csv", user)
        s3FileNameAwardValid = cls.uploadFile("awardValid.csv", user)
        s3FileNameAwardMixed = cls.uploadFile("awardMixed.csv", user)
        s3FileNameAwardMixedDelimiter = cls.uploadFile("awardMixedDelimiter.csv", user)
        s3FileNameCrossAwardFin = cls.uploadFile("cross_file_C.csv", user)
        s3FileNameCrossAward = cls.uploadFile("cross_file_D2.csv", user)
        s3FileNameCrossApprop = cls.uploadFile("cross_file_A.csv", user)
        s3FileNameCrossPgmAct = cls.uploadFile("cross_file_B.csv", user)
        s3FileNameAppropValidShortcols = cls.uploadFile("appropValidShortcols.csv", user)
        s3FileNameProgramMixedShortcols = cls.uploadFile("programActivityMixedShortcols.csv", user)
        s3FileNameAwardFinMixedShortcols = cls.uploadFile("awardFinancialMixedShortcols.csv", user)
        s3FileNameAwardValidShortcols = cls.uploadFile("awardValidShortcols.csv", user)

        # Create submissions and get IDs back
        submissionIDs = {}
        for i in range(0, 16):
            if i == 7:
               # Award financial mixed will be second quarter
                submissionIDs[i] = cls.insertSubmission(cls.jobTracker, user, datetime(2015,3,15))
            else:
                submissionIDs[i] = cls.insertSubmission(cls.jobTracker, user)

        # Create jobs
        jobDb = cls.jobTracker
        statusReady = str(jobDb.getJobStatusId("ready"))
        jobTypeCsv = str(jobDb.getJobTypeId("csv_record_validation"))
        jobTypeValidation = str(jobDb.getJobTypeId("validation"))
        jobInfoList = {
            "valid": [statusReady, jobTypeCsv, str(submissionIDs[1]), s3FileNameValid, 1],
            "mixed": [statusReady, jobTypeCsv, str(submissionIDs[2]), s3FileNameMixed, 1],
            "programValid": [statusReady, jobTypeCsv, str(submissionIDs[4]), s3FileNameProgramValid, 2],
            "programMixed": [statusReady, jobTypeCsv, str(submissionIDs[5]), s3FileNameProgramMixed, 2],
            "awardFinValid": [statusReady, jobTypeCsv, str(submissionIDs[6]), s3FileNameAwardFinValid, 3],
            "awardFinMixed": [statusReady, jobTypeCsv, str(submissionIDs[7]), s3FileNameAwardFinMixed, 3],
            "awardValid": [statusReady, jobTypeCsv, str(submissionIDs[8]), s3FileNameAwardValid, 4],
            "awardMixed": [statusReady, jobTypeCsv, str(submissionIDs[9]), s3FileNameAwardMixed, 4],
            "awardMixedDelimiter": [statusReady, jobTypeCsv, str(submissionIDs[10]), s3FileNameAwardMixedDelimiter, 4],
            "crossApprop": [statusReady, jobTypeCsv, str(submissionIDs[11]), s3FileNameCrossApprop, 1],
            "crossPgmAct": [statusReady, jobTypeCsv, str(submissionIDs[11]), s3FileNameCrossPgmAct, 2],
            "crossAwardFin": [statusReady, jobTypeCsv, str(submissionIDs[11]), s3FileNameCrossAwardFin, 3],
            "crossAward": [statusReady, jobTypeCsv, str(submissionIDs[11]), s3FileNameCrossAward, 4],
            "crossFile": [statusReady, jobTypeValidation, str(submissionIDs[11]), None, None],
            "appropValidShortcols": [statusReady, jobTypeCsv, str(submissionIDs[12]), s3FileNameAppropValidShortcols, 1],
            "programMixedShortcols": [statusReady, jobTypeCsv, str(submissionIDs[13]), s3FileNameProgramMixedShortcols, 2],
            "awardFinMixedShortcols": [statusReady, jobTypeCsv, str(submissionIDs[14]), s3FileNameAwardFinMixedShortcols, 3],
            "awardValidShortcols": [statusReady, jobTypeCsv, str(submissionIDs[15]), s3FileNameAwardValidShortcols, 4]
        }

        jobIdDict = {}
        for key in jobInfoList:
            jobInfo = jobInfoList[key]  # Done this way to be compatible with python 2 and 3
            jobInfo.append(jobDb.session)
            job = cls.addJob(*jobInfo)
            jobId = job.job_id
            jobIdDict[key] = jobId
            print("".join([str(key),": ",str(cls.jobTracker.getSubmissionId(jobId)), ", "]), end = "")

        # Load fields and rules
        FileTypeTests.load_definitions(cls.interfaces, force_tas_load)

        cls.jobIdDict = jobIdDict

    @staticmethod
    def load_definitions(interfaces, force_tas_load):
        """Load file definitions."""
        SchemaLoader.loadAllFromPath(join(CONFIG_BROKER["path"],"dataactvalidator","config"))
        SQLLoader.loadSql("sqlRules.csv")

        # @todo: here is a stop-gap solution; we've added new rules which are
        # filtering out previously usable test data. Temporarily remove these
        # rules
        to_delete = interfaces.validationDb.session.query(RuleSql).filter(
            RuleSql.rule_label.in_(('A30', 'A32', 'A33', 'B20', 'B19', 'C19')))
        for rule in to_delete:
            interfaces.validationDb.session.delete(rule)
        interfaces.validationDb.session.commit()

        # Load domain values tables
        loadDomainValues(
            join(CONFIG_BROKER["path"], "dataactvalidator", "config"),
            join(CONFIG_BROKER["path"], os.path.join("tests", "integration", "data"), "sf_133.csv"),
            join(CONFIG_BROKER["path"], os.path.join("tests", "integration", "data"), "program_activity.csv"))
        if (interfaces.validationDb.session.query(TASLookup).count() == 0
                or force_tas_load):
            # TAS table is empty, load it
            loadTas(tasFile=os.path.join(CONFIG_BROKER["path"], "tests", "integration", "data", "all_tas_betc.csv"), dropIdx=False)

    def test_approp_valid(self):
        """Test valid job."""
        jobId = self.jobIdDict["valid"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_approp_valid_shortcol(self):
        """Test valid approp job with short colnames."""
        jobId = self.jobIdDict["appropValidShortcols"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_approp_mixed(self):
        """Test mixed job with some rows failing."""
        jobId = self.jobIdDict["mixed"]
        self.passed = self.run_test(
            jobId, 200, "finished", 8212, 4, "complete", 39, True, 8, 841)

    def test_program_valid(self):
        """Test valid job."""
        jobId = self.jobIdDict["programValid"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_program_mixed(self):
        """Test mixed job with some rows failing."""
        jobId = self.jobIdDict["programMixed"]
        self.passed = self.run_test(
        jobId, 200, "finished", 11390, 4, "complete", 81, True, 29, 10840)

    def test_program_mixed_shortcols(self):
        """Test object class/program activity job with some rows failing & short colnames."""
        jobId = self.jobIdDict["programMixedShortcols"]
        self.passed = self.run_test(
            jobId, 200, "finished", 11390, 4, "complete", 81, True, 29, 10840)

    def test_award_fin_valid(self):
        """Test valid job."""
        jobId = self.jobIdDict["awardFinValid"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 11, "complete", 0, False)

    def test_award_fin_mixed(self):
        """Test mixed job with some rows failing."""
        jobId = self.jobIdDict["awardFinMixed"]
        self.passed = self.run_test(
        jobId, 200, "finished", 7332, 7, "complete", 45, True, 32, 9795)

        # Test that whitespace is converted to null
        rowThree = self.interfaces.validationDb.session.query(AwardFinancial).filter(AwardFinancial.parent_award_id == "ZZZZ").filter(AwardFinancial.submission_id == self.interfaces.jobDb.getSubmissionId(jobId)).first()
        self.assertIsNone(rowThree.agency_identifier)
        self.assertIsNone(rowThree.piid)
        # And commas removed for numeric
        rowThirteen = self.interfaces.validationDb.session.query(AwardFinancial).filter(AwardFinancial.parent_award_id == "YYYY").filter(AwardFinancial.submission_id == self.interfaces.jobDb.getSubmissionId(jobId)).first()
        self.assertEqual(rowThirteen.deobligations_recov_by_awa_cpe,26000)

    def test_award_fin_mixed_shortcols(self):
        """Test award financial job with some rows failing & short colnames."""
        jobId = self.jobIdDict["awardFinMixedShortcols"]
        self.passed = self.run_test(
            jobId, 200, "finished", 7332, 7, "complete", 45, True, 34, 11737)

    def test_award_valid(self):
        """Test valid job."""
        jobId = self.jobIdDict["awardValid"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_award_valid_shortcols(self):
        """Test valid award (financial assistance) job with short colnames."""
        jobId = self.jobIdDict["awardValidShortcols"]
        self.passed = self.run_test(
            jobId, 200, "finished", 63, 10, "complete", 0, False)

    def test_award_mixed(self):
        """Test mixed job with some rows failing."""
        jobId = self.jobIdDict["awardMixed"]
        self.passed = self.run_test(
            jobId, 200, "finished", 1139, 8, "complete", 17, True, 24, 2085)

    def test_award_mixed_delimiter(self):
        """Test mixed job with mixed delimiter"""
        jobId = self.jobIdDict["awardMixedDelimiter"]
        self.passed = self.run_test(
            jobId, 400, "invalid", False, False, "header_error", 0, False)

    def test_cross_file(self):
        crossId = self.jobIdDict["crossFile"]
        # Run jobs for A, B, C, and D2, then cross file validation job
        # Note: test files used for cross validation use the short column names
        # as a way to ensure those are handled correctly by the validator
        awardFinResponse = self.validateJob(self.jobIdDict["crossAwardFin"],self.useThreads)
        self.assertEqual(awardFinResponse.status_code, 200, msg=str(awardFinResponse.json))
        awardResponse = self.validateJob(self.jobIdDict["crossAward"],self.useThreads)
        self.assertEqual(awardResponse.status_code, 200, msg=str(awardResponse.json))
        appropResponse = self.validateJob(self.jobIdDict["crossApprop"], self.useThreads)
        self.assertEqual(appropResponse.status_code, 200, msg=str(appropResponse.json))
        pgmActResponse = self.validateJob(self.jobIdDict["crossPgmAct"], self.useThreads)
        self.assertEqual(pgmActResponse.status_code, 200, msg=str(pgmActResponse.json))
        crossFileResponse = self.validateJob(crossId, self.useThreads)
        self.assertEqual(crossFileResponse.status_code, 200, msg=str(crossFileResponse.json))

        # Check number of cross file validation errors in DB for this job
        self.assertEqual(self.interfaces.errorDb.checkNumberOfErrorsByJobId(crossId, self.interfaces.validationDb, "fatal"), 0)
        self.assertEqual(self.interfaces.errorDb.checkNumberOfErrorsByJobId(crossId, self.interfaces.validationDb, "warning"), 5)
        # Check cross file job complete
        self.waitOnJob(self.interfaces.jobDb, crossId, "finished", self.useThreads)
        # Check that cross file validation report exists and is the right size
        jobTracker = self.interfaces.jobDb

        submissionId = jobTracker.getSubmissionId(crossId)
        sizePathPairs = [
            (89, self.interfaces.errorDb.getCrossReportName(submissionId, "appropriations", "program_activity")),
            (89, self.interfaces.errorDb.getCrossReportName(submissionId, "award_financial", "award")),
            (2348, self.interfaces.errorDb.getCrossWarningReportName(submissionId, "appropriations", "program_activity")),
            (424, self.interfaces.errorDb.getCrossWarningReportName(submissionId, "award_financial", "award")),
        ]

        for size, path in sizePathPairs:
            if self.local:
                self.assertFileSizeAppxy(size, path)
            else:
                self.assertGreater(
                    s3UrlHandler.getFileSize("errors/" + path), size - 5)
                self.assertLess(
                    s3UrlHandler.getFileSize("errors/" + path), size + 5)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import commands
import os
import Queue
import settings
import time
import threading
import serial
import sys
import traceback

from thirtybirds_2_0.Network.manager import init as network_init
from thirtybirds_2_0.Updates.manager import init as updates_init

class Network(object):
    def __init__(self, hostname, network_message_handler, network_status_handler):
        self.hostname = hostname
        self.thirtybirds = network_init(
            hostname=hostname,
            role="client",
            discovery_multicastGroup=settings.discovery_multicastGroup,
            discovery_multicastPort=settings.discovery_multicastPort,
            discovery_responsePort=settings.discovery_responsePort,
            pubsub_pubPort=settings.pubsub_pubPort,
            message_callback=network_message_handler,
            status_callback=network_status_handler
        )

########################
## UTILS
########################

class Utils(object):
    def __init__(self, hostname):
        self.hostname = hostname
    def reboot(self):
        os.system("sudo reboot now")

    def remote_update_git(self, oratio, thirtybirds, update, upgrade):
        if oratio:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/oratio')
        if thirtybirds:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/thirtybirds_2_0')
        return

    def remote_update_scripts(self):
        updates_init("/home/pi/oratio", False, True)
        return

    def get_update_script_version(self):
        (updates, ghStatus, bsStatus) = updates_init("/home/pi/oratio", False, False)
        return updates.read_version_pickle()

    def get_git_timestamp(self):
        return commands.getstatusoutput("cd /home/pi/oratio/; git log -1 --format=%cd")[1]

    def get_temp(self):
        return commands.getstatusoutput("/opt/vc/bin/vcgencmd measure_temp")[1]

    def get_cpu(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[12]

    def get_uptime(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[4]

    def get_disk(self):
        # stub for now
        return "0"

    def get_client_status(self):
        return (self.hostname, self.get_update_script_version(), self.get_git_timestamp(), self.get_temp(), self.get_cpu(), self.get_uptime(), self.get_disk())

# Main handles network send/recv and can see all other classes directly
class Main(threading.Thread):
    def __init__(self, hostname):
        threading.Thread.__init__(self)
        print 10000
        self.network = Network(hostname, self.network_message_handler, self.network_status_handler)
        print 10001
        self.queue = Queue.Queue()
        self.arduino_connection = open("/dev/ttyACM0",'w')
        print 10002
        self.utils = Utils(hostname)
        self.network.thirtybirds.subscribe_to_topic("mandala_device_status")
        self.UNSET = 0
        self.FAIL = 2000
        self.PASS = 4000
        self.mandala_device_status = None
        self.mandala_tlc_ids = {
            "avl-controller":39,
            "avl-formant-1":15,
            "avl-formant-1-amplifier":4,
            "avl-formant-2":16,
            "avl-formant-2-amplifier":5,
            "avl-formant-3":17,
            "avl-formant-3-amplifier":6,
            "avl-layer-1":40,
            "avl-layer-2":11,
            "avl-layer-3":12,
            "avl-medulla":35,
            "avl-pitch-keys":18,
            "avl-pitch-keys-sensor-1":7,
            "avl-pitch-keys-sensor-2":8,
            "avl-pitch-keys-sensor-3":9,
            "avl-pitch-keys-sensor-4":10,
            "avl-settings":34,
            "avl-settings-adcs":24,
            "avl-transport":13,
            "avl-transport-encoder":0,
            "avl-voice-1":36,
            "avl-voice-1-crystal-frequency-counter":25,
            "avl-voice-1-harmonic-generators":26,
            "avl-voice-1-harmonic-volume":27,
            "avl-voice-2":37,
            "avl-voice-2-crystal-frequency-counter":28,
            "avl-voice-2-harmonic-generators":29,
            "avl-voice-2-harmonic-volume":30,
            "avl-voice-3":38,
            "avl-voice-3-crystal-frequency-counter":31,
            "avl-voice-3-harmonic-generators":32,
            "avl-voice-3-harmonic-volume":33,
            "avl-voice-keys":14,
            "avl-voice-keys-encoder-1":1,
            "avl-voice-keys-encoder-2":2,
            "avl-voice-keys-encoder-3":3
        }
        print 10003

    def network_message_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        topic, msg =  topic_msg # separating just to eval msg.  best to do it early.  it should be done in TB.
        #if len(msg) > 0:
        #    msg = eval(msg)
        self.add_to_queue(topic, msg)

    def network_status_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        print "Main.network_status_handler", topic_msg

    def add_to_queue(self, topic, msg):
        self.queue.put((topic, msg))

    def run(self):

        """
        devicenames = self.mandala_tlc_ids.keys()
        for devicename in devicenames:
            tlc_id_int = self.mandala_tlc_ids[devicename] + 5000
            tlc_id_str = "{}\n".format(tlc_id_int)
            self.ser.write(tlc_id_str)
            print self.ser.readln()
            self.ser.write("0/n")
            print self.ser.readln()
        """
        print 10004
        while True:
            print 10005
            if self.mandala_device_status == None:
                print 10006
                time.sleep(1)
                self.network.thirtybirds.send("mandala_device_request", True)
                print 10007

            print 10008
            try:
                print 10009
                topic, msg = self.queue.get(True)
                print topic, msg
                if topic == "mandala_device_status":
                    self.mandala_device_status = eval(msg)
                    devicenames = self.mandala_device_status.keys()
                    devicenames.sort()
                    for devicename in devicenames:
                        tlc_id_int = self.mandala_tlc_ids[devicename] + 5000
                        tlc_id_str = "{}\n".format(tlc_id_int)
                        if self.mandala_device_status[devicename] == "unset":
                            tlc_level_int = 0
                        if self.mandala_device_status[devicename] == "fail":
                            tlc_level_int = 100
                        if self.mandala_device_status[devicename] == "pass":
                            tlc_level_int = 4000
                        tlc_level_str = "{}\n".format(tlc_level_int)
                        print  repr(tlc_id_str),  repr(tlc_level_str),  devicename

                        time.sleep(0.01)
                        self.arduino_connection.write(tlc_id_str)
                        time.sleep(0.01)
                        self.arduino_connection.write(tlc_level_str)

                time.sleep(0.01)
            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                print e, repr(traceback.format_exception(exc_type, exc_value,exc_traceback))

def init(hostname):
    main = Main(hostname)
    main.daemon = True
    main.start()
    return main

<commit_msg>testing disabling serial DTR so Arduino doesn't reset<commit_after>import commands
import os
import Queue
import settings
import time
import threading
import serial
import sys
import traceback

from thirtybirds_2_0.Network.manager import init as network_init
from thirtybirds_2_0.Updates.manager import init as updates_init

class Network(object):
    def __init__(self, hostname, network_message_handler, network_status_handler):
        self.hostname = hostname
        self.thirtybirds = network_init(
            hostname=hostname,
            role="client",
            discovery_multicastGroup=settings.discovery_multicastGroup,
            discovery_multicastPort=settings.discovery_multicastPort,
            discovery_responsePort=settings.discovery_responsePort,
            pubsub_pubPort=settings.pubsub_pubPort,
            message_callback=network_message_handler,
            status_callback=network_status_handler
        )

########################
## UTILS
########################

class Utils(object):
    def __init__(self, hostname):
        self.hostname = hostname
    def reboot(self):
        os.system("sudo reboot now")

    def remote_update_git(self, oratio, thirtybirds, update, upgrade):
        if oratio:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/oratio')
        if thirtybirds:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/thirtybirds_2_0')
        return

    def remote_update_scripts(self):
        updates_init("/home/pi/oratio", False, True)
        return

    def get_update_script_version(self):
        (updates, ghStatus, bsStatus) = updates_init("/home/pi/oratio", False, False)
        return updates.read_version_pickle()

    def get_git_timestamp(self):
        return commands.getstatusoutput("cd /home/pi/oratio/; git log -1 --format=%cd")[1]

    def get_temp(self):
        return commands.getstatusoutput("/opt/vc/bin/vcgencmd measure_temp")[1]

    def get_cpu(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[12]

    def get_uptime(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[4]

    def get_disk(self):
        # stub for now
        return "0"

    def get_client_status(self):
        return (self.hostname, self.get_update_script_version(), self.get_git_timestamp(), self.get_temp(), self.get_cpu(), self.get_uptime(), self.get_disk())

# Main handles network send/recv and can see all other classes directly
class Main(threading.Thread):
    def __init__(self, hostname):
        threading.Thread.__init__(self)
        print 10000
        self.network = Network(hostname, self.network_message_handler, self.network_status_handler)
        print 10001
        self.queue = Queue.Queue()
        self.arduino_connection = open("/dev/ttyACM0",'w')
        print 10002
        self.utils = Utils(hostname)
        self.network.thirtybirds.subscribe_to_topic("mandala_device_status")
        self.UNSET = 0
        self.FAIL = 2000
        self.PASS = 4000
        self.mandala_device_status = None
        self.mandala_tlc_ids = {
            "avl-controller":39,
            "avl-formant-1":15,
            "avl-formant-1-amplifier":4,
            "avl-formant-2":16,
            "avl-formant-2-amplifier":5,
            "avl-formant-3":17,
            "avl-formant-3-amplifier":6,
            "avl-layer-1":40,
            "avl-layer-2":11,
            "avl-layer-3":12,
            "avl-medulla":35,
            "avl-pitch-keys":18,
            "avl-pitch-keys-sensor-1":7,
            "avl-pitch-keys-sensor-2":8,
            "avl-pitch-keys-sensor-3":9,
            "avl-pitch-keys-sensor-4":10,
            "avl-settings":34,
            "avl-settings-adcs":24,
            "avl-transport":13,
            "avl-transport-encoder":0,
            "avl-voice-1":36,
            "avl-voice-1-crystal-frequency-counter":25,
            "avl-voice-1-harmonic-generators":26,
            "avl-voice-1-harmonic-volume":27,
            "avl-voice-2":37,
            "avl-voice-2-crystal-frequency-counter":28,
            "avl-voice-2-harmonic-generators":29,
            "avl-voice-2-harmonic-volume":30,
            "avl-voice-3":38,
            "avl-voice-3-crystal-frequency-counter":31,
            "avl-voice-3-harmonic-generators":32,
            "avl-voice-3-harmonic-volume":33,
            "avl-voice-keys":14,
            "avl-voice-keys-encoder-1":1,
            "avl-voice-keys-encoder-2":2,
            "avl-voice-keys-encoder-3":3
        }
        print 10003

    def network_message_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        topic, msg =  topic_msg # separating just to eval msg.  best to do it early.  it should be done in TB.
        #if len(msg) > 0:
        #    msg = eval(msg)
        self.add_to_queue(topic, msg)

    def network_status_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        print "Main.network_status_handler", topic_msg

    def add_to_queue(self, topic, msg):
        self.queue.put((topic, msg))

    def run(self):

        """
        devicenames = self.mandala_tlc_ids.keys()
        for devicename in devicenames:
            tlc_id_int = self.mandala_tlc_ids[devicename] + 5000
            tlc_id_str = "{}\n".format(tlc_id_int)
            self.ser.write(tlc_id_str)
            print self.ser.readln()
            self.ser.write("0/n")
            print self.ser.readln()
        """
        print 10004
        while True:
            print 10005
            if self.mandala_device_status == None:
                print 10006
                time.sleep(1)
                self.network.thirtybirds.send("mandala_device_request", True)
                print 10007

            print 10008
            try:
                print 10009
                topic, msg = self.queue.get(True)
                print topic, msg
                if topic == "mandala_device_status":
                    self.mandala_device_status = eval(msg)
                    devicenames = self.mandala_device_status.keys()
                    devicenames.sort()
                    for devicename in devicenames:
                        tlc_id_int = self.mandala_tlc_ids[devicename] + 5000
                        tlc_id_str = "{}\n".format(tlc_id_int)
                        if self.mandala_device_status[devicename] == "unset":
                            tlc_level_int = 0
                        if self.mandala_device_status[devicename] == "fail":
                            tlc_level_int = 100
                        if self.mandala_device_status[devicename] == "pass":
                            tlc_level_int = 4000
                        tlc_level_str = "{}\n".format(tlc_level_int)
                        print  repr(tlc_id_str),  repr(tlc_level_str),  devicename

                        time.sleep(0.05)
                        self.arduino_connection.write(tlc_id_str)
                        time.sleep(0.05)
                        self.arduino_connection.write(tlc_level_str)

                time.sleep(0.01)
            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                print e, repr(traceback.format_exception(exc_type, exc_value,exc_traceback))

def init(hostname):
    main = Main(hostname)
    main.daemon = True
    main.start()
    return main

<|endoftext|>


<commit_before>import commands
import os
import Queue
import settings
import time
import threading
import serial
import sys
import traceback

from thirtybirds_2_0.Network.manager import init as network_init
from thirtybirds_2_0.Updates.manager import init as updates_init

class Network(object):
    def __init__(self, hostname, network_message_handler, network_status_handler):
        self.hostname = hostname
        self.thirtybirds = network_init(
            hostname=hostname,
            role="client",
            discovery_multicastGroup=settings.discovery_multicastGroup,
            discovery_multicastPort=settings.discovery_multicastPort,
            discovery_responsePort=settings.discovery_responsePort,
            pubsub_pubPort=settings.pubsub_pubPort,
            message_callback=network_message_handler,
            status_callback=network_status_handler
        )

########################
## UTILS
########################

class Utils(object):
    def __init__(self, hostname):
        self.hostname = hostname
    def reboot(self):
        os.system("sudo reboot now")

    def remote_update_git(self, oratio, thirtybirds, update, upgrade):
        if oratio:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/oratio')
        if thirtybirds:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/thirtybirds_2_0')
        return

    def remote_update_scripts(self):
        updates_init("/home/pi/oratio", False, True)
        return

    def get_update_script_version(self):
        (updates, ghStatus, bsStatus) = updates_init("/home/pi/oratio", False, False)
        return updates.read_version_pickle()

    def get_git_timestamp(self):
        return commands.getstatusoutput("cd /home/pi/oratio/; git log -1 --format=%cd")[1]

    def get_temp(self):
        return commands.getstatusoutput("/opt/vc/bin/vcgencmd measure_temp")[1]

    def get_cpu(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[12]

    def get_uptime(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[4]

    def get_disk(self):
        # stub for now
        return "0"

    def get_client_status(self):
        return (self.hostname, self.get_update_script_version(), self.get_git_timestamp(), self.get_temp(), self.get_cpu(), self.get_uptime(), self.get_disk())

# Main handles network send/recv and can see all other classes directly
class Main(threading.Thread):
    def __init__(self, hostname):
        threading.Thread.__init__(self)
        self.network = Network(hostname, self.network_message_handler, self.network_status_handler)
        self.queue = Queue.Queue()
        self.ser=serial.Serial("/dev/ttyACM0",9600)
        self.ser.baudrate=9600
        self.utils = Utils(hostname)
        self.network.thirtybirds.subscribe_to_topic("mandala_device_status")
        self.UNSET = 0
        self.FAIL = 2000
        self.PASS = 4000
        self.mandala_device_status = None
        self.mandala_tlc_ids = {
            "avl-controller":39,
            "avl-formant-1":15,
            "avl-formant-1-amplifier":4,
            "avl-formant-2":16,
            "avl-formant-2-amplifier":5,
            "avl-formant-3":17,
            "avl-formant-3-amplifier":6,
            "avl-layer-1":40,
            "avl-layer-2":11,
            "avl-layer-3":12,
            "avl-medulla":35,
            "avl-pitch-keys":18,
            "avl-pitch-keys-sensor-1":7,
            "avl-pitch-keys-sensor-2":8,
            "avl-pitch-keys-sensor-3":9,
            "avl-pitch-keys-sensor-4":10,
            "avl-settings":34,
            "avl-settings-adcs":24,
            "avl-transport":13,
            "avl-transport-encoder":0,
            "avl-voice-1":36,
            "avl-voice-1-crystal-frequency-counter":25,
            "avl-voice-1-harmonic-generators":26,
            "avl-voice-1-harmonic-volume":27,
            "avl-voice-2":37,
            "avl-voice-2-crystal-frequency-counter":28,
            "avl-voice-2-harmonic-generators":29,
            "avl-voice-2-harmonic-volume":30,
            "avl-voice-3":38,
            "avl-voice-3-crystal-frequency-counter":31,
            "avl-voice-3-harmonic-generators":32,
            "avl-voice-3-harmonic-volume":33,
            "avl-voice-keys":14,
            "avl-voice-keys-encoder-1":1,
            "avl-voice-keys-encoder-2":2,
            "avl-voice-keys-encoder-3":3
        }

    def network_message_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        topic, msg =  topic_msg # separating just to eval msg.  best to do it early.  it should be done in TB.
        #if len(msg) > 0:
        #    msg = eval(msg)
        self.add_to_queue(topic, msg)

    def network_status_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        print "Main.network_status_handler", topic_msg

    def add_to_queue(self, topic, msg):
        self.queue.put((topic, msg))

    def run(self):
        print self.ser

        self.ser.write("5000\n")
        self.ser.write("1000\n")
        self.ser.write("5001\n")
        self.ser.write("1000\n")
        self.ser.write("5002\n")
        self.ser.write("1000\n")
        self.ser.write("5003\n")
        self.ser.write("1000\n")
        self.ser.write("5004\n")
        self.ser.write("1000\n")

        print "asdfasdfasdf"

        devicenames = self.mandala_tlc_ids.keys()
        for devicename in devicenames:
            tlc_id_int = self.mandala_tlc_ids[devicename] + 5000
            tlc_id_str = "{}\n".format(tlc_id_int)
            self.ser.write(tlc_id_str)
            print self.ser.readln()
            self.ser.write("0/n")
            print self.ser.readln()
        while True:
            if self.mandala_device_status == None:
                time.sleep(1)
                self.network.thirtybirds.send("mandala_device_request", True)
            try:
                topic, msg = self.queue.get(True)
                print topic, msg
                if topic == "mandala_device_status":
                    self.mandala_device_status = eval(msg)
                    devicenames = self.mandala_device_status.keys()
                    devicenames.sort()
                    for devicename in devicenames:
                        tlc_id_int = self.mandala_tlc_ids[devicename] + 5000
                        tlc_id_str = "{}\n".format(tlc_id_int)
                        if self.mandala_device_status[devicename] == "unset":
                            tlc_level_int = 0
                        if self.mandala_device_status[devicename] == "fail":
                            tlc_level_int = 100
                        if self.mandala_device_status[devicename] == "pass":
                            tlc_level_int = 4000
                        tlc_level_str = "{}\n".format(tlc_level_int)
                        print  repr(tlc_id_str),  repr(tlc_level_str),  devicename
                        time.sleep(1)
                        self.ser.write(tlc_id_str)
                        time.sleep(1)
                        self.ser.write(tlc_level_str)

                time.sleep(0.01)
            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                print e, repr(traceback.format_exception(exc_type, exc_value,exc_traceback))

def init(hostname):
    main = Main(hostname)
    main.daemon = True
    main.start()
    return main

<commit_msg>adding long delays to serial coms because Adruino can't keep up.  Temporary solution.<commit_after>import commands
import os
import Queue
import settings
import time
import threading
import serial
import sys
import traceback

from thirtybirds_2_0.Network.manager import init as network_init
from thirtybirds_2_0.Updates.manager import init as updates_init

class Network(object):
    def __init__(self, hostname, network_message_handler, network_status_handler):
        self.hostname = hostname
        self.thirtybirds = network_init(
            hostname=hostname,
            role="client",
            discovery_multicastGroup=settings.discovery_multicastGroup,
            discovery_multicastPort=settings.discovery_multicastPort,
            discovery_responsePort=settings.discovery_responsePort,
            pubsub_pubPort=settings.pubsub_pubPort,
            message_callback=network_message_handler,
            status_callback=network_status_handler
        )

########################
## UTILS
########################

class Utils(object):
    def __init__(self, hostname):
        self.hostname = hostname
    def reboot(self):
        os.system("sudo reboot now")

    def remote_update_git(self, oratio, thirtybirds, update, upgrade):
        if oratio:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/oratio')
        if thirtybirds:
            subprocess.call(['sudo', 'git', 'pull'], cwd='/home/pi/thirtybirds_2_0')
        return

    def remote_update_scripts(self):
        updates_init("/home/pi/oratio", False, True)
        return

    def get_update_script_version(self):
        (updates, ghStatus, bsStatus) = updates_init("/home/pi/oratio", False, False)
        return updates.read_version_pickle()

    def get_git_timestamp(self):
        return commands.getstatusoutput("cd /home/pi/oratio/; git log -1 --format=%cd")[1]

    def get_temp(self):
        return commands.getstatusoutput("/opt/vc/bin/vcgencmd measure_temp")[1]

    def get_cpu(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[12]

    def get_uptime(self):
        bash_output = commands.getstatusoutput("uptime")[1]
        split_output = bash_output.split(" ")
        return split_output[4]

    def get_disk(self):
        # stub for now
        return "0"

    def get_client_status(self):
        return (self.hostname, self.get_update_script_version(), self.get_git_timestamp(), self.get_temp(), self.get_cpu(), self.get_uptime(), self.get_disk())

# Main handles network send/recv and can see all other classes directly
class Main(threading.Thread):
    def __init__(self, hostname):
        threading.Thread.__init__(self)
        self.network = Network(hostname, self.network_message_handler, self.network_status_handler)
        self.queue = Queue.Queue()
        self.arduino_connection = open("/dev/ttyACM0",'w')
        self.utils = Utils(hostname)
        self.network.thirtybirds.subscribe_to_topic("mandala_device_status")
        self.UNSET = 0
        self.FAIL = 2000
        self.PASS = 4000
        self.mandala_device_status = None
        self.mandala_tlc_ids = {
            "avl-controller":39,
            "avl-formant-1":15,
            "avl-formant-1-amplifier":4,
            "avl-formant-2":16,
            "avl-formant-2-amplifier":5,
            "avl-formant-3":17,
            "avl-formant-3-amplifier":6,
            "avl-layer-1":40,
            "avl-layer-2":11,
            "avl-layer-3":12,
            "avl-medulla":35,
            "avl-pitch-keys":18,
            "avl-pitch-keys-sensor-1":7,
            "avl-pitch-keys-sensor-2":8,
            "avl-pitch-keys-sensor-3":9,
            "avl-pitch-keys-sensor-4":10,
            "avl-settings":34,
            "avl-settings-adcs":24,
            "avl-transport":13,
            "avl-transport-encoder":0,
            "avl-voice-1":36,
            "avl-voice-1-crystal-frequency-counter":25,
            "avl-voice-1-harmonic-generators":26,
            "avl-voice-1-harmonic-volume":27,
            "avl-voice-2":37,
            "avl-voice-2-crystal-frequency-counter":28,
            "avl-voice-2-harmonic-generators":29,
            "avl-voice-2-harmonic-volume":30,
            "avl-voice-3":38,
            "avl-voice-3-crystal-frequency-counter":31,
            "avl-voice-3-harmonic-generators":32,
            "avl-voice-3-harmonic-volume":33,
            "avl-voice-keys":14,
            "avl-voice-keys-encoder-1":1,
            "avl-voice-keys-encoder-2":2,
            "avl-voice-keys-encoder-3":3
        }

    def network_message_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        topic, msg =  topic_msg # separating just to eval msg.  best to do it early.  it should be done in TB.
        #if len(msg) > 0:
        #    msg = eval(msg)
        self.add_to_queue(topic, msg)

    def network_status_handler(self, topic_msg):
        # this method runs in the thread of the caller, not the tread of Main
        print "Main.network_status_handler", topic_msg

    def add_to_queue(self, topic, msg):
        self.queue.put((topic, msg))

    def run(self):

        devicenames = self.mandala_tlc_ids.keys()
        """
        for devicename in devicenames:
            tlc_id_int = self.mandala_tlc_ids[devicename] + 5000
            tlc_id_str = "{}\n".format(tlc_id_int)
            self.ser.write(tlc_id_str)
            print self.ser.readln()
            self.ser.write("0/n")
            print self.ser.readln()
        """
        while True:
            if self.mandala_device_status == None:
                time.sleep(1)
                self.network.thirtybirds.send("mandala_device_request", True)
            try:
                topic, msg = self.queue.get(True)
                print topic, msg
                if topic == "mandala_device_status":
                    self.mandala_device_status = eval(msg)
                    devicenames = self.mandala_device_status.keys()
                    devicenames.sort()
                    for devicename in devicenames:
                        tlc_id_int = self.mandala_tlc_ids[devicename] + 5000
                        tlc_id_str = "{}\n".format(tlc_id_int)
                        if self.mandala_device_status[devicename] == "unset":
                            tlc_level_int = 0
                        if self.mandala_device_status[devicename] == "fail":
                            tlc_level_int = 100
                        if self.mandala_device_status[devicename] == "pass":
                            tlc_level_int = 4000
                        tlc_level_str = "{}\n".format(tlc_level_int)
                        print  repr(tlc_id_str),  repr(tlc_level_str),  devicename

                        time.sleep(1)
                        self.arduino_connection.write(tlc_id_str)
                        time.sleep(1)
                        self.arduino_connection.write(tlc_level_str)

                time.sleep(0.01)
            except Exception as e:
                exc_type, exc_value, exc_traceback = sys.exc_info()
                print e, repr(traceback.format_exception(exc_type, exc_value,exc_traceback))

def init(hostname):
    main = Main(hostname)
    main.daemon = True
    main.start()
    return main

<|endoftext|>