<commit_before># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import subprocess
import tempfile
import unittest
from contextlib import contextmanager
from pathlib import Path

import kfp.components as comp

def add_two_numbers(a: float, b: float) -> float:
    '''Returns sum of two arguments'''
    return a + b

@contextmanager
def components_local_output_dir_context(output_dir: str):
    old_dir = comp._components._outputs_dir
    try:
        comp._components._outputs_dir = output_dir
        yield output_dir
    finally:
        comp._components._outputs_dir = old_dir

class PythonOpTestCase(unittest.TestCase):
    def helper_test_2_in_1_out_component_using_local_call(self, func, op):
        arg1 = float(3)
        arg2 = float(5)

        expected = func(arg1, arg2)
        if isinstance(expected, tuple):
            expected = expected[0]
        expected_str = str(expected)

        with tempfile.TemporaryDirectory() as temp_dir_name:
            with components_local_output_dir_context(temp_dir_name):
                task = op(arg1, arg2)

            full_command = task.command + task.arguments

            process = subprocess.run(full_command)

            output_path = list(task.file_outputs.values())[0]
            actual_str = Path(output_path).read_text()

        self.assertEqual(float(actual_str), float(expected_str))

    def helper_test_2_in_2_out_component_using_local_call(self, func, op, output_names):
        arg1 = float(3)
        arg2 = float(5)

        expected_tuple = func(arg1, arg2)
        expected1_str = str(expected_tuple[0])
        expected2_str = str(expected_tuple[1])

        with tempfile.TemporaryDirectory() as temp_dir_name:
            with components_local_output_dir_context(temp_dir_name):
                task = op(arg1, arg2)

            full_command = task.command + task.arguments

            process = subprocess.run(full_command)

            (output_path1, output_path2) = (task.file_outputs[output_names[0]], task.file_outputs[output_names[1]])
            print(task.file_outputs)
            actual1_str = Path(output_path1).read_text()
            actual2_str = Path(output_path2).read_text()

        self.assertEqual(float(actual1_str), float(expected1_str))
        self.assertEqual(float(actual2_str), float(expected2_str))

    def test_func_to_container_op_local_call(self):
        func = add_two_numbers
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    def test_indented_func_to_container_op_local_call(self):
        def add_two_numbers_indented(a: float, b: float) -> float:
            '''Returns sum of two arguments'''
            return a + b

        func = add_two_numbers_indented
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    def test_func_to_container_op_multiple_named_typed_outputs(self):
        from typing import NamedTuple
        def add_multiply_two_numbers(a: float, b: float) -> NamedTuple('DummyName', [('sum', float), ('product', float)]):
            '''Returns sum and product of two arguments'''
            return (a + b, a * b)

        func = add_multiply_two_numbers
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_2_out_component_using_local_call(func, op, output_names=['sum', 'product'])

    @unittest.skip #TODO: #Simplified multi-output syntax is not implemented yet
    def test_func_to_container_op_multiple_named_typed_outputs_using_list_syntax(self):
        def add_multiply_two_numbers(a: float, b: float) -> [('sum', float), ('product', float)]:
            '''Returns sum and product of two arguments'''
            return (a + b, a * b)

        func = add_multiply_two_numbers
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_2_out_component_using_local_call(func, op, output_names=['sum', 'product'])

    def test_func_to_container_op_named_typed_outputs_with_underscores(self):
        from typing import NamedTuple
        def add_two_numbers_name2(a: float, b: float) -> NamedTuple('DummyName', [('output_data', float)]):
            '''Returns sum of two arguments'''
            return (a + b,)

        func = add_two_numbers_name2
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    @unittest.skip #Python does not allow NamedTuple with spaces in names: ValueError: Type names and field names must be valid identifiers: 'Output data'
    def test_func_to_container_op_named_typed_outputs_with_spaces(self):
        from typing import NamedTuple
        def add_two_numbers_name3(a: float, b: float) -> NamedTuple('DummyName', [('Output data', float)]):
            '''Returns sum of two arguments'''
            return (a + b,)

        func = add_two_numbers_name3
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    def test_handling_same_input_output_names(self):
        from typing import NamedTuple
        def add_multiply_two_numbers(a: float, b: float) -> NamedTuple('DummyName', [('a', float), ('b', float)]):
            '''Returns sum and product of two arguments'''
            return (a + b, a * b)

        func = add_multiply_two_numbers
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_2_out_component_using_local_call(func, op, output_names=['a', 'b'])

    def test_handling_same_input_default_output_names(self):
        def add_two_numbers_indented(a: float, Output: float) -> float:
            '''Returns sum of two arguments'''
            return a + Output

        func = add_two_numbers_indented
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    def test_python_component_decorator(self):
        from kfp.dsl import python_component
        import kfp.components._python_op as _python_op

        expected_name = 'Sum component name'
        expected_description = 'Sum component description'
        expected_image = 'org/image'

        @python_component(name=expected_name, description=expected_description, base_image=expected_image)
        def add_two_numbers_decorated(a: float, b: float) -> float:
            '''Returns sum of two arguments'''
            return a + b

        component_spec = _python_op._func_to_component_spec(add_two_numbers_decorated)

        self.assertEqual(component_spec.name, expected_name)
        self.assertEqual(component_spec.description.strip(), expected_description.strip())
        self.assertEqual(component_spec.implementation.container.image, expected_image)

    def test_end_to_end_python_component_pipeline_compilation(self):
        import kfp.components as comp

        #Defining the Python function
        def add(a: float, b: float) -> float:
            '''Returns sum of two arguments'''
            return a + b

        with tempfile.TemporaryDirectory() as temp_dir_name:
            add_component_file = str(Path(temp_dir_name).joinpath('add.component.yaml'))

            #Converting the function to a component. Instantiate it to create a pipeline task (ContaineOp instance)
            add_op = comp.func_to_container_op(add, base_image='python:3.5', output_component_file=add_component_file)

            #Checking that the component artifact is usable:
            add_op2 = comp.load_component_from_file(add_component_file)

            #Building the pipeline
            import kfp.dsl as dsl
            @dsl.pipeline(
                name='Calculation pipeline',
                description='A pipeline that performs arithmetic calculations.'
            )
            def calc_pipeline(
                a1=dsl.PipelineParam('a1'),
                a2=dsl.PipelineParam('a2', value='7'),
                a3=dsl.PipelineParam('a3', value='17'),
            ):
                task_1 = add_op(a1, a2)
                task_2 = add_op2(a1, a2)
                task_3 = add_op(task_1.output, task_2.output)
                task_4 = add_op2(task_3.output, a3)

            #Compiling the pipleine:
            pipeline_filename = str(Path(temp_dir_name).joinpath(calc_pipeline.__name__ + '.pipeline.tar.gz'))
            import kfp.compiler as compiler
            compiler.Compiler().compile(calc_pipeline, pipeline_filename)

if __name__ == '__main__':
    unittest.main()
<commit_msg>SDK - Removed stray debug pring from tests (#701)<commit_after># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import subprocess
import tempfile
import unittest
from contextlib import contextmanager
from pathlib import Path

import kfp.components as comp

def add_two_numbers(a: float, b: float) -> float:
    '''Returns sum of two arguments'''
    return a + b

@contextmanager
def components_local_output_dir_context(output_dir: str):
    old_dir = comp._components._outputs_dir
    try:
        comp._components._outputs_dir = output_dir
        yield output_dir
    finally:
        comp._components._outputs_dir = old_dir

class PythonOpTestCase(unittest.TestCase):
    def helper_test_2_in_1_out_component_using_local_call(self, func, op):
        arg1 = float(3)
        arg2 = float(5)

        expected = func(arg1, arg2)
        if isinstance(expected, tuple):
            expected = expected[0]
        expected_str = str(expected)

        with tempfile.TemporaryDirectory() as temp_dir_name:
            with components_local_output_dir_context(temp_dir_name):
                task = op(arg1, arg2)

            full_command = task.command + task.arguments

            process = subprocess.run(full_command)

            output_path = list(task.file_outputs.values())[0]
            actual_str = Path(output_path).read_text()

        self.assertEqual(float(actual_str), float(expected_str))

    def helper_test_2_in_2_out_component_using_local_call(self, func, op, output_names):
        arg1 = float(3)
        arg2 = float(5)

        expected_tuple = func(arg1, arg2)
        expected1_str = str(expected_tuple[0])
        expected2_str = str(expected_tuple[1])

        with tempfile.TemporaryDirectory() as temp_dir_name:
            with components_local_output_dir_context(temp_dir_name):
                task = op(arg1, arg2)

            full_command = task.command + task.arguments

            process = subprocess.run(full_command)

            (output_path1, output_path2) = (task.file_outputs[output_names[0]], task.file_outputs[output_names[1]])
            actual1_str = Path(output_path1).read_text()
            actual2_str = Path(output_path2).read_text()

        self.assertEqual(float(actual1_str), float(expected1_str))
        self.assertEqual(float(actual2_str), float(expected2_str))

    def test_func_to_container_op_local_call(self):
        func = add_two_numbers
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    def test_indented_func_to_container_op_local_call(self):
        def add_two_numbers_indented(a: float, b: float) -> float:
            '''Returns sum of two arguments'''
            return a + b

        func = add_two_numbers_indented
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    def test_func_to_container_op_multiple_named_typed_outputs(self):
        from typing import NamedTuple
        def add_multiply_two_numbers(a: float, b: float) -> NamedTuple('DummyName', [('sum', float), ('product', float)]):
            '''Returns sum and product of two arguments'''
            return (a + b, a * b)

        func = add_multiply_two_numbers
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_2_out_component_using_local_call(func, op, output_names=['sum', 'product'])

    @unittest.skip #TODO: #Simplified multi-output syntax is not implemented yet
    def test_func_to_container_op_multiple_named_typed_outputs_using_list_syntax(self):
        def add_multiply_two_numbers(a: float, b: float) -> [('sum', float), ('product', float)]:
            '''Returns sum and product of two arguments'''
            return (a + b, a * b)

        func = add_multiply_two_numbers
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_2_out_component_using_local_call(func, op, output_names=['sum', 'product'])

    def test_func_to_container_op_named_typed_outputs_with_underscores(self):
        from typing import NamedTuple
        def add_two_numbers_name2(a: float, b: float) -> NamedTuple('DummyName', [('output_data', float)]):
            '''Returns sum of two arguments'''
            return (a + b,)

        func = add_two_numbers_name2
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    @unittest.skip #Python does not allow NamedTuple with spaces in names: ValueError: Type names and field names must be valid identifiers: 'Output data'
    def test_func_to_container_op_named_typed_outputs_with_spaces(self):
        from typing import NamedTuple
        def add_two_numbers_name3(a: float, b: float) -> NamedTuple('DummyName', [('Output data', float)]):
            '''Returns sum of two arguments'''
            return (a + b,)

        func = add_two_numbers_name3
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    def test_handling_same_input_output_names(self):
        from typing import NamedTuple
        def add_multiply_two_numbers(a: float, b: float) -> NamedTuple('DummyName', [('a', float), ('b', float)]):
            '''Returns sum and product of two arguments'''
            return (a + b, a * b)

        func = add_multiply_two_numbers
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_2_out_component_using_local_call(func, op, output_names=['a', 'b'])

    def test_handling_same_input_default_output_names(self):
        def add_two_numbers_indented(a: float, Output: float) -> float:
            '''Returns sum of two arguments'''
            return a + Output

        func = add_two_numbers_indented
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    def test_python_component_decorator(self):
        from kfp.dsl import python_component
        import kfp.components._python_op as _python_op

        expected_name = 'Sum component name'
        expected_description = 'Sum component description'
        expected_image = 'org/image'

        @python_component(name=expected_name, description=expected_description, base_image=expected_image)
        def add_two_numbers_decorated(a: float, b: float) -> float:
            '''Returns sum of two arguments'''
            return a + b

        component_spec = _python_op._func_to_component_spec(add_two_numbers_decorated)

        self.assertEqual(component_spec.name, expected_name)
        self.assertEqual(component_spec.description.strip(), expected_description.strip())
        self.assertEqual(component_spec.implementation.container.image, expected_image)

    def test_end_to_end_python_component_pipeline_compilation(self):
        import kfp.components as comp

        #Defining the Python function
        def add(a: float, b: float) -> float:
            '''Returns sum of two arguments'''
            return a + b

        with tempfile.TemporaryDirectory() as temp_dir_name:
            add_component_file = str(Path(temp_dir_name).joinpath('add.component.yaml'))

            #Converting the function to a component. Instantiate it to create a pipeline task (ContaineOp instance)
            add_op = comp.func_to_container_op(add, base_image='python:3.5', output_component_file=add_component_file)

            #Checking that the component artifact is usable:
            add_op2 = comp.load_component_from_file(add_component_file)

            #Building the pipeline
            import kfp.dsl as dsl
            @dsl.pipeline(
                name='Calculation pipeline',
                description='A pipeline that performs arithmetic calculations.'
            )
            def calc_pipeline(
                a1=dsl.PipelineParam('a1'),
                a2=dsl.PipelineParam('a2', value='7'),
                a3=dsl.PipelineParam('a3', value='17'),
            ):
                task_1 = add_op(a1, a2)
                task_2 = add_op2(a1, a2)
                task_3 = add_op(task_1.output, task_2.output)
                task_4 = add_op2(task_3.output, a3)

            #Compiling the pipleine:
            pipeline_filename = str(Path(temp_dir_name).joinpath(calc_pipeline.__name__ + '.pipeline.tar.gz'))
            import kfp.compiler as compiler
            compiler.Compiler().compile(calc_pipeline, pipeline_filename)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path

import kfp.components as comp
from kfp.components._yaml_utils import load_yaml

_this_file = Path(__file__).resolve()
_this_dir = _this_file.parent
_test_data_dir = _this_dir.joinpath('test_data')
_tests_root_dir = _this_dir.parent.parent
_sdk_root_dir = _tests_root_dir.parent

sys.path.insert(0, _sdk_root_dir)

def add_two_numbers(a: float, b: float) -> float:
    '''Returns sum of two arguments'''
    return a + b

@comp.python_op
def add_two_numbers_decorated(a: float, b: float) -> float:
    '''Returns sum of two arguments'''
    return a + b

@comp.python_op(base_image='ark7/python-fire')
def add_two_numbers_decorated_with_parameters(a: float, b: float) -> float:
    '''Returns sum of two arguments'''
    return a + b

class PythonOpTestCase(unittest.TestCase):
    def helper_test_2_in_1_out_component_using_local_call(self, func, op):
        arg1 = float(3)
        arg2 = float(5)

        expected = func(arg1, arg2)
        if isinstance(expected, tuple):
            expected = expected[0]
        expected_str = str(expected)

        output_file_obj = tempfile.NamedTemporaryFile()
        output_path = output_file_obj.name

        task = op(arg1, arg2, output_path)

        full_command = task.command + task.arguments

        process = subprocess.run(full_command)

        actual_str = Path(output_path).read_text()

        self.assertEqual(float(actual_str), float(expected_str))

    def helper_test_2_in_2_out_component_using_local_call(self, func, op):
        arg1 = float(3)
        arg2 = float(5)

        expected_tuple = func(arg1, arg2)
        expected1_str = str(expected_tuple[0])
        expected2_str = str(expected_tuple[1])

        output_path1 = tempfile.NamedTemporaryFile().name
        output_path2 = tempfile.NamedTemporaryFile().name

        task = op(arg1, arg2, output_path1, output_path2)

        full_command = task.command + task.arguments

        process = subprocess.run(full_command)

        actual1_str = Path(output_path1).read_text()
        actual2_str = Path(output_path2).read_text()

        self.assertEqual(float(actual1_str), float(expected1_str))
        self.assertEqual(float(actual2_str), float(expected2_str))

    def test_func_to_container_op_local_call(self):
        func = add_two_numbers
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    def test_decorated_func_to_container_op_local_call(self):
        self.helper_test_2_in_1_out_component_using_local_call(add_two_numbers, add_two_numbers_decorated)

    def test_decorated_with_parameters_func_to_container_op_local_call(self):
        self.helper_test_2_in_1_out_component_using_local_call(add_two_numbers, add_two_numbers_decorated_with_parameters)

    def test_indented_func_to_container_op_local_call(self):
        def add_two_numbers_indented(a: float, b: float) -> float:
            '''Returns sum of two arguments'''
            return a + b

        func = add_two_numbers_indented
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    def test_indented_decorated_func_to_container_op_local_call(self):
        @comp.python_op
        def add_two_numbers_indented_decorated(a: float, b: float) -> float:
            '''Returns sum of two arguments'''
            return a + b

        self.helper_test_2_in_1_out_component_using_local_call(add_two_numbers, add_two_numbers_indented_decorated)

    def test_indented_decorated_with_parameters_func_to_container_op_local_call(self):
        @comp.python_op(base_image='ark7/python-fire')
        def add_two_numbers_indented_decorated_with_parameters(a: float, b: float) -> float:
            '''Returns sum of two arguments'''
            return a + b

        self.helper_test_2_in_1_out_component_using_local_call(add_two_numbers, add_two_numbers_indented_decorated_with_parameters)

    def test_func_to_container_op_multiple_named_typed_outputs(self):
        from typing import NamedTuple
        def add_multiply_two_numbers(a: float, b: float) -> NamedTuple('DummyName', [('sum', float), ('product', float)]):
            '''Returns sum and product of two arguments'''
            return (a + b, a * b)

        func = add_multiply_two_numbers
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_2_out_component_using_local_call(func, op)

    @unittest.skip #TODO: #Simplified multi-output syntax is not implemented yet
    def test_func_to_container_op_multiple_named_typed_outputs_using_list_syntax(self):
        def add_multiply_two_numbers(a: float, b: float) -> [('sum', float), ('product', float)]:
            '''Returns sum and product of two arguments'''
            return (a + b, a * b)

        func = add_multiply_two_numbers
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_2_out_component_using_local_call(func, op)

    def test_func_to_container_op_named_typed_outputs_with_underscores(self):
        from typing import NamedTuple
        def add_two_numbers_name2(a: float, b: float) -> NamedTuple('DummyName', [('output_data', float)]):
            '''Returns sum of two arguments'''
            return (a + b,)

        func = add_two_numbers_name2
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    @unittest.skip #Python does not allow NamedTuple with spaces in names: ValueError: Type names and field names must be valid identifiers: 'Output data'
    def test_func_to_container_op_named_typed_outputs_with_spaces(self):
        from typing import NamedTuple
        def add_two_numbers_name3(a: float, b: float) -> NamedTuple('DummyName', [('Output data', float)]):
            '''Returns sum of two arguments'''
            return (a + b,)

        func = add_two_numbers_name3
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    def test_handling_same_input_output_names(self):
        from typing import NamedTuple
        def add_multiply_two_numbers(a: float, b: float) -> NamedTuple('DummyName', [('a', float), ('b', float)]):
            '''Returns sum and product of two arguments'''
            return (a + b, a * b)

        func = add_multiply_two_numbers
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_2_out_component_using_local_call(func, op)

    def test_handling_same_input_default_output_names(self):
        def add_two_numbers_indented(a: float, Output: float) -> float:
            '''Returns sum of two arguments'''
            return a + Output

        func = add_two_numbers_indented
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    def test_end_to_end_python_component_pipeline_compilation(self):
        import kfp.components as comp
        from kfp.components import python_op

        #Defining the Python function
        def add(a: float, b: float) -> float:
            '''Returns sum of two arguments'''
            return a + b

        add_component_file = tempfile.NamedTemporaryFile(prefix='add', suffix='.component.yaml').name
        subtract_component_file = tempfile.NamedTemporaryFile(prefix='subtract', suffix='.component.yaml').name

        #Converting the function to a component. Instantiate it to create a pipeline task (ContaineOp instance)
        add_op = comp.func_to_container_op(add, base_image='ark7/python-fire', output_component_file=add_component_file)

        #Checking that the component artifact is usable:
        add_op2 = comp.load_component_from_file(add_component_file)

        #Using decorator to perform the same thing (convert function to a component):
        @python_op(base_image='ark7/python-fire', output_component_file=subtract_component_file)
        def subtract_op(a: float, b: float) -> float:
            '''Returns difference between two arguments'''
            return a - b

        #Checking that the component artifact is usable:
        subtract_op2 = comp.load_component_from_file(subtract_component_file)

        #Building the pipeline
        import kfp.dsl as dsl
        @dsl.pipeline(
            name='Calculation pipeline',
            description='A pipeline that performs arithmetic calculations.'
        )
        def calc_pipeline(
            a1=dsl.PipelineParam('a1'),
            a2=dsl.PipelineParam('a2', value='7'),
            a3=dsl.PipelineParam('a3', value='17'),
        ):
            task_11 = add_op(a1, a2)
            task_12 = subtract_op(task_11.output, a3)
            task_21 = add_op2(a1, a2)
            task_22 = subtract_op2(task_21.output, a3)
            task_3 = subtract_op(task_12.output, task_22.output)

        #Compiling the pipleine:
        pipeline_filename=calc_pipeline.__name__ + '.pipeline.tar.gz'
        import kfp.compiler as compiler
        compiler.Compiler().compile(calc_pipeline, calc_pipeline.__name__ + '.tar.gz')

if __name__ == '__main__':
    unittest.main()
<commit_msg>SDK/Tests/Components - Improve temporary file handling (#37)<commit_after># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path

import kfp.components as comp
from kfp.components._yaml_utils import load_yaml

_this_file = Path(__file__).resolve()
_this_dir = _this_file.parent
_test_data_dir = _this_dir.joinpath('test_data')
_tests_root_dir = _this_dir.parent.parent
_sdk_root_dir = _tests_root_dir.parent

sys.path.insert(0, _sdk_root_dir)

def add_two_numbers(a: float, b: float) -> float:
    '''Returns sum of two arguments'''
    return a + b

@comp.python_op
def add_two_numbers_decorated(a: float, b: float) -> float:
    '''Returns sum of two arguments'''
    return a + b

@comp.python_op(base_image='ark7/python-fire')
def add_two_numbers_decorated_with_parameters(a: float, b: float) -> float:
    '''Returns sum of two arguments'''
    return a + b

class PythonOpTestCase(unittest.TestCase):
    def helper_test_2_in_1_out_component_using_local_call(self, func, op):
        arg1 = float(3)
        arg2 = float(5)

        expected = func(arg1, arg2)
        if isinstance(expected, tuple):
            expected = expected[0]
        expected_str = str(expected)

        with tempfile.TemporaryDirectory() as temp_dir_name:
            output_path = Path(temp_dir_name).joinpath('output1')

            task = op(arg1, arg2, str(output_path))

            full_command = task.command + task.arguments

            process = subprocess.run(full_command)

            actual_str = output_path.read_text()

        self.assertEqual(float(actual_str), float(expected_str))

    def helper_test_2_in_2_out_component_using_local_call(self, func, op):
        arg1 = float(3)
        arg2 = float(5)

        expected_tuple = func(arg1, arg2)
        expected1_str = str(expected_tuple[0])
        expected2_str = str(expected_tuple[1])

        with tempfile.TemporaryDirectory() as temp_dir_name:
            output_path1 = Path(temp_dir_name).joinpath('output1')
            output_path2 = Path(temp_dir_name).joinpath('output2')

            task = op(arg1, arg2, str(output_path1), str(output_path2))

            full_command = task.command + task.arguments

            process = subprocess.run(full_command)

            actual1_str = output_path1.read_text()
            actual2_str = output_path2.read_text()

        self.assertEqual(float(actual1_str), float(expected1_str))
        self.assertEqual(float(actual2_str), float(expected2_str))

    def test_func_to_container_op_local_call(self):
        func = add_two_numbers
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    def test_decorated_func_to_container_op_local_call(self):
        self.helper_test_2_in_1_out_component_using_local_call(add_two_numbers, add_two_numbers_decorated)

    def test_decorated_with_parameters_func_to_container_op_local_call(self):
        self.helper_test_2_in_1_out_component_using_local_call(add_two_numbers, add_two_numbers_decorated_with_parameters)

    def test_indented_func_to_container_op_local_call(self):
        def add_two_numbers_indented(a: float, b: float) -> float:
            '''Returns sum of two arguments'''
            return a + b

        func = add_two_numbers_indented
        op = comp.func_to_container_op(func,output_component_file='add_two_numbers_indented.component.yaml')

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    def test_indented_decorated_func_to_container_op_local_call(self):
        @comp.python_op
        def add_two_numbers_indented_decorated(a: float, b: float) -> float:
            '''Returns sum of two arguments'''
            return a + b

        self.helper_test_2_in_1_out_component_using_local_call(add_two_numbers, add_two_numbers_indented_decorated)

    def test_indented_decorated_with_parameters_func_to_container_op_local_call(self):
        @comp.python_op(base_image='ark7/python-fire')
        def add_two_numbers_indented_decorated_with_parameters(a: float, b: float) -> float:
            '''Returns sum of two arguments'''
            return a + b

        self.helper_test_2_in_1_out_component_using_local_call(add_two_numbers, add_two_numbers_indented_decorated_with_parameters)

    def test_func_to_container_op_multiple_named_typed_outputs(self):
        from typing import NamedTuple
        def add_multiply_two_numbers(a: float, b: float) -> NamedTuple('DummyName', [('sum', float), ('product', float)]):
            '''Returns sum and product of two arguments'''
            return (a + b, a * b)

        func = add_multiply_two_numbers
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_2_out_component_using_local_call(func, op)

    @unittest.skip #TODO: #Simplified multi-output syntax is not implemented yet
    def test_func_to_container_op_multiple_named_typed_outputs_using_list_syntax(self):
        def add_multiply_two_numbers(a: float, b: float) -> [('sum', float), ('product', float)]:
            '''Returns sum and product of two arguments'''
            return (a + b, a * b)

        func = add_multiply_two_numbers
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_2_out_component_using_local_call(func, op)

    def test_func_to_container_op_named_typed_outputs_with_underscores(self):
        from typing import NamedTuple
        def add_two_numbers_name2(a: float, b: float) -> NamedTuple('DummyName', [('output_data', float)]):
            '''Returns sum of two arguments'''
            return (a + b,)

        func = add_two_numbers_name2
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    @unittest.skip #Python does not allow NamedTuple with spaces in names: ValueError: Type names and field names must be valid identifiers: 'Output data'
    def test_func_to_container_op_named_typed_outputs_with_spaces(self):
        from typing import NamedTuple
        def add_two_numbers_name3(a: float, b: float) -> NamedTuple('DummyName', [('Output data', float)]):
            '''Returns sum of two arguments'''
            return (a + b,)

        func = add_two_numbers_name3
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    def test_handling_same_input_output_names(self):
        from typing import NamedTuple
        def add_multiply_two_numbers(a: float, b: float) -> NamedTuple('DummyName', [('a', float), ('b', float)]):
            '''Returns sum and product of two arguments'''
            return (a + b, a * b)

        func = add_multiply_two_numbers
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_2_out_component_using_local_call(func, op)

    def test_handling_same_input_default_output_names(self):
        def add_two_numbers_indented(a: float, Output: float) -> float:
            '''Returns sum of two arguments'''
            return a + Output

        func = add_two_numbers_indented
        op = comp.func_to_container_op(func)

        self.helper_test_2_in_1_out_component_using_local_call(func, op)

    def test_end_to_end_python_component_pipeline_compilation(self):
        import kfp.components as comp
        from kfp.components import python_op

        #Defining the Python function
        def add(a: float, b: float) -> float:
            '''Returns sum of two arguments'''
            return a + b

        with tempfile.TemporaryDirectory() as temp_dir_name:
            add_component_file = str(Path(temp_dir_name).joinpath('add.component.yaml'))
            subtract_component_file = str(Path(temp_dir_name).joinpath('subtract.component.yaml'))

            #Converting the function to a component. Instantiate it to create a pipeline task (ContaineOp instance)
            add_op = comp.func_to_container_op(add, base_image='ark7/python-fire', output_component_file=add_component_file)

            #Checking that the component artifact is usable:
            add_op2 = comp.load_component_from_file(add_component_file)

            #Using decorator to perform the same thing (convert function to a component):
            @python_op(base_image='ark7/python-fire', output_component_file=subtract_component_file)
            def subtract_op(a: float, b: float) -> float:
                '''Returns difference between two arguments'''
                return a - b

            #Checking that the component artifact is usable:
            subtract_op2 = comp.load_component_from_file(subtract_component_file)

            #Building the pipeline
            import kfp.dsl as dsl
            @dsl.pipeline(
                name='Calculation pipeline',
                description='A pipeline that performs arithmetic calculations.'
            )
            def calc_pipeline(
                a1=dsl.PipelineParam('a1'),
                a2=dsl.PipelineParam('a2', value='7'),
                a3=dsl.PipelineParam('a3', value='17'),
            ):
                task_11 = add_op(a1, a2)
                task_12 = subtract_op(task_11.output, a3)
                task_21 = add_op2(a1, a2)
                task_22 = subtract_op2(task_21.output, a3)
                task_3 = subtract_op(task_12.output, task_22.output)

            #Compiling the pipleine:
            pipeline_filename = Path(temp_dir_name).joinpath(calc_pipeline.__name__ + '.pipeline.tar.gz')
            import kfp.compiler as compiler
            compiler.Compiler().compile(calc_pipeline, pipeline_filename)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from abc import abstractmethod, ABCMeta

import numpy as np
from lolopy.loloserver import get_java_gateway
from lolopy.utils import send_feature_array, send_1D_array
from sklearn.base import BaseEstimator, RegressorMixin, ClassifierMixin, is_regressor
from sklearn.exceptions import NotFittedError

__all__ = ['RandomForestRegressor', 'RandomForestClassifier']

class BaseLoloLearner(BaseEstimator, metaclass=ABCMeta):
    """Base object for all leaners that use Lolo.

    Contains logic for starting the JVM gateway, and the fit operations.
    It is only necessary to implement the `_make_learner` object and create an `__init__` function
    to adapt a learner from the Lolo library for use in lolopy.

    The logic for making predictions (i.e., `predict` and `predict_proba`) is specific to whether the learner
    is a classification or regression model.
    In lolo, learners are not specific to a regression or classification problem and the type of problem is determined
    when fitting data is provided to the algorithm.
    In contrast, Scikit-learn learners for regression or classification problems are different classes.
    We have implemented `BaseLoloRegressor` and `BaseLoloClassifier` abstract classes to make it easier for creating
    a classification or regression version of a Lolo base class.
    The pattern for creating a scikit-learn compatible learner is to first implement the `_make_learner` and `__init__`
    operations in a special "Mixin" class that inherits from `BaseLoloLearner`, and then create a regression- or
    classification-specific class that inherits from both `BaseClassifier` or `BaseRegressor` and your new "Mixin".
    See the RandomForest models as an example for this approach.
    """

    def __init__(self):
        self.gateway = get_java_gateway()

        # Create a placeholder for the model
        self.model_ = None
        self._compress_level = 9
        self.feature_importances_ = None

    def __getstate__(self):
        # Get the current state
        try:
            state = super(BaseLoloLearner, self).__getstate__()
        except AttributeError:
            state = self.__dict__.copy()

        # Delete the gateway data
        del state['gateway']

        # If there is a model set, replace it with the JVM copy
        if self.model_ is not None:
            state['model_'] = self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.serializeObject(self.model_,
                                                                                self._compress_level)
        return state

    def __setstate__(self, state):
        # Unpickle the object
        super(BaseLoloLearner, self).__setstate__(state)

        # Get a pointer to the gateway
        self.gateway = get_java_gateway()

        # If needed, load the model into memory
        if state['model_'] is not None:
            bytes = state.pop('model_')
            self.model_ = self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.deserializeObject(bytes)

    def fit(self, X, y, weights=None):
        # Instantiate the JVM object
        learner = self._make_learner()

        # Convert all of the training data to Java arrays
        train_data, weights_java = self._convert_train_data(X, y, weights)
        assert train_data.length() == len(X), "Array copy failed"
        assert train_data.head()._1().length() == len(X[0]), "Wrong number of features"
        assert weights_java.length() == len(X), "Weights copy failed"

        # Train the model
        result = learner.train(train_data, self.gateway.jvm.scala.Some(weights_java))

        # Unlink the training data, which is no longer needed (to save memory)
        self.gateway.detach(train_data)
        self.gateway.detach(weights_java)

        # Get the model out
        self.model_ = result.getModel()

        # Store the feature importances
        feature_importances_java = result.getFeatureImportance().get()
        feature_importances_bytes = self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.send1DArray(feature_importances_java)
        self.feature_importances_ = np.frombuffer(feature_importances_bytes, 'float')

        return self

    @abstractmethod
    def _make_learner(self):
        """Instantiate the learner used by Lolo to train a model

        Returns:
            (JavaObject) A lolo "Learner" object, which can be used to train a model"""
        pass

    def clear_model(self):
        """Utility operation for deleting model from JVM when no longer needed"""

        if self.model_ is not None:
            self.gateway.detach(self.model_)
            self.model_ = None

    def _convert_train_data(self, X, y, weights=None):
        """Convert the training data to a form accepted by Lolo

        Args:
            X (ndarray): Input variables
            y (ndarray): Output variables
            weights (ndarray): Wegihts for each sample
        Returns
            train_data (JavaObject): Pointer to the training data in Java
        """

        # Make some default weights
        if weights is None:
            weights = np.ones(len(y))

        # Convert x, y, and w to float64 and int8 with native ordering

        y = np.array(y, dtype=np.float64 if is_regressor(self) else np.int32)
        weights = np.array(weights, dtype=np.float64)

        # Convert X and y to Java Objects
        X_java = send_feature_array(self.gateway, X)
        y_java = send_1D_array(self.gateway, y, is_regressor(self))
        assert y_java.length() == len(y) == len(X)
        w_java = send_1D_array(self.gateway, weights, True)
        assert w_java.length() == len(weights)

        return self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.zipTrainingData(X_java, y_java), w_java

    def _convert_run_data(self, X):
        """Convert the data to be run by the model

        Args:
            X (ndarray): Input data
        Returns:
            (JavaObject): Pointer to run data in Java
        """
        if not isinstance(X, np.ndarray):
            X = np.array(X)
        return self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.getFeatureArray(X.tobytes(), X.shape[1], False)

    def get_importance_scores(self, X):
        """Get the importance scores for each entry in the training set for each prediction

        Args:
            X (ndarray): Inputs for each entry to be assessed
        """

        pred_result = self._get_prediction_result(X)

        y_import_bytes = self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.getImportanceScores(pred_result)
        y_import = np.frombuffer(y_import_bytes, 'float').reshape(len(X), -1)
        return y_import

    def _get_prediction_result(self, X):
        """Get the PredictionResult from the lolo JVM

        The PredictionResult class holds methods that will generate the expected predictions, uncertainty intervals, etc

        Args:
            X (ndarray): Input features for each entry
        Returns:
            (JavaObject): Prediction result produced by evaluating the model
        """

        # Check that the model is fitted
        if self.model_ is None:
            raise NotFittedError()

        # Convert the data to Java
        X_java = self._convert_run_data(X)

        # Get the PredictionResult
        pred_result = self.model_.transform(X_java)

        # Unlink the run data, which is no longer needed (to save memory)
        self.gateway.detach(X_java)
        return pred_result

class BaseLoloRegressor(BaseLoloLearner, RegressorMixin):
    """Abstract class for models that produce regression models.

    Implements the predict operation"""

    def predict(self, X, return_std=False):
        # Start the prediction process
        pred_result = self._get_prediction_result(X)

        # Pull out the expected values
        y_pred_byte = self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.getRegressionExpected(pred_result)
        y_pred = np.frombuffer(y_pred_byte, dtype='float')  # Lolo gives a byte array back

        # If desired, return the uncertainty too
        if return_std:
            # TODO: This part fails on Windows because the NativeSystemBLAS is not found. Fix that
            # TODO: This is only valid for regression models. Perhaps make a "LoloRegressor" class
            y_std_bytes = self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.getRegressionUncertainty(pred_result)
            y_std = np.frombuffer(y_std_bytes, 'float')
            return y_pred, y_std

        # Get the expected values
        return y_pred

class BaseLoloClassifier(BaseLoloLearner, ClassifierMixin):
    """Base class for classification models

    Implements a modification to the fit operation that stores the number of classes
    and the predict/predict_proba methods"""

    def fit(self, X, y, weights=None):
        # Get the number of classes
        self.n_classes_ = len(set(y))

        return super(BaseLoloClassifier, self).fit(X, y, weights)

    def predict(self, X):
        pred_result = self._get_prediction_result(X)

        # Pull out the expected values
        y_pred_byte = self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.getClassifierExpected(pred_result)
        y_pred = np.frombuffer(y_pred_byte, dtype=np.int32)  # Lolo gives a byte array back

        return y_pred

    def predict_proba(self, X):
        pred_result = self._get_prediction_result(X)

        # Copy over the class probabilities
        probs_byte = self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.getClassifierProbabilities(pred_result,
                                                                                self.n_classes_)
        probs = np.frombuffer(probs_byte, dtype='float').reshape(-1, self.n_classes_)
        return probs

class RandomForestMixin(BaseLoloLearner):
    """Random Forest base class

    Implements the _make_learner operation and the __init__ function with options specific to the RandomForest
    class in Lolo"""

    def __init__(self, num_trees=-1, use_jackknife=True, bias_learner=None,
                 leaf_learner=None, subset_strategy="auto", min_leaf_instances=1,
                 max_depth=2**30, uncertainty_calibration=False, randomize_pivot_location=False):
        """Initialize the RandomForest

        Args:
            num_trees (int): Number of trees to use in the forest
            use_jackknife (bool): Whether to use jackknife based variance estimates
            bias_learner (BaseLoloLearner): Algorithm used to model bias (default: no model)
            leaf_learner (BaseLoloLearner): Learner used at each leaf of the random forest (default: GuessTheMean)
            subset_strategy (Union[string,int,float]): Strategy used to determine number of features used at each split
                Available options:
                    "auto": Use the default for lolo (all features for regression, sqrt for classification)
                    "log2": Use the base 2 log of the number of features
                    "sqrt": Use the square root of the number of features
                    integer: Set the number of features explicitly
                    float: Use a certain fraction of the features
            min_leaf_instances (int): Minimum number of features used at each leaf
            max_depth (int): Maximum depth to which to allow the decision trees to grow
            uncertainty_calibration (bool): whether to re-calibrate the predicted uncertainty based on out-of-bag residuals
            randomize_pivot_location (bool): whether to draw pivots randomly or always select the midpoint
        """
        super(RandomForestMixin, self).__init__()

        # Store the variables
        self.num_trees = num_trees
        self.use_jackknife = use_jackknife
        self.subset_strategy = subset_strategy
        self.bias_learner = bias_learner
        self.leaf_learner = leaf_learner
        self.min_leaf_instances = min_leaf_instances
        self.max_depth = max_depth
        self.uncertainty_calibration = uncertainty_calibration
        self.randomize_pivot_location = randomize_pivot_location

    def _make_learner(self):
        #  TODO: Figure our a more succinct way of dealing with optional arguments/Option values
        #  TODO: that ^^, please
        learner = self.gateway.jvm.io.citrine.lolo.learners.RandomForest(
            self.num_trees, self.use_jackknife,
            getattr(self.gateway.jvm.io.citrine.lolo.learners.RandomForest,
                    "$lessinit$greater$default$3")() if self.bias_learner is None
            else self.gateway.jvm.scala.Some(self.bias_learner._make_learner()),
            getattr(self.gateway.jvm.io.citrine.lolo.learners.RandomForest,
                    "$lessinit$greater$default$4")() if self.leaf_learner is None
            else self.gateway.jvm.scala.Some(self.leaf_learner._make_learner()),
            self.subset_strategy,
            self.min_leaf_instances,
            self.max_depth,
            self.uncertainty_calibration,
            self.randomize_pivot_location
        )
        return learner

class RandomForestRegressor(BaseLoloRegressor, RandomForestMixin):
    """Random Forest model used for regression"""

class RandomForestClassifier(BaseLoloClassifier, RandomForestMixin):
    """Random Forest model used for classification"""

class RegressionTreeLearner(BaseLoloRegressor):
    """Regression tree learner, based on the RandomTree algorithm."""

    def __init__(self, num_features=-1, max_depth=30, min_leaf_instances=1, leaf_learner=None,
                 randomize_pivot_location=False):
        """Initialize the learner

        Args:
            num_features (int): Number of features to consider at each split (-1 to consider all features)
            max_depth (int): Maximum depth of the regression tree
            min_leaf_instances (int): Minimum number instances per leaf
            leaf_learner (BaseLoloLearner): Learner to use on the leaves
            randomize_pivot_location (bool): whether to draw pivots randomly or always select the midpoint
        """
        super(RegressionTreeLearner, self).__init__()
        self.num_features = num_features
        self.max_depth = max_depth
        self.min_leaf_instances = min_leaf_instances
        self.leaf_learner = leaf_learner
        self.randomize_pivot_location = randomize_pivot_location

    def _make_learner(self):
        if self.leaf_learner is None:
            leaf_learner = getattr(
                self.gateway.jvm.io.citrine.lolo.trees.regression.RegressionTreeLearner,
                "$lessinit$greater$default$4"
            )()
        else:
            leaf_learner = self.gateway.jvm.scala.Some(self.leaf_learner._make_learner())
        return self.gateway.jvm.io.citrine.lolo.trees.regression.RegressionTreeLearner(
            self.num_features, self.max_depth, self.min_leaf_instances,
            leaf_learner,
            self.randomize_pivot_location
        )

class LinearRegression(BaseLoloRegressor):
    """Linear ridge regression with an :math:`L_2` penalty"""

    def __init__(self, reg_param=None, fit_intercept=True):
        """Initialize the regressor"""

        super(LinearRegression, self).__init__()

        self.reg_param = reg_param
        self.fit_intercept = fit_intercept

    def _make_learner(self):
        return self.gateway.jvm.io.citrine.lolo.linear.LinearRegressionLearner(
            getattr(self.gateway.jvm.io.citrine.lolo.linear.LinearRegressionLearner,
                    "$lessinit$greater$default$1")() if self.reg_param is None
            else self.gateway.jvm.scala.Some(float(self.reg_param)),
            self.fit_intercept
        )
<commit_msg>Remove randomPivotLocation from lolopy's RegressionTreeLearner<commit_after>from abc import abstractmethod, ABCMeta

import numpy as np
from lolopy.loloserver import get_java_gateway
from lolopy.utils import send_feature_array, send_1D_array
from sklearn.base import BaseEstimator, RegressorMixin, ClassifierMixin, is_regressor
from sklearn.exceptions import NotFittedError

__all__ = ['RandomForestRegressor', 'RandomForestClassifier']

class BaseLoloLearner(BaseEstimator, metaclass=ABCMeta):
    """Base object for all leaners that use Lolo.

    Contains logic for starting the JVM gateway, and the fit operations.
    It is only necessary to implement the `_make_learner` object and create an `__init__` function
    to adapt a learner from the Lolo library for use in lolopy.

    The logic for making predictions (i.e., `predict` and `predict_proba`) is specific to whether the learner
    is a classification or regression model.
    In lolo, learners are not specific to a regression or classification problem and the type of problem is determined
    when fitting data is provided to the algorithm.
    In contrast, Scikit-learn learners for regression or classification problems are different classes.
    We have implemented `BaseLoloRegressor` and `BaseLoloClassifier` abstract classes to make it easier for creating
    a classification or regression version of a Lolo base class.
    The pattern for creating a scikit-learn compatible learner is to first implement the `_make_learner` and `__init__`
    operations in a special "Mixin" class that inherits from `BaseLoloLearner`, and then create a regression- or
    classification-specific class that inherits from both `BaseClassifier` or `BaseRegressor` and your new "Mixin".
    See the RandomForest models as an example for this approach.
    """

    def __init__(self):
        self.gateway = get_java_gateway()

        # Create a placeholder for the model
        self.model_ = None
        self._compress_level = 9
        self.feature_importances_ = None

    def __getstate__(self):
        # Get the current state
        try:
            state = super(BaseLoloLearner, self).__getstate__()
        except AttributeError:
            state = self.__dict__.copy()

        # Delete the gateway data
        del state['gateway']

        # If there is a model set, replace it with the JVM copy
        if self.model_ is not None:
            state['model_'] = self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.serializeObject(self.model_,
                                                                                self._compress_level)
        return state

    def __setstate__(self, state):
        # Unpickle the object
        super(BaseLoloLearner, self).__setstate__(state)

        # Get a pointer to the gateway
        self.gateway = get_java_gateway()

        # If needed, load the model into memory
        if state['model_'] is not None:
            bytes = state.pop('model_')
            self.model_ = self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.deserializeObject(bytes)

    def fit(self, X, y, weights=None):
        # Instantiate the JVM object
        learner = self._make_learner()

        # Convert all of the training data to Java arrays
        train_data, weights_java = self._convert_train_data(X, y, weights)
        assert train_data.length() == len(X), "Array copy failed"
        assert train_data.head()._1().length() == len(X[0]), "Wrong number of features"
        assert weights_java.length() == len(X), "Weights copy failed"

        # Train the model
        result = learner.train(train_data, self.gateway.jvm.scala.Some(weights_java))

        # Unlink the training data, which is no longer needed (to save memory)
        self.gateway.detach(train_data)
        self.gateway.detach(weights_java)

        # Get the model out
        self.model_ = result.getModel()

        # Store the feature importances
        feature_importances_java = result.getFeatureImportance().get()
        feature_importances_bytes = self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.send1DArray(feature_importances_java)
        self.feature_importances_ = np.frombuffer(feature_importances_bytes, 'float')

        return self

    @abstractmethod
    def _make_learner(self):
        """Instantiate the learner used by Lolo to train a model

        Returns:
            (JavaObject) A lolo "Learner" object, which can be used to train a model"""
        pass

    def clear_model(self):
        """Utility operation for deleting model from JVM when no longer needed"""

        if self.model_ is not None:
            self.gateway.detach(self.model_)
            self.model_ = None

    def _convert_train_data(self, X, y, weights=None):
        """Convert the training data to a form accepted by Lolo

        Args:
            X (ndarray): Input variables
            y (ndarray): Output variables
            weights (ndarray): Wegihts for each sample
        Returns
            train_data (JavaObject): Pointer to the training data in Java
        """

        # Make some default weights
        if weights is None:
            weights = np.ones(len(y))

        # Convert x, y, and w to float64 and int8 with native ordering

        y = np.array(y, dtype=np.float64 if is_regressor(self) else np.int32)
        weights = np.array(weights, dtype=np.float64)

        # Convert X and y to Java Objects
        X_java = send_feature_array(self.gateway, X)
        y_java = send_1D_array(self.gateway, y, is_regressor(self))
        assert y_java.length() == len(y) == len(X)
        w_java = send_1D_array(self.gateway, weights, True)
        assert w_java.length() == len(weights)

        return self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.zipTrainingData(X_java, y_java), w_java

    def _convert_run_data(self, X):
        """Convert the data to be run by the model

        Args:
            X (ndarray): Input data
        Returns:
            (JavaObject): Pointer to run data in Java
        """
        if not isinstance(X, np.ndarray):
            X = np.array(X)
        return self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.getFeatureArray(X.tobytes(), X.shape[1], False)

    def get_importance_scores(self, X):
        """Get the importance scores for each entry in the training set for each prediction

        Args:
            X (ndarray): Inputs for each entry to be assessed
        """

        pred_result = self._get_prediction_result(X)

        y_import_bytes = self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.getImportanceScores(pred_result)
        y_import = np.frombuffer(y_import_bytes, 'float').reshape(len(X), -1)
        return y_import

    def _get_prediction_result(self, X):
        """Get the PredictionResult from the lolo JVM

        The PredictionResult class holds methods that will generate the expected predictions, uncertainty intervals, etc

        Args:
            X (ndarray): Input features for each entry
        Returns:
            (JavaObject): Prediction result produced by evaluating the model
        """

        # Check that the model is fitted
        if self.model_ is None:
            raise NotFittedError()

        # Convert the data to Java
        X_java = self._convert_run_data(X)

        # Get the PredictionResult
        pred_result = self.model_.transform(X_java)

        # Unlink the run data, which is no longer needed (to save memory)
        self.gateway.detach(X_java)
        return pred_result

class BaseLoloRegressor(BaseLoloLearner, RegressorMixin):
    """Abstract class for models that produce regression models.

    Implements the predict operation"""

    def predict(self, X, return_std=False):
        # Start the prediction process
        pred_result = self._get_prediction_result(X)

        # Pull out the expected values
        y_pred_byte = self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.getRegressionExpected(pred_result)
        y_pred = np.frombuffer(y_pred_byte, dtype='float')  # Lolo gives a byte array back

        # If desired, return the uncertainty too
        if return_std:
            # TODO: This part fails on Windows because the NativeSystemBLAS is not found. Fix that
            # TODO: This is only valid for regression models. Perhaps make a "LoloRegressor" class
            y_std_bytes = self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.getRegressionUncertainty(pred_result)
            y_std = np.frombuffer(y_std_bytes, 'float')
            return y_pred, y_std

        # Get the expected values
        return y_pred

class BaseLoloClassifier(BaseLoloLearner, ClassifierMixin):
    """Base class for classification models

    Implements a modification to the fit operation that stores the number of classes
    and the predict/predict_proba methods"""

    def fit(self, X, y, weights=None):
        # Get the number of classes
        self.n_classes_ = len(set(y))

        return super(BaseLoloClassifier, self).fit(X, y, weights)

    def predict(self, X):
        pred_result = self._get_prediction_result(X)

        # Pull out the expected values
        y_pred_byte = self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.getClassifierExpected(pred_result)
        y_pred = np.frombuffer(y_pred_byte, dtype=np.int32)  # Lolo gives a byte array back

        return y_pred

    def predict_proba(self, X):
        pred_result = self._get_prediction_result(X)

        # Copy over the class probabilities
        probs_byte = self.gateway.jvm.io.citrine.lolo.util.LoloPyDataLoader.getClassifierProbabilities(pred_result,
                                                                                self.n_classes_)
        probs = np.frombuffer(probs_byte, dtype='float').reshape(-1, self.n_classes_)
        return probs

class RandomForestMixin(BaseLoloLearner):
    """Random Forest base class

    Implements the _make_learner operation and the __init__ function with options specific to the RandomForest
    class in Lolo"""

    def __init__(self, num_trees=-1, use_jackknife=True, bias_learner=None,
                 leaf_learner=None, subset_strategy="auto", min_leaf_instances=1,
                 max_depth=2**30, uncertainty_calibration=False, randomize_pivot_location=False):
        """Initialize the RandomForest

        Args:
            num_trees (int): Number of trees to use in the forest
            use_jackknife (bool): Whether to use jackknife based variance estimates
            bias_learner (BaseLoloLearner): Algorithm used to model bias (default: no model)
            leaf_learner (BaseLoloLearner): Learner used at each leaf of the random forest (default: GuessTheMean)
            subset_strategy (Union[string,int,float]): Strategy used to determine number of features used at each split
                Available options:
                    "auto": Use the default for lolo (all features for regression, sqrt for classification)
                    "log2": Use the base 2 log of the number of features
                    "sqrt": Use the square root of the number of features
                    integer: Set the number of features explicitly
                    float: Use a certain fraction of the features
            min_leaf_instances (int): Minimum number of features used at each leaf
            max_depth (int): Maximum depth to which to allow the decision trees to grow
            uncertainty_calibration (bool): whether to re-calibrate the predicted uncertainty based on out-of-bag residuals
            randomize_pivot_location (bool): whether to draw pivots randomly or always select the midpoint
        """
        super(RandomForestMixin, self).__init__()

        # Store the variables
        self.num_trees = num_trees
        self.use_jackknife = use_jackknife
        self.subset_strategy = subset_strategy
        self.bias_learner = bias_learner
        self.leaf_learner = leaf_learner
        self.min_leaf_instances = min_leaf_instances
        self.max_depth = max_depth
        self.uncertainty_calibration = uncertainty_calibration
        self.randomize_pivot_location = randomize_pivot_location

    def _make_learner(self):
        #  TODO: Figure our a more succinct way of dealing with optional arguments/Option values
        #  TODO: that ^^, please
        learner = self.gateway.jvm.io.citrine.lolo.learners.RandomForest(
            self.num_trees, self.use_jackknife,
            getattr(self.gateway.jvm.io.citrine.lolo.learners.RandomForest,
                    "$lessinit$greater$default$3")() if self.bias_learner is None
            else self.gateway.jvm.scala.Some(self.bias_learner._make_learner()),
            getattr(self.gateway.jvm.io.citrine.lolo.learners.RandomForest,
                    "$lessinit$greater$default$4")() if self.leaf_learner is None
            else self.gateway.jvm.scala.Some(self.leaf_learner._make_learner()),
            self.subset_strategy,
            self.min_leaf_instances,
            self.max_depth,
            self.uncertainty_calibration,
            self.randomize_pivot_location
        )
        return learner

class RandomForestRegressor(BaseLoloRegressor, RandomForestMixin):
    """Random Forest model used for regression"""

class RandomForestClassifier(BaseLoloClassifier, RandomForestMixin):
    """Random Forest model used for classification"""

class RegressionTreeLearner(BaseLoloRegressor):
    """Regression tree learner, based on the RandomTree algorithm."""

    def __init__(self, num_features=-1, max_depth=30, min_leaf_instances=1, leaf_learner=None):
        """Initialize the learner

        Args:
            num_features (int): Number of features to consider at each split (-1 to consider all features)
            max_depth (int): Maximum depth of the regression tree
            min_leaf_instances (int): Minimum number instances per leaf
            leaf_learner (BaseLoloLearner): Learner to use on the leaves
        """
        super(RegressionTreeLearner, self).__init__()
        self.num_features = num_features
        self.max_depth = max_depth
        self.min_leaf_instances = min_leaf_instances
        self.leaf_learner = leaf_learner

    def _make_learner(self):
        if self.leaf_learner is None:
            # pull out default learner
            leaf_learner = getattr(
                self.gateway.jvm.io.citrine.lolo.trees.regression.RegressionTreeLearner,
                "$lessinit$greater$default$4"
            )()
        else:
            leaf_learner = self.gateway.jvm.scala.Some(self.leaf_learner._make_learner())

        # pull out default splitter
        splitter = getattr(
            self.gateway.jvm.io.citrine.lolo.trees.regression.RegressionTreeLearner,
            "$lessinit$greater$default$5"
        )()

        return self.gateway.jvm.io.citrine.lolo.trees.regression.RegressionTreeLearner(
            self.num_features, self.max_depth, self.min_leaf_instances,
            leaf_learner, splitter
        )

class LinearRegression(BaseLoloRegressor):
    """Linear ridge regression with an :math:`L_2` penalty"""

    def __init__(self, reg_param=None, fit_intercept=True):
        """Initialize the regressor"""

        super(LinearRegression, self).__init__()

        self.reg_param = reg_param
        self.fit_intercept = fit_intercept

    def _make_learner(self):
        return self.gateway.jvm.io.citrine.lolo.linear.LinearRegressionLearner(
            getattr(self.gateway.jvm.io.citrine.lolo.linear.LinearRegressionLearner,
                    "$lessinit$greater$default$1")() if self.reg_param is None
            else self.gateway.jvm.scala.Some(float(self.reg_param)),
            self.fit_intercept
        )
<|endoftext|>


<commit_before>from . import handleprops

class ElementInfo(object):
    "Wrapper for window handler"

    @property
    def handle(self):
        return self._handle

    @handle.setter
    def handle(self, handle):
        assert handle
        self._handle = handle

    @property
    def windowText(self):
        return self._windowText

    @windowText.setter
    def windowText(self, windowText):
        self._windowText = windowText

    @property
    def controlId(self):
        return self._controlId

    @property
    def processId(self):
        return self._processId

    @property
    def className(self):
        return self._className

    @property
    def enabled(self):
        self._enabled = handleprops.isenabled(self._handle)
        return self._enabled

    @enabled.setter
    def enabled(self, enabled):
        self._enabled = enabled

    @property
    def visible(self):
        self._visible = handleprops.isvisible(self._handle)
        return self._visible

    @visible.setter
    def visible(self, visible):
        self._visible = visible

    @property
    def parent(self):
        return self._parent

    @parent.setter
    def parent(self, parent):
        self._parent = parent

    def children(self):
        pass

    def dumpWindow(self):
        pass

class NativeElementInfo(ElementInfo):
    def __init__(self, handle):
        self._handle = handle
        self._windowText = handleprops.text(handle)
        self._controlId = handleprops.controlid(handle)
        self._processId = handleprops.processid(handle)
        self._className = handleprops.classname(handle)
        self._enabled = handleprops.isenabled(handle)
        self._visible = handleprops.isvisible(handle)
        self._parent  = handleprops.parent(handle)

    def children(self):
        child_handles = handleprops.children(self.handle)
        return [NativeElementInfo(ch) for ch in child_handles]

    def dumpWindow(self):
        return handleprops.dumpwindow(self.handle)

<commit_msg>Added docstring lines. Removed  setter for _visible and _enabled<commit_after>from . import handleprops

class ElementInfo(object):
    "Wrapper for window handler"

    @property
    def handle(self):
        "Return the handle of the window"
        return self._handle

    @handle.setter
    def handle(self, handle):
        assert handle
        self._handle = handle

    @property
    def windowText(self):
        "Return the text of the window"
        return self._windowText

    @windowText.setter
    def windowText(self, windowText):
        self._windowText = windowText

    @property
    def controlId(self):
        "Return the ID of the control"
        return self._controlId

    @property
    def processId(self):
        "Return the ID of process that controls this window"
        return self._processId

    @property
    def className(self):
        "Return the class name of the window"
        return self._className

    @property
    def enabled(self):
        "Return True if the window is enabled"
        self._enabled = handleprops.isenabled(self._handle)
        return self._enabled

    @property
    def visible(self):
        "Return True if the window is visible"
        self._visible = handleprops.isvisible(self._handle)
        return self._visible

    @property
    def parent(self):
        "Return the handle of the parent of the window"
        return self._parent

    @parent.setter
    def parent(self, parent):
        self._parent = parent

    def children(self):
        pass

    def dumpWindow(self):
        pass

class NativeElementInfo(ElementInfo):
    def __init__(self, handle):
        self._handle = handle
        self._windowText = handleprops.text(handle)
        self._controlId = handleprops.controlid(handle)
        self._processId = handleprops.processid(handle)
        self._className = handleprops.classname(handle)
        self._enabled = handleprops.isenabled(handle)
        self._visible = handleprops.isvisible(handle)
        self._parent  = handleprops.parent(handle)

    def children(self):
        "Return a list of handles to the children of this window"
        child_handles = handleprops.children(self.handle)
        return [NativeElementInfo(ch) for ch in child_handles]

    def dumpWindow(self):
        "Dump a window to a set of properties"
        return handleprops.dumpwindow(self.handle)

<|endoftext|>


<commit_before>##################################################################
# Copyright 2016 OSGeo Foundation,                               #
# represented by PyWPS Project Steering Committee,               #
# licensed under MIT, Please consult LICENSE.txt for details     #
##################################################################

import os
from lxml import etree
import time
from werkzeug.wrappers import Request
from werkzeug.exceptions import HTTPException
from pywps import WPS, OWS
from pywps.app.basic import xml_response
from pywps.exceptions import NoApplicableCode
import pywps.configuration as config
from pywps.dblog import update_response
from collections import namedtuple

_STATUS = namedtuple('Status', 'ERROR_STATUS, NO_STATUS, STORE_STATUS,'
                     'STORE_AND_UPDATE_STATUS, DONE_STATUS')

STATUS = _STATUS(0, 10, 20, 30, 40)

class WPSResponse(object):

    def __init__(self, process, wps_request, uuid):
        """constructor

        :param pywps.app.Process.Process process:
        :param pywps.app.WPSRequest.WPSRequest wps_request:
        :param uuid: string this request uuid
        """

        self.process = process
        self.wps_request = wps_request
        self.outputs = {o.identifier: o for o in process.outputs}
        self.message = ''
        self.status = STATUS.NO_STATUS
        self.status_percentage = 0
        self.doc = None
        self.uuid = uuid

    def update_status(self, message=None, status_percentage=None, status=None,
                      clean=True):
        """
        Update status report of currently running process instance

        :param str message: Message you need to share with the client
        :param int status_percentage: Percent done (number betwen <0-100>)
        :param pywps.app.WPSResponse.STATUS status: process status - user should usually
            ommit this parameter
        """

        if message:
            self.message = message

        if status:
            self.status = status

        if status_percentage:
            self.status_percentage = status_percentage

        # check if storing of the status is requested
        if self.status >= STATUS.STORE_AND_UPDATE_STATUS:

            # rebuild the doc and update the status xml file
            self.doc = self._construct_doc()
            self.write_response_doc(self.doc, clean)

        update_response(self.uuid, self)

    def write_response_doc(self, doc, clean=True):
        # TODO: check if file/directory is still present, maybe deleted in mean time

        try:
            with open(self.process.status_location, 'w') as f:
                f.write(etree.tostring(doc, pretty_print=True, encoding='utf-8').decode('utf-8'))
                f.flush()
                os.fsync(f.fileno())

            if self.status >= STATUS.DONE_STATUS and clean:
                self.process.clean()

        except IOError as e:
            raise NoApplicableCode('Writing Response Document failed with : %s' % e)

    def _process_accepted(self):
        return WPS.Status(
            WPS.ProcessAccepted(self.message),
            creationTime=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime())
        )

    def _process_started(self):
        return WPS.Status(
            WPS.ProcessStarted(
                self.message,
                percentCompleted=str(self.status_percentage)
            ),
            creationTime=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime())
        )

    def _process_paused(self):
        return WPS.Status(
            WPS.ProcessPaused(
                self.message,
                percentCompleted=str(self.status_percentage)
            ),
            creationTime=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime())
        )

    def _process_succeeded(self):
        return WPS.Status(
            WPS.ProcessSucceeded(self.message),
            creationTime=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime())
        )

    def _process_failed(self):
        return WPS.Status(
            WPS.ProcessFailed(
                WPS.ExceptionReport(
                    OWS.Exception(
                        OWS.Exception(self.message),
                        exceptionCode='NoApplicableCode',
                        locater='None'
                    )
                )
            ),
            creationTime=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime())
        )

    def _construct_doc(self):
        doc = WPS.ExecuteResponse()
        doc.attrib['{http://www.w3.org/2001/XMLSchema-instance}schemaLocation'] = \
            'http://www.opengis.net/wps/1.0.0 http://schemas.opengis.net/wps/1.0.0/wpsExecute_response.xsd'
        doc.attrib['service'] = 'WPS'
        doc.attrib['version'] = '1.0.0'
        doc.attrib['{http://www.w3.org/XML/1998/namespace}lang'] = 'en-US'
        doc.attrib['serviceInstance'] = '%s%s' % (
            config.get_config_value('server', 'url'),
            '?service=WPS&request=GetCapabilities'
        )

        if self.status >= STATUS.STORE_STATUS:
            if self.process.status_location:
                doc.attrib['statusLocation'] = self.process.status_url

        # Process XML
        process_doc = WPS.Process(
            OWS.Identifier(self.process.identifier),
            OWS.Title(self.process.title)
        )
        if self.process.abstract:
            process_doc.append(OWS.Abstract(self.process.abstract))
        # TODO: See Table 32 Metadata in OGC 06-121r3
        # for m in self.process.metadata:
        #    process_doc.append(OWS.Metadata(m))
        if self.process.profile:
            process_doc.append(OWS.Profile(self.process.profile))
        process_doc.attrib['{http://www.opengis.net/wps/1.0.0}processVersion'] = self.process.version

        doc.append(process_doc)

        # Status XML
        # return the correct response depending on the progress of the process
        if self.status == STATUS.STORE_AND_UPDATE_STATUS:
            if self.status_percentage == 0:
                self.message = 'PyWPS Process %s accepted' % self.process.identifier
                status_doc = self._process_accepted()
                doc.append(status_doc)
                return doc
            elif self.status_percentage > 0:
                status_doc = self._process_started()
                doc.append(status_doc)
                return doc

        # check if process failed and display fail message
        if self.status_percentage == -1:
            status_doc = self._process_failed()
            doc.append(status_doc)
            return doc

        # TODO: add paused status

        if self.status == STATUS.DONE_STATUS:
            status_doc = self._process_succeeded()
            doc.append(status_doc)

            # DataInputs and DataOutputs definition XML if lineage=true
            if self.wps_request.lineage == 'true':
                data_inputs = [self.wps_request.inputs[i][0].execute_xml() for i in self.wps_request.inputs]
                doc.append(WPS.DataInputs(*data_inputs))

                output_definitions = [self.outputs[o].execute_xml_lineage() for o in self.outputs]
                doc.append(WPS.OutputDefinitions(*output_definitions))

            # Process outputs XML
            output_elements = [self.outputs[o].execute_xml() for o in self.outputs]
            doc.append(WPS.ProcessOutputs(*output_elements))
        return doc

    def call_on_close(self, function):
        """Custom implementation of call_on_close of werkzeug
        TODO: rewrite this using werkzeug's tools
        """
        self._close_functions.push(function)

    @Request.application
    def __call__(self, request):
        doc = None
        try:
            doc = self._construct_doc()
        except HTTPException as httpexp:
            raise httpexp
        except Exception as exp:
            raise NoApplicableCode(exp)

        if self.status >= STATUS.DONE_STATUS:
            self.process.clean()

        return xml_response(doc)
<commit_msg>fix update of exception response<commit_after>##################################################################
# Copyright 2016 OSGeo Foundation,                               #
# represented by PyWPS Project Steering Committee,               #
# licensed under MIT, Please consult LICENSE.txt for details     #
##################################################################

import os
from lxml import etree
import time
from werkzeug.wrappers import Request
from werkzeug.exceptions import HTTPException
from pywps import WPS, OWS
from pywps.app.basic import xml_response
from pywps.exceptions import NoApplicableCode
import pywps.configuration as config
from pywps.dblog import update_response
from collections import namedtuple

_STATUS = namedtuple('Status', 'ERROR_STATUS, NO_STATUS, STORE_STATUS,'
                     'STORE_AND_UPDATE_STATUS, DONE_STATUS')

STATUS = _STATUS(0, 10, 20, 30, 40)

class WPSResponse(object):

    def __init__(self, process, wps_request, uuid):
        """constructor

        :param pywps.app.Process.Process process:
        :param pywps.app.WPSRequest.WPSRequest wps_request:
        :param uuid: string this request uuid
        """

        self.process = process
        self.wps_request = wps_request
        self.outputs = {o.identifier: o for o in process.outputs}
        self.message = ''
        self.status = STATUS.NO_STATUS
        self.status_percentage = 0
        self.doc = None
        self.uuid = uuid

    def update_status(self, message=None, status_percentage=None, status=None,
                      clean=True):
        """
        Update status report of currently running process instance

        :param str message: Message you need to share with the client
        :param int status_percentage: Percent done (number betwen <0-100>)
        :param pywps.app.WPSResponse.STATUS status: process status - user should usually
            ommit this parameter
        """

        if message:
            self.message = message

        if status:
            self.status = status

        if status_percentage:
            self.status_percentage = status_percentage

        # check if storing of the status is requested
        if self.status >= STATUS.STORE_AND_UPDATE_STATUS:

            # rebuild the doc and update the status xml file
            self.doc = self._construct_doc()
            self.write_response_doc(self.doc, clean)

        update_response(self.uuid, self)

    def write_response_doc(self, doc, clean=True):
        # TODO: check if file/directory is still present, maybe deleted in mean time

        try:
            with open(self.process.status_location, 'w') as f:
                f.write(etree.tostring(doc, pretty_print=True, encoding='utf-8').decode('utf-8'))
                f.flush()
                os.fsync(f.fileno())

            if self.status >= STATUS.DONE_STATUS and clean:
                self.process.clean()

        except IOError as e:
            raise NoApplicableCode('Writing Response Document failed with : %s' % e)

    def _process_accepted(self):
        return WPS.Status(
            WPS.ProcessAccepted(self.message),
            creationTime=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime())
        )

    def _process_started(self):
        return WPS.Status(
            WPS.ProcessStarted(
                self.message,
                percentCompleted=str(self.status_percentage)
            ),
            creationTime=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime())
        )

    def _process_paused(self):
        return WPS.Status(
            WPS.ProcessPaused(
                self.message,
                percentCompleted=str(self.status_percentage)
            ),
            creationTime=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime())
        )

    def _process_succeeded(self):
        return WPS.Status(
            WPS.ProcessSucceeded(self.message),
            creationTime=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime())
        )

    def _process_failed(self):
        return WPS.Status(
            WPS.ProcessFailed(
                WPS.ExceptionReport(
                    OWS.Exception(
                        OWS.ExceptionText(self.message),
                        exceptionCode='NoApplicableCode',
                        locater='None'
                    )
                )
            ),
            creationTime=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.localtime())
        )

    def _construct_doc(self):
        doc = WPS.ExecuteResponse()
        doc.attrib['{http://www.w3.org/2001/XMLSchema-instance}schemaLocation'] = \
            'http://www.opengis.net/wps/1.0.0 http://schemas.opengis.net/wps/1.0.0/wpsExecute_response.xsd'
        doc.attrib['service'] = 'WPS'
        doc.attrib['version'] = '1.0.0'
        doc.attrib['{http://www.w3.org/XML/1998/namespace}lang'] = 'en-US'
        doc.attrib['serviceInstance'] = '%s%s' % (
            config.get_config_value('server', 'url'),
            '?service=WPS&request=GetCapabilities'
        )

        if self.status >= STATUS.STORE_STATUS:
            if self.process.status_location:
                doc.attrib['statusLocation'] = self.process.status_url

        # Process XML
        process_doc = WPS.Process(
            OWS.Identifier(self.process.identifier),
            OWS.Title(self.process.title)
        )
        if self.process.abstract:
            process_doc.append(OWS.Abstract(self.process.abstract))
        # TODO: See Table 32 Metadata in OGC 06-121r3
        # for m in self.process.metadata:
        #    process_doc.append(OWS.Metadata(m))
        if self.process.profile:
            process_doc.append(OWS.Profile(self.process.profile))
        process_doc.attrib['{http://www.opengis.net/wps/1.0.0}processVersion'] = self.process.version

        doc.append(process_doc)

        # Status XML
        # return the correct response depending on the progress of the process
        if self.status == STATUS.STORE_AND_UPDATE_STATUS:
            if self.status_percentage == 0:
                self.message = 'PyWPS Process %s accepted' % self.process.identifier
                status_doc = self._process_accepted()
                doc.append(status_doc)
                return doc
            elif self.status_percentage > 0:
                status_doc = self._process_started()
                doc.append(status_doc)
                return doc

        # check if process failed and display fail message
        if self.status_percentage == -1:
            status_doc = self._process_failed()
            doc.append(status_doc)
            return doc

        # TODO: add paused status

        if self.status == STATUS.DONE_STATUS:
            status_doc = self._process_succeeded()
            doc.append(status_doc)

            # DataInputs and DataOutputs definition XML if lineage=true
            if self.wps_request.lineage == 'true':
                data_inputs = [self.wps_request.inputs[i][0].execute_xml() for i in self.wps_request.inputs]
                doc.append(WPS.DataInputs(*data_inputs))

                output_definitions = [self.outputs[o].execute_xml_lineage() for o in self.outputs]
                doc.append(WPS.OutputDefinitions(*output_definitions))

            # Process outputs XML
            output_elements = [self.outputs[o].execute_xml() for o in self.outputs]
            doc.append(WPS.ProcessOutputs(*output_elements))
        return doc

    def call_on_close(self, function):
        """Custom implementation of call_on_close of werkzeug
        TODO: rewrite this using werkzeug's tools
        """
        self._close_functions.push(function)

    @Request.application
    def __call__(self, request):
        doc = None
        try:
            doc = self._construct_doc()
        except HTTPException as httpexp:
            raise httpexp
        except Exception as exp:
            raise NoApplicableCode(exp)

        if self.status >= STATUS.DONE_STATUS:
            self.process.clean()

        return xml_response(doc)
<|endoftext|>


<commit_before>from react import jsx
from jinja2 import Template
import json

class ReactTemplate(object):
    """Base React class.

        This class handles the basic transformation of the React jsx code.
        It takes a dictionary of options, renders the template, and
        writes the transformed javascript file.

        Args:
            template (str): jinja2 template for the react jsx code.
            template_args (dict): options to be populated in the template.
            path (str): path of the file to create.
    """
    def __init__(self, template, template_args, path):
        self.transformer = jsx.JSXTransformer()
        self.template = template
        self.args = template_args
        self.path = path

    def write_to_file(self, s):
        """
        """
        f = open(self.path, 'w')
        f.write(s)
        f.close()

    def to_js(self):
        """
        """
        t = Template(self.template)
        js = self.transformer.transform_string(t.render(**self.args))
        self.write_to_file(js)

def format_props(props):
    """ Formats props for the React template.

        Args:
            props (dict): properties to be written to the template.

        Returns:
            Two lists, one containing variable names and the other
            containing a list of props to be fed to the React template.

    """
    vars_ = []
    props_ = []
    for k, v in list(props.items()):
        vars_.append(Template("var {{k}} = {{v}};").render(k=k,v=json.dumps(v)))
        props_.append(Template("{{k}} = {{v}}").render(k=k, v="{"+k+"}"))
    return "\n".join(vars_), "\n".join(props_)

class ReactComponent(ReactTemplate):
    """Class to generate javascript from parameters.

        This class create a javascript file by compiling a JSX string containing
        React components. A single parent component is rendered and sub-components
        are passed into the component as props.

        Args:
            name (str): name of the ReactComponent to create.
            path (str): location of the javascript file containing the component.
            element_id (str): html element id.
            props (dict): properties of the react component to render.
            static_path (str): output file destination.
    """
    _react = (
    """
    var Component = require("{{path}}").{{name}};
    {{vars}}
    React.render(
        <Component
        {{props}} />,
        document.getElementById("{{id}}")
    );
    """)
    def __init__(self, name, path, element_id, props={}, static_path=""):
        vars_, props_ = format_props(props)

        params = {
            "name": name,
            "path": path,
            "vars": vars_,
            "props": props_,
            "id": element_id
        }
        for k, v in list(props.items()):
            params[k] = v

        super(ReactComponent, self).__init__(
            self._react, params, static_path)
        self.to_js()

<commit_msg>removing pyreact dependence; modify template to include es6 syntax; adding template and option for exporting a module (for use with react-router);<commit_after>from jinja2 import Template
import json

class ReactTemplate(object):
    """Base React class.

        This class handles the basic transformation of the React jsx code.
        It takes a dictionary of options, renders the template, and
        writes the transformed javascript file.

        Args:
            template (str): jinja2 template for the react jsx code.
            template_args (dict): options to be populated in the template.
            path (str): path of the file to create.
    """
    def __init__(self, template, template_args, path):
        self.template = template
        self.args = template_args
        self.path = path

    def write_to_file(self, s):
        """
        """
        f = open(self.path, 'w')
        f.write(s)
        f.close()

    def to_js(self):
        """
        """
        t = Template(self.template)
        js = t.render(**self.args)
        self.write_to_file(js)

def format_props(props, prop_template="{{k}} = { {{v}} }", delim="\n"):
    """ Formats props for the React template.

        Args:
            props (dict): properties to be written to the template.

        Returns:
            Two lists, one containing variable names and the other
            containing a list of props to be fed to the React template.

    """
    vars_ = []
    props_ = []
    for k, v in list(props.items()):
        vars_.append(Template("var {{k}} = {{v}};").render(k=k,v=json.dumps(v)))
        props_.append(Template(prop_template).render(k=k, v=k))
    return "\n".join(vars_), delim.join(props_)

class ReactComponent(ReactTemplate):
    """Class to generate javascript from parameters.

        This class create a javascript file by compiling a JSX string containing
        React components. A single parent component is rendered and sub-components
        are passed into the component as props.

        Args:
            name (str): name of the ReactComponent to create.
            path (str): location of the javascript file containing the component.
            element_id (str): html element id.
            props (dict): properties of the react component to render.
            static_path (str): output file destination.
    """
    _react = (
    """
    import React from 'react';
    import ReactDOM from 'react-dom';
    import { {{name}} as Component} from './{{path}}';
    {{vars}}
    ReactDOM.render(
        <Component
        {{props}} />,
        document.getElementById("{{id}}")
    );
    """)

    _react_export = (
    """
    import React from 'react';
    import ReactDOM from 'react-dom';
    import { {{name}} as Component} from './{{path}}';

    {{vars}}

    class {{alias}} extends React.Component {
        constructor(props) {
            super(props);
        }

        render() {
            return (
                <Component { ...this.props } />
            );
        }
    }
    {{alias}}.defaultProps = {
        {{props}}
    };
    export default {{alias}};

    """)
    def __init__(self, name, path, element_id,
        props={}, static_path="", alias=None):

        _template = self._react
        prop_template = "{{k}} = { {{v}} }"
        delim = "\n"
        if alias:
            prop_template = "{{k}} : {{v}}"
            delim = ",\n"
            _template = self._react_export

        vars_, props_ = format_props(props,
            prop_template=prop_template, delim=delim)

        params = {
            "name": name,
            "alias": alias,
            "path": path,
            "vars": vars_,
            "props": props_,
            "id": element_id
        }
        for k, v in list(props.items()):
            params[k] = v

        super(ReactComponent, self).__init__(
            _template, params, static_path)
        self.to_js()
<|endoftext|>