<commit_before># -*- coding: utf-8 -*-
#
'''
I/O for VTK, VTU, Exodus etc.

.. moduleauthor:: Nico Schlömer <nico.schloemer@gmail.com>
'''
import base64
import numpy
import struct
import zlib

from .gmsh_io import num_nodes_per_cell

def _cells_from_data(connectivity, offsets, types):
    # create cells
    vtk_to_meshio_type = {
        1: 'vertex',
        3: 'line',
        5: 'triangle',
        9: 'quad',
        10: 'tetra',
        12: 'hexahedron',
        13: 'wedge',
        14: 'pyramid',
        21: 'line3',
        22: 'triangle6',
        23: 'quad8',
        24: 'tetra10',
        25: 'hexahedron20',
        }

    # assert (types == types[0]).all()

    # Translate it into the cells dictionary.
    # `connectivity` is a one-dimensional vector with
    # (p0, p1, ... ,pk, p10, p11, ..., p1k, ...

    # Collect types into bins.
    # See <https://stackoverflow.com/q/47310359/353337> for better
    # alternatives.
    uniques = numpy.unique(types)
    bins = {u: numpy.where(types == u)[0] for u in uniques}

    cells = {}
    for tpe, b in bins.items():
        meshio_type = vtk_to_meshio_type[tpe]
        n = num_nodes_per_cell[meshio_type]
        indices = numpy.array([
            # The offsets point to the _end_ of the indices
            numpy.arange(n) + o - n for o in offsets[b]
            ])
        cells[meshio_type] = connectivity[indices]

    return cells

def _read_binary(data, byte_order, header_type, data_type):
    # https://docs.python.org/2/library/struct.html
    vtu_to_struct_type = {
        'UInt32': ('I', 4),
        'UInt64': ('Q', 8),
        'Float32': ('f', 4),
        'Float64': ('d', 8),
        }

    # process the header
    byte_string = base64.b64decode(data)
    bo = '<' if byte_order == 'LittleEndian' else '>'
    symbol, num_bytes = vtu_to_struct_type[header_type]
    num_blocks = int(struct.unpack(
        bo + symbol, byte_string[0:num_bytes]
        )[0])
    # Not needed:
    # uncompressed_size = int(struct.unpack(
    #     bo + symbol, byte_string[num_bytes:2*num_bytes]
    #     )[0])
    # last_block_size = int(struct.unpack(
    #     bo + symbol, byte_string[2*num_bytes:3*num_bytes]
    #     )[0])
    # block_sizes = [
    #     int(struct.unpack(
    #         bo + symbol,
    #         byte_string[(3+k)*num_bytes:(4+k)*num_bytes]
    #         )[0])
    #     for k in range(num_blocks)
    #     ]

    # Check how many characters the header occupies. This is
    # determined according to base64 encoding.
    header_num_bytes = (3 + num_blocks) * num_bytes
    if header_num_bytes % 3 == 0:
        header_num_chars = header_num_bytes // 3 * 4
    elif header_num_bytes % 3 == 1:
        header_num_chars = (header_num_bytes+2) // 3 * 4
    else:
        assert header_num_bytes % 3 == 2
        header_num_chars = (header_num_bytes+1) // 3 * 4

    # process the compressed data
    compressed_data = data[header_num_chars:]
    decompressed = \
        zlib.decompress(base64.b64decode(compressed_data))

    struct_type, num_bytes = vtu_to_struct_type[data_type]

    assert len(decompressed) % num_bytes == 0

    out = numpy.array([
        struct.unpack(
            struct_type,
            decompressed[num_bytes*k:num_bytes*(k+1)]
            )[0]
        for k in range(len(decompressed) // num_bytes)
        ])

    return out

def read(filename):
    from lxml import etree as ET

    tree = ET.parse(filename)
    root = tree.getroot()

    assert root.tag == 'VTKFile'
    assert root.attrib['type'] == 'UnstructuredGrid'
    assert root.attrib['version'] in ['0.1', '1.0'], \
        'Unknown VTU file version \'{}\'.'.format(root.attrib['version'])
    assert root.attrib['compressor'] == 'vtkZLibDataCompressor'

    header_type = (
        root.attrib['header_type'] if 'header_type' in root.attrib
        else 'UInt32'
        )

    byte_order = root.attrib['byte_order']
    assert byte_order in ['LittleEndian', 'BigEndian'], \
        'Unknown byte order \'{}\'.'.format(byte_order)

    children = root.getchildren()
    assert len(children) == 1
    grid = children[0]
    assert grid.tag == 'UnstructuredGrid'

    pieces = grid.getchildren()
    assert len(pieces) == 1

    piece = pieces[0]

    num_points = int(piece.attrib['NumberOfPoints'])

    points = None
    point_data = {}
    cell_data_raw = {}
    field_data = {}

    connectivity = None
    offsets = None
    types = None

    vtu_to_numpy_type = {
        'Float64': numpy.float64,
        'Int64': numpy.int64,
        'UInt8': numpy.uint8,
        'UInt32': numpy.uint32,
        }

    for child in piece.getchildren():
        if child.tag == 'Points':
            c = child.getchildren()
            assert len(c) == 1
            c = c[0]
            assert c.tag == 'DataArray'
            assert c.attrib['Name'] == 'Points'

            if c.attrib['format'] == 'ascii':
                points = numpy.array(
                    c.text.split(),
                    dtype=vtu_to_numpy_type[c.attrib['type']]
                    )
            else:
                assert c.attrib['format'] == 'binary', \
                    'Unknown data format \'{}\'.'.format(c.attrib['format'])
                points = _read_binary(
                    c.text.strip(), byte_order, header_type, c.attrib['type']
                    )

            num_components = int(c.attrib['NumberOfComponents'])
            points = points.reshape(num_points, num_components)

        elif child.tag == 'Cells':
            for data in child.getchildren():
                assert data.tag == 'DataArray'
                if data.attrib['Name'] == 'connectivity':
                    if data.attrib['format'] == 'ascii':
                        connectivity = numpy.array(
                            data.text.split(),
                            dtype=vtu_to_numpy_type[data.attrib['type']]
                            )
                    else:
                        assert data.attrib['format'] == 'binary'

                        exit(1)

                elif data.attrib['Name'] == 'offsets':
                    if data.attrib['format'] == 'ascii':
                        offsets = numpy.array(
                            data.text.split(),
                            dtype=vtu_to_numpy_type[data.attrib['type']]
                            )
                    else:
                        assert data.attrib['format'] == 'binary'

                else:
                    assert data.attrib['Name'] == 'types', \
                        'Unknown array \'{}\'.'.format(data.attrib['Name'])
                    if data.attrib['format'] == 'ascii':
                        types = numpy.array(
                            data.text.split(),
                            dtype=vtu_to_numpy_type[data.attrib['type']]
                            )
                    else:
                        assert data.attrib['format'] == 'binary'
                        # import binascii
                        # print(data.text.strip())
                        # d = base64.b64decode(data.text.strip())
                        # print(d)
                        # d = base64.b64decode(data.text)
                        # print(d)
                        # d = base64.b64decode('AQAAAACAAAACAAAACgAAAA==')
                        # print(struct.unpack('I', d[0:4]))
                        # print(struct.unpack('I', d[4:8]))
                        # print(struct.unpack('I', d[8:12]))
                        # print(struct.unpack('I', d[12:16]))

        elif child.tag == 'PointData':
            for c in child.getchildren():
                assert c.tag == 'DataArray'
                assert c.attrib['format'] == 'ascii'
                point_data[c.attrib['Name']] = numpy.array(
                    c.text.split(),
                    dtype=vtu_to_numpy_type[c.attrib['type']]
                    )

        else:
            assert child.tag == 'CellData', \
                'Unknown tag \'{}\'.'.format(child.tag)
            for c in child.getchildren():
                assert c.tag == 'DataArray'
                assert c.attrib['format'] == 'ascii'
                cell_data_raw[c.attrib['Name']] = numpy.array(
                    c.text.split(),
                    dtype=vtu_to_numpy_type[c.attrib['type']]
                    )

    assert points is not None
    assert connectivity is not None
    assert offsets is not None
    assert types is not None

    cells = _cells_from_data(connectivity, offsets, types)

    # get point data in shape
    for key in point_data:
        if len(point_data[key]) != len(points):
            point_data[key] = point_data[key].reshape(len(points), -1)

    # get cell data in shape
    cell_data = {}
    num_all_cells = sum([len(c) for c in cells.values()])
    for key in cell_data_raw:
        d = cell_data_raw[key]
        if len(d) != num_all_cells:
            d = d.reshape(num_all_cells, -1)

        r = 0
        for k in cells:
            cell_data[k] = {key: d[r:r+len(cells[k])]}
            r += len(cells[k])

    return points, cells, point_data, cell_data, field_data

def write(filetype,
          filename,
          points,
          cells,
          point_data=None,
          cell_data=None,
          field_data=None
          ):
    # pylint: disable=import-error
    from .vtk_io import write as vtk_write
    return vtk_write(
        filetype, filename, points, cells,
        point_data=point_data, cell_data=cell_data, field_data=field_data
        )
<commit_msg>vtu: improvements for cell data<commit_after># -*- coding: utf-8 -*-
#
'''
I/O for VTK, VTU, Exodus etc.

.. moduleauthor:: Nico Schlömer <nico.schloemer@gmail.com>
'''
import base64
import numpy
import struct
import zlib

from .gmsh_io import num_nodes_per_cell

def _cells_from_data(connectivity, offsets, types):
    # create cells
    vtk_to_meshio_type = {
        1: 'vertex',
        3: 'line',
        5: 'triangle',
        9: 'quad',
        10: 'tetra',
        12: 'hexahedron',
        13: 'wedge',
        14: 'pyramid',
        21: 'line3',
        22: 'triangle6',
        23: 'quad8',
        24: 'tetra10',
        25: 'hexahedron20',
        }

    # assert (types == types[0]).all()

    # Translate it into the cells dictionary.
    # `connectivity` is a one-dimensional vector with
    # (p0, p1, ... ,pk, p10, p11, ..., p1k, ...

    # Collect types into bins.
    # See <https://stackoverflow.com/q/47310359/353337> for better
    # alternatives.
    uniques = numpy.unique(types)
    bins = {u: numpy.where(types == u)[0] for u in uniques}

    cells = {}
    for tpe, b in bins.items():
        meshio_type = vtk_to_meshio_type[tpe]
        n = num_nodes_per_cell[meshio_type]
        indices = numpy.array([
            # The offsets point to the _end_ of the indices
            numpy.arange(n) + o - n for o in offsets[b]
            ])
        cells[meshio_type] = connectivity[indices]

    return cells

def _read_binary(data, byte_order, header_type, data_type):
    # https://docs.python.org/2/library/struct.html
    vtu_to_struct_type = {
        'Float32': ('f', 4),
        'Float64': ('d', 8),
        'Int32': ('i', 4),
        'Int64': ('q', 8),
        'UInt8': ('B', 1),
        'UInt32': ('I', 4),
        'UInt64': ('Q', 8),
        }

    # process the header
    byte_string = base64.b64decode(data)
    bo = '<' if byte_order == 'LittleEndian' else '>'
    symbol, num_bytes = vtu_to_struct_type[header_type]
    num_blocks = int(struct.unpack(
        bo + symbol, byte_string[0:num_bytes]
        )[0])
    # Not needed:
    # uncompressed_size = int(struct.unpack(
    #     bo + symbol, byte_string[num_bytes:2*num_bytes]
    #     )[0])
    # last_block_size = int(struct.unpack(
    #     bo + symbol, byte_string[2*num_bytes:3*num_bytes]
    #     )[0])
    # block_sizes = [
    #     int(struct.unpack(
    #         bo + symbol,
    #         byte_string[(3+k)*num_bytes:(4+k)*num_bytes]
    #         )[0])
    #     for k in range(num_blocks)
    #     ]

    # Check how many characters the header occupies. This is
    # determined according to base64 encoding.
    header_num_bytes = (3 + num_blocks) * num_bytes
    if header_num_bytes % 3 == 0:
        header_num_chars = header_num_bytes // 3 * 4
    elif header_num_bytes % 3 == 1:
        header_num_chars = (header_num_bytes+2) // 3 * 4
    else:
        assert header_num_bytes % 3 == 2
        header_num_chars = (header_num_bytes+1) // 3 * 4

    # process the compressed data
    compressed_data = data[header_num_chars:]
    decompressed = \
        zlib.decompress(base64.b64decode(compressed_data))

    struct_type, num_bytes = vtu_to_struct_type[data_type]

    assert len(decompressed) % num_bytes == 0

    out = numpy.array([
        struct.unpack(
            struct_type,
            decompressed[num_bytes*k:num_bytes*(k+1)]
            )[0]
        for k in range(len(decompressed) // num_bytes)
        ])

    return out

def read(filename):
    from lxml import etree as ET

    tree = ET.parse(filename)
    root = tree.getroot()

    assert root.tag == 'VTKFile'
    assert root.attrib['type'] == 'UnstructuredGrid'
    assert root.attrib['version'] in ['0.1', '1.0'], \
        'Unknown VTU file version \'{}\'.'.format(root.attrib['version'])
    assert root.attrib['compressor'] == 'vtkZLibDataCompressor'

    header_type = (
        root.attrib['header_type'] if 'header_type' in root.attrib
        else 'UInt32'
        )

    byte_order = root.attrib['byte_order']
    assert byte_order in ['LittleEndian', 'BigEndian'], \
        'Unknown byte order \'{}\'.'.format(byte_order)

    children = root.getchildren()
    assert len(children) == 1
    grid = children[0]
    assert grid.tag == 'UnstructuredGrid'

    pieces = grid.getchildren()
    assert len(pieces) == 1

    piece = pieces[0]

    num_points = int(piece.attrib['NumberOfPoints'])

    points = None
    point_data = {}
    cell_data_raw = {}
    field_data = {}

    connectivity = None
    offsets = None
    types = None

    vtu_to_numpy_type = {
        'Float64': numpy.float64,
        'Int64': numpy.int64,
        'UInt8': numpy.uint8,
        'UInt32': numpy.uint32,
        }

    for child in piece.getchildren():
        if child.tag == 'Points':
            c = child.getchildren()
            assert len(c) == 1
            c = c[0]
            assert c.tag == 'DataArray'
            assert c.attrib['Name'] == 'Points'

            if c.attrib['format'] == 'ascii':
                points = numpy.array(
                    c.text.split(),
                    dtype=vtu_to_numpy_type[c.attrib['type']]
                    )
            else:
                assert c.attrib['format'] == 'binary', \
                    'Unknown data format \'{}\'.'.format(c.attrib['format'])
                points = _read_binary(
                    c.text.strip(), byte_order, header_type, c.attrib['type']
                    )

            num_components = int(c.attrib['NumberOfComponents'])
            points = points.reshape(num_points, num_components)

        elif child.tag == 'Cells':
            for data in child.getchildren():
                assert data.tag == 'DataArray'
                if data.attrib['Name'] == 'connectivity':
                    if data.attrib['format'] == 'ascii':
                        connectivity = numpy.array(
                            data.text.split(),
                            dtype=vtu_to_numpy_type[data.attrib['type']]
                            )
                    else:
                        assert data.attrib['format'] == 'binary'
                        connectivity = _read_binary(
                            data.text.strip(),
                            byte_order, header_type, data.attrib['type']
                            )

                elif data.attrib['Name'] == 'offsets':
                    if data.attrib['format'] == 'ascii':
                        offsets = numpy.array(
                            data.text.split(),
                            dtype=vtu_to_numpy_type[data.attrib['type']]
                            )
                    else:
                        assert data.attrib['format'] == 'binary'
                        offsets = _read_binary(
                            data.text.strip(),
                            byte_order, header_type, data.attrib['type']
                            )

                else:
                    assert data.attrib['Name'] == 'types', \
                        'Unknown array \'{}\'.'.format(data.attrib['Name'])
                    if data.attrib['format'] == 'ascii':
                        types = numpy.array(
                            data.text.split(),
                            dtype=vtu_to_numpy_type[data.attrib['type']]
                            )
                    else:
                        assert data.attrib['format'] == 'binary'
                        types = _read_binary(
                            data.text.strip(),
                            byte_order, header_type, data.attrib['type']
                            )

        elif child.tag == 'PointData':
            for c in child.getchildren():
                assert c.tag == 'DataArray'
                assert c.attrib['format'] == 'ascii'
                point_data[c.attrib['Name']] = numpy.array(
                    c.text.split(),
                    dtype=vtu_to_numpy_type[c.attrib['type']]
                    )

        else:
            assert child.tag == 'CellData', \
                'Unknown tag \'{}\'.'.format(child.tag)
            for c in child.getchildren():
                assert c.tag == 'DataArray'
                assert c.attrib['format'] == 'ascii'
                cell_data_raw[c.attrib['Name']] = numpy.array(
                    c.text.split(),
                    dtype=vtu_to_numpy_type[c.attrib['type']]
                    )

    assert points is not None
    assert connectivity is not None
    assert offsets is not None
    assert types is not None

    cells = _cells_from_data(connectivity, offsets, types)

    # get point data in shape
    for key in point_data:
        if len(point_data[key]) != len(points):
            point_data[key] = point_data[key].reshape(len(points), -1)

    # get cell data in shape
    cell_data = {}
    num_all_cells = sum([len(c) for c in cells.values()])
    for key in cell_data_raw:
        d = cell_data_raw[key]
        if len(d) != num_all_cells:
            d = d.reshape(num_all_cells, -1)

        r = 0
        for k in cells:
            cell_data[k] = {key: d[r:r+len(cells[k])]}
            r += len(cells[k])

    return points, cells, point_data, cell_data, field_data

def write(filetype,
          filename,
          points,
          cells,
          point_data=None,
          cell_data=None,
          field_data=None
          ):
    # pylint: disable=import-error
    from .vtk_io import write as vtk_write
    return vtk_write(
        filetype, filename, points, cells,
        point_data=point_data, cell_data=cell_data, field_data=field_data
        )
<|endoftext|>


<commit_before>from django.conf.urls import patterns, url

from api import views

urlpatterns = patterns(
    '',
    url(r'^image/$', views.image_handler, name='image'),
    url(r'^recipe/$', views.recipe_handler, name='recipe')
)
<commit_msg>fix bug<commit_after>from django.conf.urls import patterns, url

from api import views

urlpatterns = patterns(
    '',
    url(r'^image/$', views.image_handler, name='image'),
    url(r'^recipe/$', views.recipe_handler, name='recipe'),
    url(r'^list/$', views.list_handler, name='list')
)
<|endoftext|>


<commit_before>from __future__ import absolute_import

import datetime
from sunpy.time import TimeRange

def test_goes_event_list():
    trange=TimeRange('2011-06-07 00:00','2011-06-08 00:00')
    result=goes.get_goes_event_list(trange,goes_class_filter='M1')
    assert type(result) == list
    assert type(result[0]) == dict
    assert type(result[0]['event_date'] == str)
    assert type(result[0]['goes_location'] == tuple)
    assert type(result[0]['peak_time'] == datetime)
    assert type(result[0]['start_time'] == datetime)
    assert type(result[0]['end_time'] == datetime)
    assert type(result[0]['goes_class'] == str)
    assert type(result[0]['noaa_active_region'] == int)
    assert result[0]['event_date'] == '2011-06-07'
    assert result[0]['goes_location'] == (54, -21)
    assert result[0]['start_time'] == datetime.datetime(2011,6,7,6,16)
    assert result[0]['peak_time'] == datetime.datetime(2011,6,7,6,41)
    assert result[0]['end_time'] == datetime.datetime(2011,6,7,6,59)
    assert result[0]['goes_class'] == 'M2.5'
    assert result[0]['noaa_active_region'] == 11226

<commit_msg>fix for test_goes.py<commit_after>from __future__ import absolute_import

import datetime
from sunpy.time import TimeRange
from sunpy.instr import goes

def test_goes_event_list():
    trange=TimeRange('2011-06-07 00:00','2011-06-08 00:00')
    result=goes.get_goes_event_list(trange,goes_class_filter='M1')
    assert type(result) == list
    assert type(result[0]) == dict
    assert type(result[0]['event_date'] == str)
    assert type(result[0]['goes_location'] == tuple)
    assert type(result[0]['peak_time'] == datetime)
    assert type(result[0]['start_time'] == datetime)
    assert type(result[0]['end_time'] == datetime)
    assert type(result[0]['goes_class'] == str)
    assert type(result[0]['noaa_active_region'] == int)
    assert result[0]['event_date'] == '2011-06-07'
    assert result[0]['goes_location'] == (54, -21)
    assert result[0]['start_time'] == datetime.datetime(2011,6,7,6,16)
    assert result[0]['peak_time'] == datetime.datetime(2011,6,7,6,41)
    assert result[0]['end_time'] == datetime.datetime(2011,6,7,6,59)
    assert result[0]['goes_class'] == 'M2.5'
    assert result[0]['noaa_active_region'] == 11226

<|endoftext|>


<commit_before>import unittest
import time
import os
from datetime import datetime
import multiprocessing
import threading
from time import sleep
import json

import bashtasks as bashtasks_mod
import bashtasks.rabbit_util as rabbit_util
from bashtasks.constants import TASK_REQUESTS_POOL, TASK_RESPONSES_POOL
from test.pika_assertions import assertMessageInQueue
import executor

rabbit_host = os.getenv('RABBIT_HOST', '127.0.0.1')
rabbit_user = os.getenv('RABBIT_USER', 'guest')
rabbit_pass = os.getenv('RABBIT_PASS', 'guest')

unavailable_rabbit = not rabbit_util.is_rabbit_available(host=rabbit_host,
                                                         usr=rabbit_user,
                                                         pas=rabbit_pass)

class FakeConnection:
    is_open = False

    def close(*args, **kwargs):
        pass

class FakeChannel:
    _impl = FakeConnection()
    is_open = False

    def close(*args, **kwargs):
        pass

class TestBashTasks(unittest.TestCase):
    def setUp(self):
        pass

    def tearDown(self):
        bashtasks_mod.reset()

    def test_init_returns_bashtasks(self):
        bashtask = bashtasks_mod.init(channel=FakeChannel())
        isBashTask = hasattr(bashtask, 'post_task')
        self.assertTrue(isBashTask)

@unittest.skipIf(unavailable_rabbit, "SKIP integration Tests: rabbitmq NOT available")
class IntegTestPostTask(unittest.TestCase):
    def setUp(self):
        rabbit_util.purge(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)

    def tearDown(self):
        rabbit_util.purge(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
        bashtasks_mod.reset()

    def test_post_task_sends_message(self):
        bashtasks = bashtasks_mod.init(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
        bashtasks.post_task(['ls', '-la'])

        body = assertMessageInQueue(TASK_REQUESTS_POOL, host=rabbit_host,
                                    usr=rabbit_user, pas=rabbit_pass)

    def test_post_task_creates_correct_task_msg(self):
        ls_task = ['ls', '-la']
        bashtasks = bashtasks_mod.init(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
        msg = bashtasks.post_task(ls_task)

        self.assertEqual(msg['command'], ls_task)
        self.assertEqual(msg['reply_to'], TASK_RESPONSES_POOL)
        self.assertTrue('correlation_id' in msg)
        self.assertTrue('request_ts' in msg)

def start_executor_process():
    p = multiprocessing.Process(target=executor.start_executor,
                                args=(rabbit_host, rabbit_user, rabbit_pass))
    p.start()
    time.sleep(0.1)
    return p

def kill_executor_process(p):
    p.terminate()

@unittest.skipIf(unavailable_rabbit, "SKIP integration Tests: rabbitmq NOT available")
class IntegTestExecuteTask(unittest.TestCase):
    def setUp(self):
        rabbit_util.purge(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)

    def tearDown(self):
        rabbit_util.purge(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
        bashtasks_mod.reset()

    def test_execute_task_raises_on_timeout(self):
        ls_task = ['ls', '-la']
        bashtasks = bashtasks_mod.init(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
        with self.assertRaises(Exception):
            response_msg = bashtasks.execute_task(ls_task, timeout=0.3)

    def test_execute_task_returns_correct_response_msg(self):
        try:
            p = start_executor_process()

            bashtasks = bashtasks_mod.init(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
            ls_task = ['ls', '-la']
            response_msg = bashtasks.execute_task(ls_task)

            self.assertTrue('returncode' in response_msg)
            self.assertTrue('request_ts' in response_msg)
            self.assertEqual(response_msg['command'], ls_task)
            self.assertEqual(response_msg['reply_to'], TASK_RESPONSES_POOL)
            self.assertTrue('correlation_id' in response_msg)
        finally:
            kill_executor_process(p)
            time.sleep(0.2)

@unittest.skipIf(unavailable_rabbit, "SKIP integration Tests: rabbitmq NOT available")
class IntegTestTaskResponseSubscriber(unittest.TestCase):
    def setUp(self):
        rabbit_util.purge(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)

    def tearDown(self):
        rabbit_util.purge(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
        bashtasks_mod.reset()

    def test_subscribe(self):
        ls_task = ['ls', '-la']
        response_msg = {}

        def start_subscriber():
            def on_response_received(msg):
                body = json.loads(msg.body.decode('utf-8'))
                response_msg.update(body)
                msg.ack()

            subscriber = bashtasks_mod.init_subscriber(host=rabbit_host,
                                                       usr=rabbit_user, pas=rabbit_pass)
            subscriber.subscribe(on_response_received)

        try:
            # subscribe to responses.
            # prepare executor, and send task.
            # subscribe to responses and check response arrives

            subscriber_th = threading.Thread(target=start_subscriber,
                                             args=(),
                                             name='subscriber_th')
            subscriber_th.daemon = True
            subscriber_th.start()

            p = start_executor_process()

            bashtasks = bashtasks_mod.init(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
            posted_msg = bashtasks.post_task(ls_task)

            sleep(0.5)  # give rabbit and subscriber time to work
            self.assertEqual(ls_task, response_msg['command'])
        except Exception as e:
            print('Got exception in test:', repr(e))
            import traceback
            print(traceback.format_exc())

        finally:
            kill_executor_process(p)
            time.sleep(0.2)

if __name__ == '__main__':
    unittest.main()
<commit_msg>add tests for retries<commit_after>import unittest
import time
import os
from datetime import datetime
import multiprocessing
import threading
from time import sleep
import json

import bashtasks as bashtasks_mod
import bashtasks.rabbit_util as rabbit_util
from bashtasks.constants import TASK_REQUESTS_POOL, TASK_RESPONSES_POOL
from test.pika_assertions import assertMessageInQueue
import executor

rabbit_host = os.getenv('RABBIT_HOST', '127.0.0.1')
rabbit_user = os.getenv('RABBIT_USER', 'guest')
rabbit_pass = os.getenv('RABBIT_PASS', 'guest')

unavailable_rabbit = not rabbit_util.is_rabbit_available(host=rabbit_host,
                                                         usr=rabbit_user,
                                                         pas=rabbit_pass)

class FakeConnection:
    is_open = False

    def close(*args, **kwargs):
        pass

class FakeChannel:
    _impl = FakeConnection()
    is_open = False

    def close(*args, **kwargs):
        pass

class TestBashTasks(unittest.TestCase):
    def setUp(self):
        pass

    def tearDown(self):
        bashtasks_mod.reset()

    def test_init_returns_bashtasks(self):
        bashtask = bashtasks_mod.init(channel=FakeChannel())
        isBashTask = hasattr(bashtask, 'post_task')
        self.assertTrue(isBashTask)

@unittest.skipIf(unavailable_rabbit, "SKIP integration Tests: rabbitmq NOT available")
class IntegTestPostTask(unittest.TestCase):
    def setUp(self):
        rabbit_util.purge(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)

    def tearDown(self):
        rabbit_util.purge(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
        bashtasks_mod.reset()

    def test_post_task_sends_message(self):
        bashtasks = bashtasks_mod.init(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
        bashtasks.post_task(['ls', '-la'])

        body = assertMessageInQueue(TASK_REQUESTS_POOL, host=rabbit_host,
                                    usr=rabbit_user, pas=rabbit_pass)

    def test_post_task_creates_correct_task_msg(self):
        ls_task = ['ls', '-la']
        bashtasks = bashtasks_mod.init(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
        msg = bashtasks.post_task(ls_task)

        self.assertEqual(msg['command'], ls_task)
        self.assertEqual(msg['reply_to'], TASK_RESPONSES_POOL)
        self.assertTrue('correlation_id' in msg)
        self.assertTrue('request_ts' in msg)

def start_executor_process(tasks_nr=1):
    p = multiprocessing.Process(target=executor.start_executor,
                                args=(rabbit_host, rabbit_user, rabbit_pass, tasks_nr))
    p.start()
    time.sleep(0.1)
    return p

def kill_executor_process(p):
    p.terminate()

@unittest.skipIf(unavailable_rabbit, "SKIP integration Tests: rabbitmq NOT available")
class IntegTestExecuteTask(unittest.TestCase):
    def setUp(self):
        rabbit_util.purge(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)

    def tearDown(self):
        rabbit_util.purge(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
        bashtasks_mod.reset()

    def test_execute_task_raises_on_timeout(self):
        ls_task = ['ls', '-la']
        bashtasks = bashtasks_mod.init(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
        with self.assertRaises(Exception):
            response_msg = bashtasks.execute_task(ls_task, timeout=0.3)

    def test_execute_task_returns_correct_response_msg(self):
        try:
            p = start_executor_process()

            bashtasks = bashtasks_mod.init(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
            ls_task = ['ls', '-la']
            response_msg = bashtasks.execute_task(ls_task)

            self.assertTrue('returncode' in response_msg)
            self.assertTrue('request_ts' in response_msg)
            self.assertEqual(response_msg['command'], ls_task)
            self.assertEqual(response_msg['reply_to'], TASK_RESPONSES_POOL)
            self.assertTrue('correlation_id' in response_msg)
        finally:
            kill_executor_process(p)
            time.sleep(0.2)

@unittest.skipIf(unavailable_rabbit, "SKIP integration Tests: rabbitmq NOT available")
class IntegTestTaskResponseSubscriber(unittest.TestCase):
    def setUp(self):
        rabbit_util.purge(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)

    def tearDown(self):
        rabbit_util.purge(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
        bashtasks_mod.reset()

    def test_subscribe(self):
        ls_task = ['ls', '-la']
        response_msg = {}

        def start_subscriber():
            def on_response_received(msg):
                body = json.loads(msg.body.decode('utf-8'))
                response_msg.update(body)
                msg.ack()
            subscriber = bashtasks_mod.init_subscriber(host=rabbit_host,
                                                       usr=rabbit_user, pas=rabbit_pass)
            subscriber.subscribe(on_response_received)

        try:
            # subscribe to responses.
            # prepare executor, and send task.
            # subscribe to responses and check response arrives

            subscriber_th = threading.Thread(target=start_subscriber,
                                             args=(),
                                             name='subscriber_th')
            subscriber_th.daemon = True
            subscriber_th.start()

            p = start_executor_process()

            bashtasks = bashtasks_mod.init(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
            posted_msg = bashtasks.post_task(ls_task)
            start_time = time()
            while 'command' not in response_msg and time() - start_time < 10:  # give rabbit and subscriber time to work
                sleep(0.1)
            self.assertTrue('command' in response_msg)
            self.assertEqual(ls_task, response_msg['command'])
        except Exception as e:
            print('Got exception in test:', repr(e))
            import traceback
            print(traceback.format_exc())
            print('response_msg::::::', repr(response_msg))

        finally:
            kill_executor_process(p)
            time.sleep(0.2)

@unittest.skipIf(unavailable_rabbit, "SKIP integration Tests: rabbitmq NOT available")
class IntegTestRetries(unittest.TestCase):
    def setUp(self):
        rabbit_util.purge(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)

    def tearDown(self):
        rabbit_util.purge(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
        bashtasks_mod.reset()

    def test_execute_task_OK_returns_zero_retries(self):
        try:
            p = start_executor_process()
            max_retries = 5
            bashtasks = bashtasks_mod.init(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
            ls_task = ['ls', '-la']
            response_msg = bashtasks.execute_task(ls_task, max_retries=max_retries)

            self.assertEqual(response_msg['retries'], 0)
            self.assertEqual(response_msg['max_retries'], max_retries)
            self.assertEqual(response_msg['returncode'], 0)
        finally:
            kill_executor_process(p)
            time.sleep(0.2)

    def test_execute_task_KO_returns_max_retries(self):
        try:
            max_retries = 2
            p = start_executor_process(tasks_nr=max_retries+1)
            bashtasks = bashtasks_mod.init(host=rabbit_host, usr=rabbit_user, pas=rabbit_pass)
            ls_task = ['non-existent-command']
            response_msg = bashtasks.execute_task(ls_task, max_retries=max_retries)

            self.assertEqual(response_msg['retries'], max_retries)
            self.assertEqual(response_msg['max_retries'], max_retries)
            self.assertNotEqual(response_msg['returncode'], 0)
        except Exception as e:
            raise e
        finally:
            kill_executor_process(p)
            time.sleep(0.2)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from ..args import VariableArgs

class JteratorInitArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorInitArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        self.merge = self._merge_params['default']
        super(JteratorInitArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline', 'merge'}

    @property
    def merge(self):
        '''
        Returns
        -------
        bool
            indicator whether output data should be merged with data generated
            by previous pipelines

        Warning
        -------
        When ``False`` data of previous pipelines will be deleted.
        '''
        return self._merge

    @merge.setter
    def merge(self, value):
        if not isinstance(value, self._merge_params['type']):
            raise TypeError('Argument "merge" must have type %s'
                            % self._merge_params['type'])
        self._merge = value

    @property
    def _merge_params(self):
        return {
            'default': False,
            'type': bool,
            'help': '''
                merge output with data generated by previous pipelines to
                prevent that data is overwritten
            '''
        }

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorRunArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorRunArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        self.plot = self._plot_params['default']
        super(JteratorRunArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline', 'plot'}

    @property
    def plot(self):
        '''
        Returns
        -------
        bool
            indicator that modules should generate plots (default: ``False``)
        '''
        return self._plot

    @plot.setter
    def plot(self, value):
        if not isinstance(value, self._plot_params['type']):
            raise TypeError('Attribute "plot" must have type %s.'
                            % self._plot_params['type'])
        self._plot = value

    @property
    def _plot_params(self):
        return {
            'type': bool,
            'default': False,
            'help': '''
                activate plotting
            '''

        }

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorSubmitArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorSubmitArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        super(JteratorSubmitArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline'}

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorResubmitArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorResubmitArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        super(JteratorResubmitArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline'}

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorCollectArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorCollectArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        super(JteratorCollectArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline'}

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorLogArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorLogArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        super(JteratorLogArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline'}

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorInfoArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorInfoArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        super(JteratorInfoArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline'}

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorCreateArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorCreateArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        self.repo_dir = self._repo_dir_params['default']
        self.skel_dir = self._skel_dir_params['default']
        super(JteratorCreateArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline', 'repo_dir', 'skel_dir'}

    @property
    def repo_dir(self):
        '''
        Returns
        -------
        str
            path to repository directory where module files are located
        '''
        return self._repo_dir

    @repo_dir.setter
    def repo_dir(self, value):
        if not(isinstance(value, self._repo_dir_params['type']) or
                value is None):
            raise TypeError('Attribute "repo_dir" must have type %s'
                            % self._repo_dir_params['type'])
        self._repo_dir = value

    @property
    def _repo_dir_params(self):
        return {
            'type': str,
            'default': None,
            'help': '''
                path to repository directory where module files are located
            '''
        }

    @property
    def skel_dir(self):
        '''
        Returns
        -------
        str
            path to a directory that represents a project skeleton
        '''
        return self._skel_dir

    @skel_dir.setter
    def skel_dir(self, value):
        if not(isinstance(value, self._skel_dir_params['type']) or
                value is None):
            raise TypeError('Attribute "skel_dir" must have type %s'
                            % self._skel_dir_params['type'])
        self._skel_dir = value

    @property
    def _skel_dir_params(self):
        return {
            'type': str,
            'default': None,
            'help': '''
                path to repository directory where module files are located
            '''
        }

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorCheckArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorCheckArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        super(JteratorCheckArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline'}

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorRemoveArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorRemoveArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        super(JteratorRemoveArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline'}

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }
<commit_msg>Fix bug in parsing of args in jterator<commit_after>from ..args import VariableArgs

class JteratorInitArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorInitArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        self.merge = self._merge_params['default']
        super(JteratorInitArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline', 'merge'}

    @property
    def merge(self):
        '''
        Returns
        -------
        bool
            indicator whether output data should be merged with data generated
            by previous pipelines

        Warning
        -------
        When ``False`` data of previous pipelines will be deleted.
        '''
        return self._merge

    @merge.setter
    def merge(self, value):
        if not isinstance(value, self._merge_params['type']):
            raise TypeError('Argument "merge" must have type %s'
                            % self._merge_params['type'])
        self._merge = value

    @property
    def _merge_params(self):
        return {
            'default': False,
            'type': bool,
            'help': '''
                merge output with data generated by previous pipelines to
                prevent that data is overwritten
            '''
        }

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipeline

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorRunArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorRunArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        self.plot = self._plot_params['default']
        super(JteratorRunArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline', 'plot'}

    @property
    def plot(self):
        '''
        Returns
        -------
        bool
            indicator that modules should generate plots (default: ``False``)
        '''
        return self._plot

    @plot.setter
    def plot(self, value):
        if not isinstance(value, self._plot_params['type']):
            raise TypeError('Attribute "plot" must have type %s.'
                            % self._plot_params['type'])
        self._plot = value

    @property
    def _plot_params(self):
        return {
            'type': bool,
            'default': False,
            'help': '''
                activate plotting
            '''

        }

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorSubmitArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorSubmitArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        super(JteratorSubmitArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline'}

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorResubmitArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorResubmitArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        super(JteratorResubmitArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline'}

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorCollectArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorCollectArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        super(JteratorCollectArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline'}

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorLogArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorLogArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        super(JteratorLogArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline'}

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorInfoArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorInfoArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        super(JteratorInfoArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline'}

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorCreateArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorCreateArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        self.repo_dir = self._repo_dir_params['default']
        self.skel_dir = self._skel_dir_params['default']
        super(JteratorCreateArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline', 'repo_dir', 'skel_dir'}

    @property
    def repo_dir(self):
        '''
        Returns
        -------
        str
            path to repository directory where module files are located
        '''
        return self._repo_dir

    @repo_dir.setter
    def repo_dir(self, value):
        if not(isinstance(value, self._repo_dir_params['type']) or
                value is None):
            raise TypeError('Attribute "repo_dir" must have type %s'
                            % self._repo_dir_params['type'])
        self._repo_dir = value

    @property
    def _repo_dir_params(self):
        return {
            'type': str,
            'default': None,
            'help': '''
                path to repository directory where module files are located
            '''
        }

    @property
    def skel_dir(self):
        '''
        Returns
        -------
        str
            path to a directory that represents a project skeleton
        '''
        return self._skel_dir

    @skel_dir.setter
    def skel_dir(self, value):
        if not(isinstance(value, self._skel_dir_params['type']) or
                value is None):
            raise TypeError('Attribute "skel_dir" must have type %s'
                            % self._skel_dir_params['type'])
        self._skel_dir = value

    @property
    def _skel_dir_params(self):
        return {
            'type': str,
            'default': None,
            'help': '''
                path to repository directory where module files are located
            '''
        }

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorCheckArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorCheckArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        super(JteratorCheckArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline'}

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }

class JteratorRemoveArgs(VariableArgs):

    def __init__(self, **kwargs):
        '''
        Initialize an instance of class JteratorRemoveArgs.

        Parameters
        ----------
        **kwargs: dict
            arguments as key-value pairs
        '''
        super(JteratorRemoveArgs, self).__init__(**kwargs)

    @property
    def _required_args(self):
        return {'pipeline'}

    @property
    def _persistent_attrs(self):
        return {'pipeline'}

    @property
    def pipeline(self):
        '''
        Returns
        -------
        str
            name of the pipeline that should be processed
        '''
        return self._pipelines

    @pipeline.setter
    def pipeline(self, value):
        if not isinstance(value, self._pipeline_params['type']):
            raise TypeError('Argument "pipeline" must have type %s'
                            % self._pipeline_params['type'])
        self._pipeline = value

    @property
    def _pipeline_params(self):
        return {
            'type': str,
            'help': '''
                name of the pipeline that should be processed
            '''
        }
<|endoftext|>


<commit_before># Copyright (C) 2015-2016 Regents of the University of California
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Kills any running jobs trees in a rogue toil.
"""
from __future__ import absolute_import
import logging
import os
import signal

from toil.lib.bioio import getBasicOptionParser
from toil.lib.bioio import parseBasicOptions
from toil.common import Toil, jobStoreLocatorHelp, Config
from toil.version import version

logger = logging.getLogger( __name__ )

def main():
    parser = getBasicOptionParser()

    parser.add_argument("jobStore", type=str,
                        help="The location of the job store used by the workflow whose jobs should "
                             "be killed." + jobStoreLocatorHelp)
    parser.add_argument("--version", action='version', version=version)
    options = parseBasicOptions(parser)
    config = Config()
    config.setOptions(options)

    #
    config.jobStore = config.jobStore[5:] if config.jobStore.startswith('file:') else config.jobStore

    # ':' means an aws/google/azure jobstore; use the old (broken?) method
    if ':' in config.jobStore:
        jobStore = Toil.resumeJobStore(config.jobStore)
        logger.info("Starting routine to kill running jobs in the toil workflow: %s", config.jobStore)
        # TODO: This behaviour is now broken src: https://github.com/DataBiosphere/toil/commit/a3d65fc8925712221e4cda116d1825d4a1e963a1
        batchSystem = Toil.createBatchSystem(jobStore.config)  # This should automatically kill the existing jobs.. so we're good.
        for jobID in batchSystem.getIssuedBatchJobIDs():  # Just in case we do it again.
            batchSystem.killBatchJobs(jobID)
        logger.info("All jobs SHOULD have been killed")
    # otherwise, kill the pid recorded in the jobstore
    else:
        pid_log = os.path.join(os.path.abspath(config.jobStore), 'pid.log')
        with open(pid_log, 'r') as f:
            pid2kill = f.read().strip()
        try:
            os.kill(int(pid2kill), signal.SIGKILL)
            logger.info("Toil process: %s successfully terminated." % str(pid2kill))
        except OSError:
            logger.error("Toil process: %s could not be terminated." % str(pid2kill))
            raise
<commit_msg>Log message.<commit_after># Copyright (C) 2015-2016 Regents of the University of California
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Kills any running jobs trees in a rogue toil.
"""
from __future__ import absolute_import
import logging
import os
import signal

from toil.lib.bioio import getBasicOptionParser
from toil.lib.bioio import parseBasicOptions
from toil.common import Toil, jobStoreLocatorHelp, Config
from toil.version import version

logger = logging.getLogger( __name__ )

def main():
    parser = getBasicOptionParser()

    parser.add_argument("jobStore", type=str,
                        help="The location of the job store used by the workflow whose jobs should "
                             "be killed." + jobStoreLocatorHelp)
    parser.add_argument("--version", action='version', version=version)
    options = parseBasicOptions(parser)
    config = Config()
    config.setOptions(options)

    #
    config.jobStore = config.jobStore[5:] if config.jobStore.startswith('file:') else config.jobStore

    # ':' means an aws/google/azure jobstore; use the old (broken?) method
    if ':' in config.jobStore:
        jobStore = Toil.resumeJobStore(config.jobStore)
        logger.info("Starting routine to kill running jobs in the toil workflow: %s", config.jobStore)
        # TODO: This behaviour is now broken src: https://github.com/DataBiosphere/toil/commit/a3d65fc8925712221e4cda116d1825d4a1e963a1
        batchSystem = Toil.createBatchSystem(jobStore.config)  # Should automatically kill existing jobs, so we're good.
        for jobID in batchSystem.getIssuedBatchJobIDs():  # Just in case we do it again.
            batchSystem.killBatchJobs(jobID)
        logger.info("All jobs SHOULD have been killed")
    # otherwise, kill the pid recorded in the jobstore
    else:
        pid_log = os.path.join(os.path.abspath(config.jobStore), 'pid.log')
        with open(pid_log, 'r') as f:
            pid2kill = f.read().strip()
        try:
            os.kill(int(pid2kill), signal.SIGKILL)
            logger.info("Toil process %s successfully terminated." % str(pid2kill))
        except OSError:
            logger.error("Toil process %s could not be terminated." % str(pid2kill))
            raise
<|endoftext|>


<commit_before># http://djangosnippets.org/snippets/1289/

from django import template
from django.template import loader, Node, Variable
from django.utils.encoding import smart_str, smart_unicode
from django.template.defaulttags import url
from django.template import VariableDoesNotExist

register = template.Library()

@register.tag
def breadcrumb(parser, token):
    """
    Renders the breadcrumb.
    Examples:
        {% breadcrumb "Title of breadcrumb" url_var %}
        {% breadcrumb context_var  url_var %}
        {% breadcrumb "Just the title" %}
        {% breadcrumb just_context_var %}

    Parameters:
    -First parameter is the title of the crumb,
    -Second (optional) parameter is the url variable to link to, produced by url tag, i.e.:
        {% url person_detail object.id as person_url %}
        then:
        {% breadcrumb person.name person_url %}

    @author Andriy Drozdyuk
    """
    bits = token.split_contents()
    # Insert empty CSS class.
    bits.insert(1, '""')
    token.contents = ' '.join(bits)
    bits = token.split_contents()[1:]
    return BreadcrumbNode(bits)

@register.tag
def breadcrumb_css(parser, token):
    """Same as breadcrumb, except specify a CSS class in front."""
    bits = token.split_contents()[1:]
    return BreadcrumbNode(bits)

@register.tag
def breadcrumb_url(parser, token, css_class='""'):
    """
    Same as breadcrumb
    but instead of url context variable takes in all the
    arguments URL tag takes.
        {% breadcrumb "Title of breadcrumb" person_detail person.id %}
        {% breadcrumb person.name person_detail person.id %}
    """

    bits = token.split_contents()
    if len(bits)==2:
        return breadcrumb(parser, token)

    # Extract our extra title parameter
    title = bits.pop(1)
    token.contents = ' '.join(bits)
    # Shove it back into to the token to url parse it.
    url_node = url(parser, token)

    return UrlBreadcrumbNode(css_class, title, url_node)

@register.tag
def breadcrumb_url_css(parser, token):
    """Same as breadcrumb_url, except specify a CSS class in front."""
    bits = token.split_contents()
    css_class = bits.pop(1)
    print 'css_class', css_class, 'bits', bits
    token.contents = ' '.join(bits)
    return breadcrumb_url(parser, token, css_class)

class BreadcrumbNode(Node):
    def __init__(self, vars):
        self.vars = map(Variable,vars)
        # 0 -> css class
        # 1 -> title
        # 2 -> url

    def render(self, context):
        css_class = self.vars[0].var
        if css_class.find("'") == -1 and css_class.find('"') == -1:
            try:
                val = self.vars[0]
                css_class = val.resolve(context)
            except:
                css_class = ''
        else:
            css_class = css_class.strip("'").strip('"')
            css_class = smart_unicode(css_class)
        #
        title = self.vars[1].var
        if title.find("'") == -1 and title.find('"') == -1:
            try:
                val = self.vars[1]
                title = val.resolve(context)
            except:
                title = ''
        else:
            title = title.strip("'").strip('"')
            title = smart_unicode(title)
        #
        url = None
        if len(self.vars)>2:
            val = self.vars[2]
            try:
                url = val.resolve(context)
            except VariableDoesNotExist:
                print 'URL does not exist', val
                url = None
        #
        return create_crumb(css_class, title, url)

class UrlBreadcrumbNode(Node):
    def __init__(self, css_class, title, url_node):
        self.css_class = Variable(css_class)
        self.title = Variable(title)
        self.url_node = url_node

    def render(self, context):
        title = self.title.var
        if title.find("'") == -1 and title.find('"') == -1:
            try:
                val = self.title
                title = val.resolve(context)
            except:
                title = ''
        else:
            title = title.strip("'").strip('"')
            title = smart_unicode(title)
        #
        css_class = self.css_class.var
        if css_class.find("'") == -1 and css_class.find('"') == -1:
            try:
                val = self.css_class
                css_class = val.resolve(context)
            except:
                css_class = ''
        else:
            css_class = css_class.strip("'").strip('"')
            css_class = smart_unicode(css_class)
        #
        url = self.url_node.render(context)
        return create_crumb(css_class, title, url)

def create_crumb(css_class, title, url):
    """
    Helper function
    """
    if url:
        crumb = "
{title}
".format(**locals())
    else:
        crumb = "
{title}
".format(**locals())

    return crumb
<commit_msg>remove print<commit_after># http://djangosnippets.org/snippets/1289/

from django import template
from django.template import loader, Node, Variable
from django.utils.encoding import smart_str, smart_unicode
from django.template.defaulttags import url
from django.template import VariableDoesNotExist

register = template.Library()

@register.tag
def breadcrumb(parser, token):
    """
    Renders the breadcrumb.
    Examples:
        {% breadcrumb "Title of breadcrumb" url_var %}
        {% breadcrumb context_var  url_var %}
        {% breadcrumb "Just the title" %}
        {% breadcrumb just_context_var %}

    Parameters:
    -First parameter is the title of the crumb,
    -Second (optional) parameter is the url variable to link to, produced by url tag, i.e.:
        {% url person_detail object.id as person_url %}
        then:
        {% breadcrumb person.name person_url %}

    @author Andriy Drozdyuk
    """
    bits = token.split_contents()
    # Insert empty CSS class.
    bits.insert(1, '""')
    token.contents = ' '.join(bits)
    bits = token.split_contents()[1:]
    return BreadcrumbNode(bits)

@register.tag
def breadcrumb_css(parser, token):
    """Same as breadcrumb, except specify a CSS class in front."""
    bits = token.split_contents()[1:]
    return BreadcrumbNode(bits)

@register.tag
def breadcrumb_url(parser, token, css_class='""'):
    """
    Same as breadcrumb
    but instead of url context variable takes in all the
    arguments URL tag takes.
        {% breadcrumb "Title of breadcrumb" person_detail person.id %}
        {% breadcrumb person.name person_detail person.id %}
    """

    bits = token.split_contents()
    if len(bits)==2:
        return breadcrumb(parser, token)

    # Extract our extra title parameter
    title = bits.pop(1)
    token.contents = ' '.join(bits)
    # Shove it back into to the token to url parse it.
    url_node = url(parser, token)

    return UrlBreadcrumbNode(css_class, title, url_node)

@register.tag
def breadcrumb_url_css(parser, token):
    """Same as breadcrumb_url, except specify a CSS class in front."""
    bits = token.split_contents()
    css_class = bits.pop(1)
    token.contents = ' '.join(bits)
    return breadcrumb_url(parser, token, css_class)

class BreadcrumbNode(Node):
    def __init__(self, vars):
        self.vars = map(Variable,vars)
        # 0 -> css class
        # 1 -> title
        # 2 -> url

    def render(self, context):
        css_class = self.vars[0].var
        if css_class.find("'") == -1 and css_class.find('"') == -1:
            try:
                val = self.vars[0]
                css_class = val.resolve(context)
            except:
                css_class = ''
        else:
            css_class = css_class.strip("'").strip('"')
            css_class = smart_unicode(css_class)
        #
        title = self.vars[1].var
        if title.find("'") == -1 and title.find('"') == -1:
            try:
                val = self.vars[1]
                title = val.resolve(context)
            except:
                title = ''
        else:
            title = title.strip("'").strip('"')
            title = smart_unicode(title)
        #
        url = None
        if len(self.vars)>2:
            val = self.vars[2]
            try:
                url = val.resolve(context)
            except VariableDoesNotExist:
                print 'URL does not exist', val
                url = None
        #
        return create_crumb(css_class, title, url)

class UrlBreadcrumbNode(Node):
    def __init__(self, css_class, title, url_node):
        self.css_class = Variable(css_class)
        self.title = Variable(title)
        self.url_node = url_node

    def render(self, context):
        title = self.title.var
        if title.find("'") == -1 and title.find('"') == -1:
            try:
                val = self.title
                title = val.resolve(context)
            except:
                title = ''
        else:
            title = title.strip("'").strip('"')
            title = smart_unicode(title)
        #
        css_class = self.css_class.var
        if css_class.find("'") == -1 and css_class.find('"') == -1:
            try:
                val = self.css_class
                css_class = val.resolve(context)
            except:
                css_class = ''
        else:
            css_class = css_class.strip("'").strip('"')
            css_class = smart_unicode(css_class)
        #
        url = self.url_node.render(context)
        return create_crumb(css_class, title, url)

def create_crumb(css_class, title, url):
    """
    Helper function
    """
    if url:
        crumb = "
{title}
".format(**locals())
    else:
        crumb = "
{title}
".format(**locals())

    return crumb
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2015-2016 MIT Probabilistic Computing Project

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#    http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
from scipy.stats import multivariate_normal, norm

from gpmcc.utils.xy_gpm import synthetic

class LinearGpm(synthetic.SyntheticXyGpm):
    """Y = X + N(0,noise)."""

    def simulate_xy(self, size=None):
        return self.rng.multivariate_normal(
            [0,0], cov=[[1,1-self.noise],[1-self.noise,1]], size=size)

    def logpdf_xy(self, x, y):
        return multivariate_normal.logpdf([x,y], [0,0],
            cov=[[1,1-self.noise],[1-self.noise,1]])

    def logpdf_x(self, x):
        return norm.logpdf(x, scale=1)

    def logpdf_y(self, y):
        return norm.logpdf(y, scale=1)

    def logpdf_x_given_y(self, x, y):
        raise NotImplementedError

    def logpdf_y_given_x(self, y, x):
        raise NotImplementedError

    def mutual_information(self):
        cov = 1-self.noise
        return .5 * np.log(1/np.abs(1-cov**2))
<commit_msg>Use simpler equation for bivariate mi.<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2015-2016 MIT Probabilistic Computing Project

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#    http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
from scipy.stats import multivariate_normal, norm

from gpmcc.utils.xy_gpm import synthetic

class LinearGpm(synthetic.SyntheticXyGpm):
    """Y = X + N(0,noise)."""

    def simulate_xy(self, size=None):
        return self.rng.multivariate_normal(
            [0,0], cov=[[1,1-self.noise],[1-self.noise,1]], size=size)

    def logpdf_xy(self, x, y):
        return multivariate_normal.logpdf([x,y], [0,0],
            cov=[[1,1-self.noise],[1-self.noise,1]])

    def logpdf_x(self, x):
        return norm.logpdf(x, scale=1)

    def logpdf_y(self, y):
        return norm.logpdf(y, scale=1)

    def logpdf_x_given_y(self, x, y):
        raise NotImplementedError

    def logpdf_y_given_x(self, y, x):
        raise NotImplementedError

    def mutual_information(self):
        cov = 1-self.noise
        return -.5 * np.log(1-cov**2)
<|endoftext|>