<commit_before>import unittest
import astropy.io.fits as fits
import math
import tempfile
from tkp.utility.fits import fix_reference_dec

class TestFixReferenceDec(unittest.TestCase):
    def test_dec_90(self):
        # Default unit is degrees.
        self._test_for_reference_dec(90.0)

    def test_dec_minus90(self):
        # Default unit is degrees.
        self._test_for_reference_dec(-90.0)

    def test_dec_90_deg(self):
        self._test_for_reference_dec(90.0, "deg")

    def test_dec_pi_by_2_rad(self):
        self._test_for_reference_dec(math.pi/2, "rad")

    def _test_for_reference_dec(self, refdec, unit=None):
        with tempfile.NamedTemporaryFile() as temp_fits:
            h = fits.PrimaryHDU()
            h.header["CRVAL2"] = refdec
            if unit:
                h.header.update("CUNIT2", unit)
            h.writeto(temp_fits.name)
            fix_reference_dec(temp_fits.name)
            self.assertLess(abs(fits.getheader(temp_fits.name)['CRVAL2']), abs(refdec))
<commit_msg>fix fits test #547 (#551)<commit_after>import unittest
import astropy.io.fits as fits
import math
import tempfile
from tkp.utility.fits import fix_reference_dec

class TestFixReferenceDec(unittest.TestCase):
    def test_dec_90(self):
        # Default unit is degrees.
        self._test_for_reference_dec(90.0)

    def test_dec_minus90(self):
        # Default unit is degrees.
        self._test_for_reference_dec(-90.0)

    def test_dec_90_deg(self):
        self._test_for_reference_dec(90.0, "deg")

    def test_dec_pi_by_2_rad(self):
        self._test_for_reference_dec(math.pi/2, "rad")

    def _test_for_reference_dec(self, refdec, unit=None):
        with tempfile.NamedTemporaryFile() as temp_fits:
            h = fits.PrimaryHDU()
            h.header["CRVAL2"] = refdec
            if unit:
                h.header["CUNIT2"] = unit
            h.writeto(temp_fits.name)
            fix_reference_dec(temp_fits.name)
            self.assertLess(abs(fits.getheader(temp_fits.name)['CRVAL2']), abs(refdec))
<|endoftext|>


<commit_before>"""
Tests of neo.io.asciisignalio
"""

import os
import unittest
import json
import csv
import numpy as np
import quantities as pq
from numpy.testing import assert_array_almost_equal, assert_array_equal
from neo.io import AsciiSignalIO
from neo.test.iotest.common_io_test import BaseTestIO
from neo.core import AnalogSignal, Segment, Block

class TestAsciiSignalIOWithTestFiles(BaseTestIO, unittest.TestCase):
    ioclass = AsciiSignalIO
    entities_to_download = [
        'asciisignal'
    ]
    entities_to_test = [
        'asciisignal/File_asciisignal_2.txt',
        'asciisignal/File_asciisignal_3.txt',
    ]

class TestAsciiSignalIO(unittest.TestCase):

    def test_genfromtxt_expect_success(self):
        sample_data = np.random.uniform(size=(200, 3))
        filename = "test_genfromtxt_expect_success.txt"
        np.savetxt(filename, sample_data, delimiter=' ')
        sampling_rate = 1 * pq.kHz
        io = AsciiSignalIO(filename, sampling_rate=sampling_rate, delimiter=' ',
                           units='mV', method='genfromtxt')
        block = io.read_block()

        signal1 = block.segments[0].analogsignals[1]
        assert_array_almost_equal(signal1.reshape(-1).magnitude, sample_data[:, 1],
                                  decimal=6)
        self.assertEqual(len(block.segments[0].analogsignals), 3)
        self.assertEqual(signal1.t_stop, sample_data.shape[0] / sampling_rate)
        self.assertEqual(signal1.units, pq.mV)

        os.remove(filename)

    # test_genfromtxt_expect_failure

    def test_csv_expect_success(self):
        filename = 'test_csv_expect_success.csv'
        sample_data = [
            (-65, -65, -65, 0.5),
            (-64.8, -64.5, -64.0, 0.6),
            (-64.6, -64.2, -77.0, 0.7),
            (-64.3, -64.0, -99.9, 0.8)
        ]
        with open(filename, 'w') as csvfile:
            writer = csv.writer(csvfile, delimiter=',',
                                quotechar='|', quoting=csv.QUOTE_MINIMAL)
            for row in sample_data:
                writer.writerow(row)

        io = AsciiSignalIO(filename, usecols=(0, 1, 3), timecolumn=2,
                           # note that timecolumn applies to the remaining columns
                           # after applying usecols
                           time_units="ms", delimiter=',', units="mV", method='csv',
                           signal_group_mode='all-in-one', t_start=0.5)

        block = io.read_block()
        signal = block.segments[0].analogsignals[0]
        self.assertEqual(signal.shape, (4, 2))  # two columns remaining after usecols
                                                # and timecolumn applied
        assert_array_almost_equal(signal[:, 1].reshape(-1).magnitude,
                                  np.array(sample_data)[:, 1],
                                  decimal=5)
        self.assertAlmostEqual(signal.sampling_period, 0.1 * pq.ms)

        os.remove(filename)
    # test_csv_expect_failure
    # test_homemade_expect_success

    def test_homemade_expect_success(self):
        filename = 'test_homemade_expect_success.txt'
        sample_data = [
            (-65, -65, -65, 0.5),
            (-64.8, -64.5, -64.0, 0.6),
            (-64.6, -64.2, -77.0, 0.7),
            (-64.3, -64.0, -99.9, 0.8)
        ]
        with open(filename, 'w') as datafile:
            datafile.write("# a comment\n")
            for row in sample_data:
                datafile.write("\t ".join(map(str, row)) + "\t\n")

        io = AsciiSignalIO(filename, usecols=(0, 1, 3), timecolumn=2, skiprows=1,
                           time_units="ms", delimiter='\t', units="mV", method='homemade',
                           signal_group_mode='all-in-one', t_start=0.5)

        block = io.read_block()
        signal = block.segments[0].analogsignals[0]
        self.assertEqual(signal.shape, (4, 2))  # two columns remaining after usecols
                                                # and timecolumn applied
        assert_array_almost_equal(signal[:, 1].reshape(-1).magnitude,
                                  np.array(sample_data)[:, 1],
                                  decimal=5)
        self.assertAlmostEqual(signal.sampling_period, 0.1 * pq.ms)

        os.remove(filename)

    # test_homemade_expect_failure

    def test_callable_expect_success(self):
        sample_data = np.random.uniform(size=(200, 3))
        filename = "test_genfromtxt_expect_success.txt"
        np.savetxt(filename, sample_data, delimiter=' ')
        sampling_rate = 1 * pq.kHz

        def reader(filename, comment_rows):
            return np.genfromtxt(filename, delimiter=' ', usecols=None,
                                 skip_header=comment_rows or 0, dtype='f')

        io = AsciiSignalIO(filename, sampling_rate=sampling_rate, delimiter=' ',
                           units='mV', method=reader)
        block = io.read_block()

        signal1 = block.segments[0].analogsignals[1]
        assert_array_almost_equal(signal1.reshape(-1).magnitude, sample_data[:, 1],
                                  decimal=6)
        self.assertEqual(len(block.segments[0].analogsignals), 3)
        self.assertEqual(signal1.t_stop, sample_data.shape[0] / sampling_rate)
        self.assertEqual(signal1.units, pq.mV)

        os.remove(filename)

    # test_skiprows
    # test_usecols

    def test_timecolumn(self):
        sample_data = np.random.uniform(size=(200, 3))
        sampling_period = 0.5
        time_data = sampling_period * np.arange(sample_data.shape[0])
        combined_data = np.hstack((sample_data, time_data[:, np.newaxis]))
        filename = "test_multichannel.txt"
        np.savetxt(filename, combined_data, delimiter=' ')
        io = AsciiSignalIO(filename, delimiter=' ',
                           units='mV', method='genfromtxt', timecolumn=-1,
                           time_units='ms', signal_group_mode='split-all')
        block = io.read_block()

        signal1 = block.segments[0].analogsignals[1]
        assert_array_almost_equal(signal1.reshape(-1).magnitude, sample_data[:, 1],
                                  decimal=6)
        self.assertEqual(signal1.sampling_period, sampling_period * pq.ms)
        self.assertEqual(len(block.segments[0].analogsignals), 3)
        self.assertEqual(signal1.t_stop, sample_data.shape[0] * sampling_period * pq.ms)
        self.assertEqual(signal1.units, pq.mV)

        os.remove(filename)

    def test_multichannel(self):
        sample_data = np.random.uniform(size=(200, 3))
        filename = "test_multichannel.txt"
        np.savetxt(filename, sample_data, delimiter=' ')
        sampling_rate = 1 * pq.kHz
        io = AsciiSignalIO(filename, sampling_rate=sampling_rate, delimiter=' ',
                           units='mV', method='genfromtxt',
                           signal_group_mode='all-in-one')
        block = io.read_block()

        signal = block.segments[0].analogsignals[0]
        assert_array_almost_equal(signal.magnitude, sample_data,
                                  decimal=6)
        self.assertEqual(len(block.segments[0].analogsignals), 1)
        self.assertEqual(signal.t_stop, sample_data.shape[0] / sampling_rate)
        self.assertEqual(signal.units, pq.mV)

        os.remove(filename)

    def test_multichannel_with_timecolumn(self):
        sample_data = np.random.uniform(size=(200, 3))
        sampling_period = 0.5
        time_data = sampling_period * np.arange(sample_data.shape[0])
        combined_data = np.hstack((time_data[:, np.newaxis], sample_data))
        filename = "test_multichannel.txt"
        np.savetxt(filename, combined_data, delimiter=' ')
        io = AsciiSignalIO(filename, delimiter=' ',
                           units='mV', method='genfromtxt', timecolumn=0,
                           time_units='ms',
                           signal_group_mode='all-in-one')
        block = io.read_block()

        signal = block.segments[0].analogsignals[0]
        assert_array_almost_equal(signal.magnitude, sample_data,
                                  decimal=6)
        self.assertEqual(signal.sampling_period, sampling_period * pq.ms)
        self.assertEqual(len(block.segments[0].analogsignals), 1)
        self.assertEqual(signal.t_stop, sample_data.shape[0] * sampling_period * pq.ms)
        self.assertEqual(signal.units, pq.mV)

        os.remove(filename)

    def test_multichannel_with_negative_timecolumn(self):
        sample_data = np.random.uniform(size=(200, 3))
        sampling_period = 0.5
        time_data = sampling_period * np.arange(sample_data.shape[0])
        combined_data = np.hstack((sample_data, time_data[:, np.newaxis]))
        filename = "test_multichannel.txt"
        np.savetxt(filename, combined_data, delimiter=' ')
        io = AsciiSignalIO(filename, delimiter=' ',
                           units='mV', method='genfromtxt', timecolumn=-1,
                           time_units='ms',
                           signal_group_mode='all-in-one')
        block = io.read_block()

        signal = block.segments[0].analogsignals[0]
        assert_array_almost_equal(signal.magnitude, sample_data,
                                  decimal=6)
        self.assertEqual(signal.sampling_period, sampling_period * pq.ms)
        self.assertEqual(len(block.segments[0].analogsignals), 1)
        self.assertEqual(signal.t_stop, sample_data.shape[0] * sampling_period * pq.ms)
        self.assertEqual(signal.units, pq.mV)

        os.remove(filename)

    # test write without timecolumn
    # test write with timecolumn
    # test write with units/timeunits different from those of signal

    def test_read_with_json_metadata(self):
        sample_data = np.random.uniform(size=(200, 3))
        filename = "test_read_with_json_metadata.txt"
        metadata_filename = "test_read_with_json_metadata_about.json"
        np.savetxt(filename, sample_data, delimiter=' ')

        metadata = {
            "filename": filename,
            "delimiter": " ",
            "timecolumn": None,
            "units": "mV",
            "time_units": "ms",
            "sampling_rate": {
                "value": 1.0,
                "units": "kHz"
            },
            "method": "genfromtxt",
            "signal_group_mode": 'split-all'
        }
        with open(metadata_filename, "w") as fp:
            json.dump(metadata, fp)

        expected_sampling_rate = 1 * pq.kHz
        io = AsciiSignalIO(filename)
        block = io.read_block()

        signal1 = block.segments[0].analogsignals[1]
        assert_array_almost_equal(signal1.reshape(-1).magnitude, sample_data[:, 1],
                                  decimal=6)
        self.assertEqual(len(block.segments[0].analogsignals), 3)
        self.assertEqual(signal1.sampling_rate, expected_sampling_rate)
        self.assertEqual(signal1.t_stop, sample_data.shape[0] / signal1.sampling_rate)
        self.assertEqual(signal1.units, pq.mV)

        os.remove(filename)
        os.remove(metadata_filename)

    def test_roundtrip_with_json_metadata(self):
        sample_data = np.random.uniform(size=(200, 3))
        filename = "test_roundtrip_with_json_metadata.txt"
        metadata_filename = "test_roundtrip_with_json_metadata_about.json"
        signal1 = AnalogSignal(sample_data, units="pA", sampling_rate=2 * pq.kHz)
        seg1 = Segment()
        block1 = Block()
        seg1.analogsignals.append(signal1)
        seg1.block = block1
        block1.segments.append(seg1)

        iow = AsciiSignalIO(filename, metadata_filename=metadata_filename)
        iow.write_block(block1)
        self.assertTrue(os.path.exists(metadata_filename))

        ior = AsciiSignalIO(filename)
        block2 = ior.read_block()
        assert len(block2.segments[0].analogsignals) == 3
        signal2 = block2.segments[0].analogsignals[1]

        assert_array_almost_equal(signal1.magnitude[:, 1], signal2.magnitude.reshape(-1),
                                  decimal=7)
        self.assertEqual(signal1.units, signal2.units)
        self.assertEqual(signal1.sampling_rate, signal2.sampling_rate)
        assert_array_equal(signal1.times, signal2.times)

        os.remove(filename)
        os.remove(metadata_filename)

    def test_genfromtxt_irregular_expect_success(self):
        sample_data = np.random.uniform(size=(200, 3))
        sample_data[:, 0] = np.sort(sample_data[:, 0])  # make column 0 the time column
        filename = "test_genfromtxt_irregular_expect_success.txt"
        np.savetxt(filename, sample_data, delimiter=' ')

        io = AsciiSignalIO(filename, delimiter=' ', timecolumn=0,
                           units='mV', method='genfromtxt', signal_group_mode='split-all')
        block = io.read_block()

        signal1 = block.segments[0].irregularlysampledsignals[1]
        assert_array_almost_equal(signal1.reshape(-1).magnitude, sample_data[:, 2],
                                  decimal=6)
        self.assertEqual(len(block.segments[0].analogsignals), 0)
        self.assertEqual(len(block.segments[0].irregularlysampledsignals), 2)
        self.assertEqual(signal1.units, pq.mV)

        os.remove(filename)

    def test_irregular_multichannel(self):
        sample_data = np.random.uniform(size=(200, 3))
        sample_data[:, 0] = np.sort(sample_data[:, 0])  # make column 0 the time column
        filename = "test_irregular_multichannel.txt"
        np.savetxt(filename, sample_data, delimiter=' ')

        io = AsciiSignalIO(filename, delimiter=' ', timecolumn=0,
                           units='mV', method='genfromtxt', signal_group_mode='all-in-one')
        block = io.read_block()

        signal = block.segments[0].irregularlysampledsignals[0]
        assert_array_almost_equal(signal.magnitude, sample_data[:, 1:3],
                                  decimal=6)
        self.assertEqual(len(block.segments[0].analogsignals), 0)
        self.assertEqual(len(block.segments[0].irregularlysampledsignals), 1)
        self.assertEqual(signal.shape, (200, 2))
        self.assertEqual(signal.units, pq.mV)

        os.remove(filename)

if __name__ == "__main__":
    unittest.main()
<commit_msg>Update test_asciisignalio.py<commit_after>"""
Tests of neo.io.asciisignalio
"""

import os
import unittest
import json
import csv
import numpy as np
import quantities as pq
from numpy.testing import assert_array_almost_equal, assert_array_equal
from neo.io import AsciiSignalIO
from neo.test.iotest.common_io_test import BaseTestIO
from neo.core import AnalogSignal, Segment, Block

class TestAsciiSignalIOWithTestFiles(BaseTestIO, unittest.TestCase):
    ioclass = AsciiSignalIO
    entities_to_download = [
        'asciisignal'
    ]
    entities_to_test = [
        'asciisignal/File_asciisignal_2.txt',
        'asciisignal/File_asciisignal_3.txt',
    ]

class TestAsciiSignalIO(unittest.TestCase):

    def test_genfromtxt_expect_success(self):
        sample_data = np.random.uniform(size=(200, 3))
        filename = "test_genfromtxt_expect_success.txt"
        np.savetxt(filename, sample_data, delimiter=' ')
        sampling_rate = 1 * pq.kHz
        io = AsciiSignalIO(filename, sampling_rate=sampling_rate, delimiter=' ',
                           units='mV', method='genfromtxt')
        block = io.read_block()

        signal1 = block.segments[0].analogsignals[1]
        assert_array_almost_equal(signal1.reshape(-1).magnitude, sample_data[:, 1],
                                  decimal=6)
        self.assertEqual(len(block.segments[0].analogsignals), 3)
        self.assertEqual(signal1.t_stop, sample_data.shape[0] / sampling_rate)
        self.assertEqual(signal1.units, pq.mV)

        os.remove(filename)

    # test_genfromtxt_expect_failure

    def test_csv_expect_success(self):
        filename = 'test_csv_expect_success.csv'
        sample_data = [
            (-65, -65, -65, 0.5),
            (-64.8, -64.5, -64.0, 0.6),
            (-64.6, -64.2, -77.0, 0.7),
            (-64.3, -64.0, -99.9, 0.8)
        ]
        with open(filename, 'w') as csvfile:
            writer = csv.writer(csvfile, delimiter=',',
                                quotechar='|', quoting=csv.QUOTE_MINIMAL)
            for row in sample_data:
                writer.writerow(row)

        io = AsciiSignalIO(filename, usecols=(0, 1, 3), timecolumn=2,
                           # note that timecolumn applies to the remaining columns
                           # after applying usecols
                           time_units="ms", delimiter=',', units="mV", method='csv',
                           signal_group_mode='all-in-one', t_start=0.5)

        block = io.read_block()
        signal = block.segments[0].analogsignals[0]
        self.assertEqual(signal.shape, (4, 2))  # two columns remaining after usecols
                                                # and timecolumn applied
        assert_array_almost_equal(signal[:, 1].reshape(-1).magnitude,
                                  np.array(sample_data)[:, 1],
                                  decimal=5)
        self.assertAlmostEqual(signal.sampling_period, 0.1 * pq.ms)

        os.remove(filename)
    # test_csv_expect_failure
    # test_homemade_expect_success

    def test_homemade_expect_success(self):
        filename = 'test_homemade_expect_success.txt'
        sample_data = [
            (-65, -65, -65, 0.5),
            (-64.8, -64.5, -64.0, 0.6),
            (-64.6, -64.2, -77.0, 0.7),
            (-64.3, -64.0, -99.9, 0.8)
        ]
        with open(filename, 'w') as datafile:
            datafile.write("# a comment\n")
            for row in sample_data:
                datafile.write("\t ".join(map(str, row)) + "\t\n")

        io = AsciiSignalIO(filename, usecols=(0, 1, 3), timecolumn=2, skiprows=1,
                           time_units="ms", delimiter='\t', units="mV", method='homemade',
                           signal_group_mode='all-in-one', t_start=0.5)

        block = io.read_block()
        signal = block.segments[0].analogsignals[0]
        self.assertEqual(signal.shape, (4, 2))  # two columns remaining after usecols
                                                # and timecolumn applied
        assert_array_almost_equal(signal[:, 1].reshape(-1).magnitude,
                                  np.array(sample_data)[:, 1],
                                  decimal=5)
        self.assertAlmostEqual(signal.sampling_period, 0.1 * pq.ms)

        os.remove(filename)

    # test_homemade_expect_failure

    def test_callable_expect_success(self):
        sample_data = np.random.uniform(size=(200, 3))
        filename = "test_genfromtxt_expect_success.txt"
        np.savetxt(filename, sample_data, delimiter=' ')
        sampling_rate = 1 * pq.kHz

        def reader(filename, comment_rows):
            return np.genfromtxt(filename, delimiter=' ', usecols=None,
                                 skip_header=comment_rows or 0, dtype='f')

        io = AsciiSignalIO(filename, sampling_rate=sampling_rate, delimiter=' ',
                           units='mV', method=reader)
        block = io.read_block()

        signal1 = block.segments[0].analogsignals[1]
        assert_array_almost_equal(signal1.reshape(-1).magnitude, sample_data[:, 1],
                                  decimal=6)
        self.assertEqual(len(block.segments[0].analogsignals), 3)
        self.assertEqual(signal1.t_stop, sample_data.shape[0] / sampling_rate)
        self.assertEqual(signal1.units, pq.mV)

        os.remove(filename)

    # test usecols
    # test skiprows

    def test_timecolumn(self):
        sample_data = np.random.uniform(size=(200, 3))
        sampling_period = 0.5
        time_data = sampling_period * np.arange(sample_data.shape[0])
        combined_data = np.hstack((sample_data, time_data[:, np.newaxis]))
        filename = "test_multichannel.txt"
        np.savetxt(filename, combined_data, delimiter=' ')
        io = AsciiSignalIO(filename, delimiter=' ',
                           units='mV', method='genfromtxt', timecolumn=-1,
                           time_units='ms', signal_group_mode='split-all')
        block = io.read_block()

        signal1 = block.segments[0].analogsignals[1]
        assert_array_almost_equal(signal1.reshape(-1).magnitude, sample_data[:, 1],
                                  decimal=6)
        self.assertEqual(signal1.sampling_period, sampling_period * pq.ms)
        self.assertEqual(len(block.segments[0].analogsignals), 3)
        self.assertEqual(signal1.t_stop, sample_data.shape[0] * sampling_period * pq.ms)
        self.assertEqual(signal1.units, pq.mV)

        os.remove(filename)

    def test_multichannel(self):
        sample_data = np.random.uniform(size=(200, 3))
        filename = "test_multichannel.txt"
        np.savetxt(filename, sample_data, delimiter=' ')
        sampling_rate = 1 * pq.kHz
        io = AsciiSignalIO(filename, sampling_rate=sampling_rate, delimiter=' ',
                           units='mV', method='genfromtxt',
                           signal_group_mode='all-in-one')
        block = io.read_block()

        signal = block.segments[0].analogsignals[0]
        assert_array_almost_equal(signal.magnitude, sample_data,
                                  decimal=6)
        self.assertEqual(len(block.segments[0].analogsignals), 1)
        self.assertEqual(signal.t_stop, sample_data.shape[0] / sampling_rate)
        self.assertEqual(signal.units, pq.mV)

        os.remove(filename)

    def test_multichannel_with_timecolumn(self):
        sample_data = np.random.uniform(size=(200, 3))
        sampling_period = 0.5
        time_data = sampling_period * np.arange(sample_data.shape[0])
        combined_data = np.hstack((time_data[:, np.newaxis], sample_data))
        filename = "test_multichannel.txt"
        np.savetxt(filename, combined_data, delimiter=' ')
        io = AsciiSignalIO(filename, delimiter=' ',
                           units='mV', method='genfromtxt', timecolumn=0,
                           time_units='ms',
                           signal_group_mode='all-in-one')
        block = io.read_block()

        signal = block.segments[0].analogsignals[0]
        assert_array_almost_equal(signal.magnitude, sample_data,
                                  decimal=6)
        self.assertEqual(signal.sampling_period, sampling_period * pq.ms)
        self.assertEqual(len(block.segments[0].analogsignals), 1)
        self.assertEqual(signal.t_stop, sample_data.shape[0] * sampling_period * pq.ms)
        self.assertEqual(signal.units, pq.mV)

        os.remove(filename)

    def test_multichannel_with_negative_timecolumn(self):
        sample_data = np.random.uniform(size=(200, 3))
        sampling_period = 0.5
        time_data = sampling_period * np.arange(sample_data.shape[0])
        combined_data = np.hstack((sample_data, time_data[:, np.newaxis]))
        filename = "test_multichannel.txt"
        np.savetxt(filename, combined_data, delimiter=' ')
        io = AsciiSignalIO(filename, delimiter=' ',
                           units='mV', method='genfromtxt', timecolumn=-1,
                           time_units='ms',
                           signal_group_mode='all-in-one')
        block = io.read_block()

        signal = block.segments[0].analogsignals[0]
        assert_array_almost_equal(signal.magnitude, sample_data,
                                  decimal=6)
        self.assertEqual(signal.sampling_period, sampling_period * pq.ms)
        self.assertEqual(len(block.segments[0].analogsignals), 1)
        self.assertEqual(signal.t_stop, sample_data.shape[0] * sampling_period * pq.ms)
        self.assertEqual(signal.units, pq.mV)

        os.remove(filename)

    # test write without timecolumn
    # test write with timecolumn
    # test write with units/timeunits different from those of signal

    def test_read_with_json_metadata(self):
        sample_data = np.random.uniform(size=(200, 3))
        filename = "test_read_with_json_metadata.txt"
        metadata_filename = "test_read_with_json_metadata_about.json"
        np.savetxt(filename, sample_data, delimiter=' ')

        metadata = {
            "filename": filename,
            "delimiter": " ",
            "timecolumn": None,
            "units": "mV",
            "time_units": "ms",
            "sampling_rate": {
                "value": 1.0,
                "units": "kHz"
            },
            "method": "genfromtxt",
            "signal_group_mode": 'split-all'
        }
        with open(metadata_filename, "w") as fp:
            json.dump(metadata, fp)

        expected_sampling_rate = 1 * pq.kHz
        io = AsciiSignalIO(filename)
        block = io.read_block()

        signal1 = block.segments[0].analogsignals[1]
        assert_array_almost_equal(signal1.reshape(-1).magnitude, sample_data[:, 1],
                                  decimal=6)
        self.assertEqual(len(block.segments[0].analogsignals), 3)
        self.assertEqual(signal1.sampling_rate, expected_sampling_rate)
        self.assertEqual(signal1.t_stop, sample_data.shape[0] / signal1.sampling_rate)
        self.assertEqual(signal1.units, pq.mV)

        os.remove(filename)
        os.remove(metadata_filename)

    def test_roundtrip_with_json_metadata(self):
        sample_data = np.random.uniform(size=(200, 3))
        filename = "test_roundtrip_with_json_metadata.txt"
        metadata_filename = "test_roundtrip_with_json_metadata_about.json"
        signal1 = AnalogSignal(sample_data, units="pA", sampling_rate=2 * pq.kHz)
        seg1 = Segment()
        block1 = Block()
        seg1.analogsignals.append(signal1)
        seg1.block = block1
        block1.segments.append(seg1)

        iow = AsciiSignalIO(filename, metadata_filename=metadata_filename)
        iow.write_block(block1)
        self.assertTrue(os.path.exists(metadata_filename))

        ior = AsciiSignalIO(filename)
        block2 = ior.read_block()
        assert len(block2.segments[0].analogsignals) == 3
        signal2 = block2.segments[0].analogsignals[1]

        assert_array_almost_equal(signal1.magnitude[:, 1], signal2.magnitude.reshape(-1),
                                  decimal=7)
        self.assertEqual(signal1.units, signal2.units)
        self.assertEqual(signal1.sampling_rate, signal2.sampling_rate)
        assert_array_equal(signal1.times, signal2.times)

        os.remove(filename)
        os.remove(metadata_filename)

    def test_genfromtxt_irregular_expect_success(self):
        sample_data = np.random.uniform(size=(200, 3))
        sample_data[:, 0] = np.sort(sample_data[:, 0])  # make column 0 the time column
        filename = "test_genfromtxt_irregular_expect_success.txt"
        np.savetxt(filename, sample_data, delimiter=' ')

        io = AsciiSignalIO(filename, delimiter=' ', timecolumn=0,
                           units='mV', method='genfromtxt', signal_group_mode='split-all')
        block = io.read_block()

        signal1 = block.segments[0].irregularlysampledsignals[1]
        assert_array_almost_equal(signal1.reshape(-1).magnitude, sample_data[:, 2],
                                  decimal=6)
        self.assertEqual(len(block.segments[0].analogsignals), 0)
        self.assertEqual(len(block.segments[0].irregularlysampledsignals), 2)
        self.assertEqual(signal1.units, pq.mV)

        os.remove(filename)

    def test_irregular_multichannel(self):
        sample_data = np.random.uniform(size=(200, 3))
        sample_data[:, 0] = np.sort(sample_data[:, 0])  # make column 0 the time column
        filename = "test_irregular_multichannel.txt"
        np.savetxt(filename, sample_data, delimiter=' ')

        io = AsciiSignalIO(filename, delimiter=' ', timecolumn=0,
                           units='mV', method='genfromtxt', signal_group_mode='all-in-one')
        block = io.read_block()

        signal = block.segments[0].irregularlysampledsignals[0]
        assert_array_almost_equal(signal.magnitude, sample_data[:, 1:3],
                                  decimal=6)
        self.assertEqual(len(block.segments[0].analogsignals), 0)
        self.assertEqual(len(block.segments[0].irregularlysampledsignals), 1)
        self.assertEqual(signal.shape, (200, 2))
        self.assertEqual(signal.units, pq.mV)

        os.remove(filename)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>from django.contrib.gis.gdal import SpatialReference
from django.contrib.gis.utils import LayerMapping
from django.db import IntegrityError
import dbfread
import os

from parcel.models import Parcel

name = "Somerville, MA"

# Note: This is the WKT from M274TaxPar.prj, with one important difference. The
# given WKT specifies Lambert_Conformal_Conic, which is ambiguous. The intended
# meaning is Lambert_Conformal_Conic_2SP.
srs_wkt = 'PROJCS["NAD_1983_StatePlane_Massachusetts_Mainland_FIPS_2001",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Lambert_Conformal_Conic_2SP"],PARAMETER["False_Easting",200000.0],PARAMETER["False_Northing",750000.0],PARAMETER["Central_Meridian",-71.5],PARAMETER["Standard_Parallel_1",41.71666666666667],PARAMETER["Standard_Parallel_2",42.68333333333333],PARAMETER["Latitude_Of_Origin",41.0],UNIT["Meter",1.0]]'

def import_shapes(logger):
    srs = SpatialReference(srs_wkt)
    lm = LayerMapping(
        Parcel,
        "/data/shapefiles/M274TaxPar.shp", {
            "shape_leng": "SHAPE_Leng",
            "shape_area": "SHAPE_Area",
            "map_par_id": "MAP_PAR_ID",
            "loc_id": "LOC_ID",
            "poly_type": "POLY_TYPE",
            "map_no": "MAP_NO",
            "source": "SOURCE",
            "plan_id": "PLAN_ID",
            "last_edit": "LAST_EDIT",
            "town_id": "TOWN_ID",
            "shape": "POLYGON"
        },
        source_srs=srs)
    lm.save(strict=True)

def add_assessor_data(logger, file_name="M274Assess.dbf"):
    logger.info("Adding assessor data...")
    # This could be done more efficiently, but it only needs to run once.
    path = os.path.join("/data/shapefiles", file_name)
    dbf = dbfread.DBF(path)
    copy_attributes = [
        "LOT_SIZE", "USE_CODE", "YEAR_BUILT", "BLD_AREA", "UNITS", "STYLE",
        "STORIES", "NUM_ROOMS", "ZONING"
    ]
    for entry in dbf:
        try:
            parcel = Parcel.objects.get(loc_id=entry["LOC_ID"])
            parcel.address_num = entry["ADDR_NUM"]
            parcel.full_street = entry["FULL_STR"]
            parcel.save()
            for attr in copy_attributes:
                if attr in entry:
                    try:
                        parcel.attributes.create(
                            name=attr, value=str(entry[attr]))
                    except IntegrityError:
                        continue
        except Parcel.DoesNotExist:
            continue

def run(logger=None):
    import_shapes(logger)
    add_assessor_data(logger)
<commit_msg>Avoid re-importing some parcel assessor data<commit_after>from django.contrib.gis.gdal import SpatialReference
from django.contrib.gis.utils import LayerMapping
from django.db import IntegrityError
import dbfread
import os

from parcel.models import Parcel

name = "Somerville, MA"

# Note: This is the WKT from M274TaxPar.prj, with one important difference. The
# given WKT specifies Lambert_Conformal_Conic, which is ambiguous. The intended
# meaning is Lambert_Conformal_Conic_2SP.
srs_wkt = 'PROJCS["NAD_1983_StatePlane_Massachusetts_Mainland_FIPS_2001",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Lambert_Conformal_Conic_2SP"],PARAMETER["False_Easting",200000.0],PARAMETER["False_Northing",750000.0],PARAMETER["Central_Meridian",-71.5],PARAMETER["Standard_Parallel_1",41.71666666666667],PARAMETER["Standard_Parallel_2",42.68333333333333],PARAMETER["Latitude_Of_Origin",41.0],UNIT["Meter",1.0]]'

def import_shapes(logger):
    srs = SpatialReference(srs_wkt)
    lm = LayerMapping(
        Parcel,
        "/data/shapefiles/M274TaxPar.shp", {
            "shape_leng": "SHAPE_Leng",
            "shape_area": "SHAPE_Area",
            "map_par_id": "MAP_PAR_ID",
            "loc_id": "LOC_ID",
            "poly_type": "POLY_TYPE",
            "map_no": "MAP_NO",
            "source": "SOURCE",
            "plan_id": "PLAN_ID",
            "last_edit": "LAST_EDIT",
            "town_id": "TOWN_ID",
            "shape": "POLYGON"
        },
        source_srs=srs)
    lm.save(strict=True)

def add_assessor_data(logger, file_name="M274Assess.dbf"):
    if logger:
        logger.info("Adding assessor data...")
    # This could be done more efficiently, but it only needs to run once.
    path = os.path.join("/data/shapefiles", file_name)
    dbf = dbfread.DBF(path)
    copy_attributes = [
        "LOT_SIZE", "USE_CODE", "YEAR_BUILT", "BLD_AREA", "UNITS", "STYLE",
        "STORIES", "NUM_ROOMS", "ZONING"
    ]
    processed_ids = set()
    for entry in dbf:
        loc_id = entry["LOC_ID"]
        if loc_id in processed_ids:
            continue
        processed_ids.add(loc_id)

        try:
            parcel = Parcel.objects.get(loc_id=loc_id)
            parcel.address_num = entry["ADDR_NUM"]
            parcel.full_street = entry["FULL_STR"]
            parcel.save()
            for attr in copy_attributes:
                if attr in entry:
                    try:
                        parcel.attributes.create(
                            name=attr, value=str(entry[attr]))
                    except IntegrityError:
                        continue
        except Parcel.DoesNotExist:
            continue

def run(logger=None):
    import_shapes(logger)
    add_assessor_data(logger)
<|endoftext|>


<commit_before>from asyncio import get_event_loop
from unittest import TestCase, mock

from rfxcom.transport import AsyncioTransport

from rfxcom.protocol import RESET_PACKET, STATUS_PACKET

class AsyncioTransportTestCase(TestCase):

    def test_loop_once(self):

        loop = get_event_loop()

        def handler(*args, **kwargs):
            pass

        device = mock.MagicMock()

        AsyncioTransport(device, loop, callback=handler)
        loop._run_once()

        device.write.assert_has_call(bytearray(RESET_PACKET))
        device.write.assert_has_call(bytearray(STATUS_PACKET))
<commit_msg>unit test for AsyncioTransport rewritten for constructor and setup<commit_after>from unittest import TestCase, mock

from rfxcom.transport import AsyncioTransport
from rfxcom.protocol import RESET_PACKET, MODE_PACKET, STATUS_PACKET

class AsyncioTransportTestCase(TestCase):

    @mock.patch('asyncio.AbstractEventLoop')
    @mock.patch('serial.Serial')
    def test_transport_constructor(self, device, loop):
        unit = AsyncioTransport(device, loop, callback=mock.Mock())
        loop.add_writer.assert_called_once_with(device.fd, unit.setup)

    @mock.patch('asyncio.AbstractEventLoop')
    @mock.patch('serial.Serial')
    @mock.patch('rfxcom.transport.AsyncioTransport.write')
    def test_transport_setup(self, unit_write, device, loop):

        unit = AsyncioTransport(device, loop, callback=mock.Mock())
        unit.setup()

        loop.remove_writer.assert_called_with(device.fd)
        device.write.assert_called_once(RESET_PACKET)
        loop.call_later.assert_any_call(mock.ANY,
                                        super(AsyncioTransport, unit).write,
                                        STATUS_PACKET)
        unit_write.assert_called_once_with(MODE_PACKET)
<|endoftext|>


<commit_before># @author: Marek
# cykliczne zapytanie do VT

# TODO: sprawdzic, czy conr ma odpowiedni format
# TODO: sprawdzic czy mamy juz takie SHA w bazie

import logging
from traceback import print_exc

from server.requesthandlers import vt_request
from apscheduler.events import SchedulerEvent

def cycle_handler(request, response, scheduler):
    # obliczamy sha256 dla pliku binarnego
    headers = request.headers
    data = request.json()
    sha256 = data.get('sha256')
    cron = data.get('cron')
    content_type = headers.get('Content-Type')

    # ' application/json'?
    if (headers.get('Content-Type') == 'application/json'):
        logging.error("Wrong Header! Expected: json")
        response.status = "415 Unsupported Media Type"
        response.body = {
            "error": "Please use content_type: application/json"
        }
        return response

    try:
        scheduler.add_job(func=vt_request.request_to_vt, arg=sha256, trigger=cron, id=sha256, replace_existing=True)
    except Exception as e:
        print ('type is:', e.__class__.__name__)
        print_exc()

    # tworzymy response
    response.status = "202 Accepted"
    response.body = {
        "message": "Your schedule has been created"
    }

    return response
<commit_msg>cron fixed<commit_after># @author: Marek
# cykliczne zapytanie do VT

# TODO: sprawdzic, czy conr ma odpowiedni format
# TODO: sprawdzic czy mamy juz takie SHA w bazie

import logging
from traceback import print_exc

from server.requesthandlers import vt_request
from apscheduler.events import SchedulerEvent

def cycle_handler(request, response, scheduler):
    # obliczamy sha256 dla pliku binarnego
    headers = request.headers
    data = request.json()
    sha256 = data.get('sha256')
    cron = data.get('cron')
    content_type = headers.get('Content-Type')

    # ' application/json'?
    if (headers.get('Content-Type') == 'application/json'):
        logging.error("Wrong Header! Expected: json")
        response.status = "415 Unsupported Media Type"
        response.body = {
            "error": "Please use content_type: application/json"
        }
        return response

    try:
        scheduler.add_job(func=lambda : vt_request.request_to_vt(sha256),
                          trigger='cron',id=sha256,
                          replace_existing=True, **cron)
    except Exception as e:
        print ('type is:', e.__class__.__name__)
        print_exc()

    # tworzymy response
    response.status = "202 Accepted"
    response.body = {
        "message": "Your schedule has been created"
    }

    return response
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Manifest format conversion"""
from __future__ import unicode_literals
from resources.lib.globals import g
import re
import base64
import uuid
import xbmc
import xml.etree.ElementTree as ET

import resources.lib.common as common

def convert_to_dash(manifest):
    """Convert a Netflix style manifest to MPEGDASH manifest"""
    seconds = manifest['duration'] / 1000
    init_length = seconds / 2 * 12 + 20 * 1000
    duration = "PT" + str(seconds) + ".00S"

    root = _mpd_manifest_root(duration)
    period = ET.SubElement(root, 'Period', start='PT0S', duration=duration)
    protection = _protection_info(manifest)

    for video_track in manifest['video_tracks']:
        _convert_video_track(
            video_track, period, init_length, protection)

    default_audio_language_index = _get_default_audio_language(manifest)
    for index, audio_track in enumerate(manifest['audio_tracks']):
        _convert_audio_track(audio_track, period, init_length,
                             default=(index == default_audio_language_index))

    default_subtitle_language_index = _get_default_subtitle_language(manifest)
    for index, text_track in enumerate(manifest['timedtexttracks']):
        if text_track['isNoneTrack']:
            continue
        _convert_text_track(text_track, period,
                            default=(index == default_subtitle_language_index))

    xml = ET.tostring(root, encoding='utf-8', method='xml')
    common.save_file('manifest.mpd', xml)
    return xml.replace('\n', '').replace('\r', '')

def _mpd_manifest_root(duration):
    root = ET.Element('MPD')
    root.attrib['xmlns'] = 'urn:mpeg:dash:schema:mpd:2011'
    root.attrib['xmlns:cenc'] = 'urn:mpeg:cenc:2013'
    root.attrib['mediaPresentationDuration'] = duration
    return root

def _protection_info(manifest):
    try:
        pssh = None
        keyid = None
        if 'drmHeader' in manifest:
            pssh = manifest['drmHeader']['bytes']
            keyid = manifest['drmHeader']['keyId']
    except (KeyError, AttributeError, IndexError):
        pssh = None
        keyid = None
    return {'pssh': pssh, 'keyid': keyid}

def _convert_video_track(video_track, period, init_length, protection):
    adaptation_set = ET.SubElement(
        parent=period,
        tag='AdaptationSet',
        mimeType='video/mp4',
        contentType='video')
    _add_protection_info(adaptation_set, **protection)
    for downloadable in video_track['streams']:
        _convert_video_downloadable(
            downloadable, adaptation_set, init_length)

def _add_protection_info(adaptation_set, pssh, keyid):
    if keyid:
        protection = ET.SubElement(
            parent=adaptation_set,
            tag='ContentProtection',
            value='cenc',
            schemeIdUri='urn:mpeg:dash:mp4protection:2011').set(
                'cenc:default_KID', str(uuid.UUID(bytes=keyid)))
    protection = ET.SubElement(
        parent=adaptation_set,
        tag='ContentProtection',
        schemeIdUri='urn:uuid:EDEF8BA9-79D6-4ACE-A3C8-27DCD51D21ED')
    ET.SubElement(
        parent=protection,
        tag='widevine:license',
        robustness_level='HW_SECURE_CODECS_REQUIRED')
    if pssh:
        ET.SubElement(protection, 'cenc:pssh').text = pssh

def _convert_video_downloadable(downloadable, adaptation_set,
                                init_length):
    representation = ET.SubElement(
        parent=adaptation_set,
        tag='Representation',
        width=str(downloadable['res_w']),
        height=str(downloadable['res_h']),
        bandwidth=str(downloadable['bitrate'] * 1024),
        nflxContentProfile=str(downloadable['content_profile']),
        codecs=_determine_video_codec(downloadable['content_profile']),
        mimeType='video/mp4')
    _add_base_url(representation, downloadable['urls'][0]['url'])
    _add_segment_base(representation, init_length)

def _determine_video_codec(content_profile):
    if 'hevc' in content_profile:
        return 'hevc'
    elif 'vp9' in content_profile:
        return 'vp9.0.' + content_profile[14:16]
    return 'h264'

def _convert_audio_track(audio_track, period, init_length, default):
    channels_count = {'1.0': '1', '2.0': '2', '5.1': '6', '7.1': '8'}
    impaired = 'true' if audio_track['trackType'] == 'ASSISTIVE' else 'false'
    original = 'true' if audio_track['isNative'] else 'false'
    default = 'true' if default else 'false'

    adaptation_set = ET.SubElement(
        parent=period,
        tag='AdaptationSet',
        lang=audio_track['language'],
        contentType='audio',
        mimeType='audio/mp4',
        impaired=impaired,
        original=original,
        default=default)
    for downloadable in audio_track['streams']:
        _convert_audio_downloadable(
            downloadable, adaptation_set, init_length,
            channels_count[downloadable['channels']])

def _convert_audio_downloadable(downloadable, adaptation_set, init_length,
                                channels_count):
    representation = ET.SubElement(
        parent=adaptation_set,
        tag='Representation',
        codecs='ec-3' if 'ddplus' in downloadable['content_profile'] else 'aac',
        bandwidth=str(downloadable['bitrate'] * 1024),
        mimeType='audio/mp4')
    ET.SubElement(
        parent=representation,
        tag='AudioChannelConfiguration',
        schemeIdUri='urn:mpeg:dash:23003:3:audio_channel_configuration:2011',
        value=channels_count)
    _add_base_url(representation, downloadable['urls'][0]['url'])
    _add_segment_base(representation, init_length)

def _convert_text_track(text_track, period, default):
    if text_track.get('ttDownloadables'):
        # Only one subtitle representation per adaptationset
        downloadable = text_track['ttDownloadables']
        #common.save_file('downloadable.log', str(downloadable))

        content_profile = downloadable.keys()[0]
        is_ios8 = content_profile == 'webvtt-lssdh-ios8'

        adaptation_set = ET.SubElement(
            parent=period,
            tag='AdaptationSet',
            lang=text_track.get('language'),
            codecs=('stpp', 'wvtt')[is_ios8],
            contentType='text',
            mimeType=('application/ttml+xml', 'text/vtt')[is_ios8])
        role = ET.SubElement(
            parent=adaptation_set,
            tag='Role',
            schemeIdUri='urn:mpeg:dash:role:2011')
        if text_track.get('isForcedNarrative'):
            role.set("value", "forced")
        else:
            if default:
                role.set("value", "main")

        representation = ET.SubElement(
            parent=adaptation_set,
            tag='Representation',
            nflxProfile=content_profile)
        _add_base_url(representation, downloadable[content_profile]['downloadUrls'].values()[0])

def _add_base_url(representation, base_url):
    ET.SubElement(
        parent=representation,
        tag='BaseURL').text = base_url

def _add_segment_base(representation, init_length):
    ET.SubElement(
        parent=representation,
        tag='SegmentBase',
        indexRange='0-' + str(init_length),
        indexRangeExact='true')

def _get_default_audio_language(manifest):
    channelList = {'1.0': '1', '2.0': '2'}
    channelListDolby = {'5.1': '6', '7.1': '8'}

    # Read language in kodi settings
    audio_language = common.json_rpc('Settings.GetSettingValue', {'setting': 'locale.audiolanguage'})
    audio_language = xbmc.convertLanguage(audio_language['value'].encode('utf-8'), xbmc.ISO_639_1)
    audio_language = audio_language if audio_language else xbmc.getLanguage(xbmc.ISO_639_1, False)
    audio_language = audio_language if audio_language else 'en'

    # Try to find the preferred language with the right channels
    if g.ADDON.getSettingBool('enable_dolby_sound'):
        for index, audio_track in enumerate(manifest['audio_tracks']):
            if audio_track['language'] == audio_language and audio_track['channels'] in channelListDolby:
                return index
                break
    # If dolby audio track not exists check other channels list
    for index, audio_track in enumerate(manifest['audio_tracks']):
        if audio_track['language'] == audio_language and audio_track['channels'] in channelList:
            return index
            break

    # If there is no matches to preferred language, try to sets the original language track as default
        # Check if the dolby audio track in selected language exists
    if g.ADDON.getSettingBool('enable_dolby_sound'):
        for index, audio_track in enumerate(manifest['audio_tracks']):
            if audio_track['isNative'] and audio_track['channels'] in channelListDolby:
                return index
                break
    # If dolby audio track not exists check other channels list
    for index, audio_track in enumerate(manifest['audio_tracks']):
        if audio_track['isNative'] and audio_track['channels'] in channelList:
            return index
            break
    return 0

def _get_default_subtitle_language(manifest):
    subtitle_language = common.json_rpc('Settings.GetSettingValue', {'setting': 'locale.subtitlelanguage'})
    if subtitle_language['value'] == 'forced_only':
        # Leave the selection of forced subtitles to kodi
        return -1
    else:
        subtitle_language = xbmc.convertLanguage(subtitle_language['value'].encode('utf-8'), xbmc.ISO_639_1)
        subtitle_language = subtitle_language if subtitle_language else xbmc.getLanguage(xbmc.ISO_639_1, False)
        subtitle_language = subtitle_language if subtitle_language else 'en'

        for index, text_track in enumerate(manifest['timedtexttracks']):
            if text_track['isNoneTrack']:
                continue
            if not text_track.get('isForcedNarrative') and text_track['language'] == subtitle_language:
                return index
                break
        return -1
<commit_msg>converter: workaround for locale with country code<commit_after># -*- coding: utf-8 -*-
"""Manifest format conversion"""
from __future__ import unicode_literals
from resources.lib.globals import g
import re
import base64
import uuid
import xbmc
import xml.etree.ElementTree as ET

import resources.lib.common as common

def convert_to_dash(manifest):
    """Convert a Netflix style manifest to MPEGDASH manifest"""
    seconds = manifest['duration'] / 1000
    init_length = seconds / 2 * 12 + 20 * 1000
    duration = "PT" + str(seconds) + ".00S"

    root = _mpd_manifest_root(duration)
    period = ET.SubElement(root, 'Period', start='PT0S', duration=duration)
    protection = _protection_info(manifest)

    for video_track in manifest['video_tracks']:
        _convert_video_track(
            video_track, period, init_length, protection)

    _fix_locale_languages(manifest['audio_tracks'])
    _fix_locale_languages(manifest['timedtexttracks'])

    default_audio_language_index = _get_default_audio_language(manifest)
    for index, audio_track in enumerate(manifest['audio_tracks']):
        _convert_audio_track(audio_track, period, init_length,
                             default=(index == default_audio_language_index))

    default_subtitle_language_index = _get_default_subtitle_language(manifest)
    for index, text_track in enumerate(manifest['timedtexttracks']):
        if text_track['isNoneTrack']:
            continue
        _convert_text_track(text_track, period,
                            default=(index == default_subtitle_language_index))

    xml = ET.tostring(root, encoding='utf-8', method='xml')
    common.save_file('manifest.mpd', xml)
    return xml.replace('\n', '').replace('\r', '')

def _mpd_manifest_root(duration):
    root = ET.Element('MPD')
    root.attrib['xmlns'] = 'urn:mpeg:dash:schema:mpd:2011'
    root.attrib['xmlns:cenc'] = 'urn:mpeg:cenc:2013'
    root.attrib['mediaPresentationDuration'] = duration
    return root

def _protection_info(manifest):
    try:
        pssh = None
        keyid = None
        if 'drmHeader' in manifest:
            pssh = manifest['drmHeader']['bytes']
            keyid = manifest['drmHeader']['keyId']
    except (KeyError, AttributeError, IndexError):
        pssh = None
        keyid = None
    return {'pssh': pssh, 'keyid': keyid}

def _convert_video_track(video_track, period, init_length, protection):
    adaptation_set = ET.SubElement(
        parent=period,
        tag='AdaptationSet',
        mimeType='video/mp4',
        contentType='video')
    _add_protection_info(adaptation_set, **protection)
    for downloadable in video_track['streams']:
        _convert_video_downloadable(
            downloadable, adaptation_set, init_length)

def _add_protection_info(adaptation_set, pssh, keyid):
    if keyid:
        protection = ET.SubElement(
            parent=adaptation_set,
            tag='ContentProtection',
            value='cenc',
            schemeIdUri='urn:mpeg:dash:mp4protection:2011').set(
                'cenc:default_KID', str(uuid.UUID(bytes=keyid)))
    protection = ET.SubElement(
        parent=adaptation_set,
        tag='ContentProtection',
        schemeIdUri='urn:uuid:EDEF8BA9-79D6-4ACE-A3C8-27DCD51D21ED')
    ET.SubElement(
        parent=protection,
        tag='widevine:license',
        robustness_level='HW_SECURE_CODECS_REQUIRED')
    if pssh:
        ET.SubElement(protection, 'cenc:pssh').text = pssh

def _convert_video_downloadable(downloadable, adaptation_set,
                                init_length):
    representation = ET.SubElement(
        parent=adaptation_set,
        tag='Representation',
        width=str(downloadable['res_w']),
        height=str(downloadable['res_h']),
        bandwidth=str(downloadable['bitrate'] * 1024),
        nflxContentProfile=str(downloadable['content_profile']),
        codecs=_determine_video_codec(downloadable['content_profile']),
        mimeType='video/mp4')
    _add_base_url(representation, downloadable['urls'][0]['url'])
    _add_segment_base(representation, init_length)

def _determine_video_codec(content_profile):
    if 'hevc' in content_profile:
        return 'hevc'
    elif 'vp9' in content_profile:
        return 'vp9.0.' + content_profile[14:16]
    return 'h264'

def _convert_audio_track(audio_track, period, init_length, default):
    channels_count = {'1.0': '1', '2.0': '2', '5.1': '6', '7.1': '8'}
    impaired = 'true' if audio_track['trackType'] == 'ASSISTIVE' else 'false'
    original = 'true' if audio_track['isNative'] else 'false'
    default = 'true' if default else 'false'

    adaptation_set = ET.SubElement(
        parent=period,
        tag='AdaptationSet',
        lang=audio_track['language'],
        contentType='audio',
        mimeType='audio/mp4',
        impaired=impaired,
        original=original,
        default=default)
    for downloadable in audio_track['streams']:
        _convert_audio_downloadable(
            downloadable, adaptation_set, init_length,
            channels_count[downloadable['channels']])

def _convert_audio_downloadable(downloadable, adaptation_set, init_length,
                                channels_count):
    representation = ET.SubElement(
        parent=adaptation_set,
        tag='Representation',
        codecs='ec-3' if 'ddplus' in downloadable['content_profile'] else 'aac',
        bandwidth=str(downloadable['bitrate'] * 1024),
        mimeType='audio/mp4')
    ET.SubElement(
        parent=representation,
        tag='AudioChannelConfiguration',
        schemeIdUri='urn:mpeg:dash:23003:3:audio_channel_configuration:2011',
        value=channels_count)
    _add_base_url(representation, downloadable['urls'][0]['url'])
    _add_segment_base(representation, init_length)

def _convert_text_track(text_track, period, default):
    if text_track.get('ttDownloadables'):
        # Only one subtitle representation per adaptationset
        downloadable = text_track['ttDownloadables']
        #common.save_file('downloadable.log', str(downloadable))

        content_profile = downloadable.keys()[0]
        is_ios8 = content_profile == 'webvtt-lssdh-ios8'

        adaptation_set = ET.SubElement(
            parent=period,
            tag='AdaptationSet',
            lang=text_track.get('language'),
            codecs=('stpp', 'wvtt')[is_ios8],
            contentType='text',
            mimeType=('application/ttml+xml', 'text/vtt')[is_ios8])
        role = ET.SubElement(
            parent=adaptation_set,
            tag='Role',
            schemeIdUri='urn:mpeg:dash:role:2011')
        if text_track.get('isForcedNarrative'):
            role.set("value", "forced")
        else:
            if default:
                role.set("value", "main")

        representation = ET.SubElement(
            parent=adaptation_set,
            tag='Representation',
            nflxProfile=content_profile)
        _add_base_url(representation, downloadable[content_profile]['downloadUrls'].values()[0])

def _add_base_url(representation, base_url):
    ET.SubElement(
        parent=representation,
        tag='BaseURL').text = base_url

def _add_segment_base(representation, init_length):
    ET.SubElement(
        parent=representation,
        tag='SegmentBase',
        indexRange='0-' + str(init_length),
        indexRangeExact='true')

def _get_default_audio_language(manifest):
    channelList = {'1.0': '1', '2.0': '2'}
    channelListDolby = {'5.1': '6', '7.1': '8'}

    # Read language in kodi settings
    audio_language = common.json_rpc('Settings.GetSettingValue', {'setting': 'locale.audiolanguage'})
    audio_language = xbmc.convertLanguage(audio_language['value'].encode('utf-8'), xbmc.ISO_639_1)
    audio_language = audio_language if audio_language else xbmc.getLanguage(xbmc.ISO_639_1, False)
    audio_language = audio_language if audio_language else 'en'

    # Try to find the preferred language with the right channels
    if g.ADDON.getSettingBool('enable_dolby_sound'):
        for index, audio_track in enumerate(manifest['audio_tracks']):
            if audio_track['language'] == audio_language and audio_track['channels'] in channelListDolby:
                return index
    # If dolby audio track not exists check other channels list
    for index, audio_track in enumerate(manifest['audio_tracks']):
        if audio_track['language'] == audio_language and audio_track['channels'] in channelList:
            return index

    # If there is no matches to preferred language, try to sets the original language track as default
        # Check if the dolby audio track in selected language exists
    if g.ADDON.getSettingBool('enable_dolby_sound'):
        for index, audio_track in enumerate(manifest['audio_tracks']):
            if audio_track['isNative'] and audio_track['channels'] in channelListDolby:
                return index
    # If dolby audio track not exists check other channels list
    for index, audio_track in enumerate(manifest['audio_tracks']):
        if audio_track['isNative'] and audio_track['channels'] in channelList:
            return index
    return 0

def _get_default_subtitle_language(manifest):
    subtitle_language = common.json_rpc('Settings.GetSettingValue', {'setting': 'locale.subtitlelanguage'})
    if subtitle_language['value'] == 'forced_only':
        # Leave the selection of forced subtitles to kodi
        return -1
    else:
        subtitle_language = xbmc.convertLanguage(subtitle_language['value'].encode('utf-8'), xbmc.ISO_639_1)
        subtitle_language = subtitle_language if subtitle_language else xbmc.getLanguage(xbmc.ISO_639_1, False)
        subtitle_language = subtitle_language if subtitle_language else 'en'

        for index, text_track in enumerate(manifest['timedtexttracks']):
            if text_track['isNoneTrack']:
                continue
            if not text_track.get('isForcedNarrative') and text_track['language'] == subtitle_language:
                return index
        return -1

def _fix_locale_languages(data_list):
    """Replace locale code, Kodi does not understand the country code"""
    # Get all the ISO 639-1 codes (without country)
    locale_list_nocountry = []
    for item in data_list:
        if item.get('isNoneTrack',False):
            continue
        if len(item['language']) == 2 and not item['language'] in locale_list_nocountry:
            locale_list_nocountry.append(item['language'])
    # Replace the locale languages with country with a new one
    for item in data_list:
        if item.get('isNoneTrack',False):
            continue
        if len(item['language']) == 2:
            continue
        item['language'] = _adjust_locale(item['language'], item['language'][0:2] in locale_list_nocountry)

def _adjust_locale(locale_code, lang_code_without_country_exists):
    """
    Locale conversion helper
    Conversion table to prevent Kodi to display es-ES as Spanish - Spanish, pt-BR as Portuguese - Breton, and so on
    """
    locale_conversion_table = {
        'es-ES': 'es-Spain',
        'pt-BR': 'pt-Brazil',
        'fr-CA': 'fr-Canada',
        'ar-EG': 'ar-Egypt',
        'nl-BE': 'nl-Belgium'
    }
    language_code = locale_code[0:2]
    if not lang_code_without_country_exists:
        return language_code
    else:
        if locale_code in locale_conversion_table:
            return locale_conversion_table[locale_code]
        else:
            common.debug('AdjustLocale - missing mapping conversion for locale: {}'.format(locale_code))
            return locale_code
<|endoftext|>


<commit_before>""" Discovers Belkin Wemo devices. """

from . import SSDPDiscoverable

class Discoverable(SSDPDiscoverable):
    """ Adds support for discovering Belkin WeMo platform devices. """

    def info_from_entry(self, entry):
        """ Returns most important info from a uPnP entry. """
        device = entry.description.find('device')

        return (device.find('friendlyName').text,
                device.find('modelName').text,
                entry.values['location'])

    def get_entries(self):
        """ Returns all Belkin Wemo entries. """
        return self.find_by_st('urn:Belkin:service:manufacture:1')
<commit_msg>Update discovery method for Wemo devices<commit_after>""" Discovers Belkin Wemo devices. """

from . import SSDPDiscoverable

class Discoverable(SSDPDiscoverable):
    """ Adds support for discovering Belkin WeMo platform devices. """

    def info_from_entry(self, entry):
        """ Returns most important info from a uPnP entry. """
        device = entry.description.find('device')

        return (device.find('friendlyName').text,
                device.find('modelName').text,
                entry.values['location'])

    def get_entries(self):
        """ Returns all Belkin Wemo entries. """
        return self.find_by_device_description(
            {'manufacturer': 'Belkin International Inc.'})
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#  Copyright 2011 Takeshi KOMIYA
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import functools
from blockdiag.utils import Box, XY

class LazyReciever(object):
    def __init__(self, target):
        self.target = target
        self.calls = []
        self.nested = []

    def __getattr__(self, name):
        return self.get_lazy_method(name)

    def get_lazy_method(self, name):
        method = self._find_method(name)

        def _(*args, **kwargs):
            if name in self.target.self_generative_methods:
                ret = method(self.target, *args, **kwargs)
                reciever = LazyReciever(ret)
                self.nested.append(reciever)
                return reciever
            else:
                self.calls.append((method, args, kwargs))
                return self

        if method.__name__ in self.target.nosideeffect_methods:
            return functools.partial(method, self.target)
        else:
            return _

    def _find_method(self, name):
        for p in self.target.__class__.__mro__:
            if name in p.__dict__:
                return p.__dict__[name]

        raise AttributeError("%s instance has no attribute '%s'" %
                             (self.target.__class__.__name__, name))

    def _run(self):
        for recv in self.nested:
            recv._run()

        for method, args, kwargs in self.calls:
            method(self.target, *args, **kwargs)

class LineJumpDrawFilter(LazyReciever):
    def __init__(self, target, jump_radius):
        super(LineJumpDrawFilter, self).__init__(target)
        self.ytree = []
        self.x_cross = {}
        self.y_cross = {}
        self.forward = 'holizonal'
        self.jump_radius = jump_radius
        self.jump_shift = 0

    def set_options(self, **kwargs):
        if 'jump_forward' in kwargs:
            self.forward = kwargs['jump_forward']

        if 'jump_radius' in kwargs:
            self.jump_radius = kwargs['jump_radius']

        if 'jump_shift' in kwargs:
            self.jump_shift = kwargs['jump_shift']

    def _run(self):
        for recv in self.nested:
            recv._run()

        line_method = self._find_method("line")
        for method, args, kwargs in self.calls:
            if method == line_method and kwargs.get('jump'):
                ((x1, y1), (x2, y2)) = args[0]
                if self.forward == 'holizonal' and y1 == y2:
                    self._holizonal_jumpline(x1, y1, x2, y2, **kwargs)
                    continue
                elif self.forward == 'vertical' and x1 == x2:
                    self._vertical_jumpline(x1, y1, x2, y2, **kwargs)
                    continue

            method(self.target, *args, **kwargs)

    def _holizonal_jumpline(self, x1, y1, x2, y2, **kwargs):
        y = y1
        if x2 < x1:
            x1, x2 = x2, x1

        for x in sorted(self.x_cross.get(y, [])):
            if x1 < x and x < x2:
                arckwargs = dict(kwargs)
                del arckwargs['jump']

                x += self.jump_shift
                r = self.jump_radius
                line = (XY(x1, y), XY(x - r, y))
                self.target.line(line, **kwargs)
                box = Box(x - r, y - r, x + r, y + r)
                self.target.arc(box, 180, 0, **arckwargs)
                x1 = x + r

        self.target.line((XY(x1, y), XY(x2, y)), **kwargs)

    def _vertical_jumpline(self, x1, y1, x2, y2, **kwargs):
        x = x1
        if y2 < y1:
            y1, y2 = y2, y1

        for y in sorted(self.y_cross.get(x, [])):
            if y1 < y and y < y2:
                arckwargs = dict(kwargs)
                del arckwargs['jump']

                y += self.jump_shift
                r = self.jump_radius
                line = (XY(x, y1), XY(x, y - r))
                self.target.line(line, **kwargs)
                box = Box(x - r, y - r, x + r, y + r)
                self.target.arc(box, 270, 90, **arckwargs)
                y1 = y + r

        self.target.line((XY(x, y1), XY(x, y2)), **kwargs)

    def line(self, xy, **kwargs):
        from bisect import insort
        for st, ed in zip(xy[:-1], xy[1:]):
            self.get_lazy_method("line")((st, ed), **kwargs)

            if 'jump' in kwargs and kwargs['jump'] is True:
                if st.y == ed.y:    # horizonal
                    insort(self.ytree, (st.y, 0, (st, ed)))
                elif st.x == ed.x:  # vertical
                    insort(self.ytree, (max(st.y, ed.y), -1, (st, ed)))
                    insort(self.ytree, (min(st.y, ed.y), +1, (st, ed)))

    def save(self, *args, **kwargs):
        # Search crosspoints
        from bisect import insort, bisect_left, bisect_right
        xtree = []
        for y, _, ((x1, y1), (x2, y2)) in self.ytree:
            if x2 < x1:
                x1, x2 = x2, x1
            if y2 < y1:
                y1, y2 = y2, y1

            if y == y1:
                insort(xtree, x1)

            if y == y2:
                del xtree[bisect_left(xtree, x1)]
                for x in xtree[bisect_right(xtree, x1):bisect_left(xtree, x2)]:
                    self.x_cross.setdefault(y, set()).add(x)
                    self.y_cross.setdefault(x, set()).add(y)

        self._run()
        return self.target.save(*args, **kwargs)
<commit_msg>Fix could not execute with python-2.4; Use blockdiag.utils.functools<commit_after># -*- coding: utf-8 -*-
#  Copyright 2011 Takeshi KOMIYA
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from blockdiag.utils import functools, Box, XY

class LazyReciever(object):
    def __init__(self, target):
        self.target = target
        self.calls = []
        self.nested = []

    def __getattr__(self, name):
        return self.get_lazy_method(name)

    def get_lazy_method(self, name):
        method = self._find_method(name)

        def _(*args, **kwargs):
            if name in self.target.self_generative_methods:
                ret = method(self.target, *args, **kwargs)
                reciever = LazyReciever(ret)
                self.nested.append(reciever)
                return reciever
            else:
                self.calls.append((method, args, kwargs))
                return self

        if method.__name__ in self.target.nosideeffect_methods:
            return functools.partial(method, self.target)
        else:
            return _

    def _find_method(self, name):
        for p in self.target.__class__.__mro__:
            if name in p.__dict__:
                return p.__dict__[name]

        raise AttributeError("%s instance has no attribute '%s'" %
                             (self.target.__class__.__name__, name))

    def _run(self):
        for recv in self.nested:
            recv._run()

        for method, args, kwargs in self.calls:
            method(self.target, *args, **kwargs)

class LineJumpDrawFilter(LazyReciever):
    def __init__(self, target, jump_radius):
        super(LineJumpDrawFilter, self).__init__(target)
        self.ytree = []
        self.x_cross = {}
        self.y_cross = {}
        self.forward = 'holizonal'
        self.jump_radius = jump_radius
        self.jump_shift = 0

    def set_options(self, **kwargs):
        if 'jump_forward' in kwargs:
            self.forward = kwargs['jump_forward']

        if 'jump_radius' in kwargs:
            self.jump_radius = kwargs['jump_radius']

        if 'jump_shift' in kwargs:
            self.jump_shift = kwargs['jump_shift']

    def _run(self):
        for recv in self.nested:
            recv._run()

        line_method = self._find_method("line")
        for method, args, kwargs in self.calls:
            if method == line_method and kwargs.get('jump'):
                ((x1, y1), (x2, y2)) = args[0]
                if self.forward == 'holizonal' and y1 == y2:
                    self._holizonal_jumpline(x1, y1, x2, y2, **kwargs)
                    continue
                elif self.forward == 'vertical' and x1 == x2:
                    self._vertical_jumpline(x1, y1, x2, y2, **kwargs)
                    continue

            method(self.target, *args, **kwargs)

    def _holizonal_jumpline(self, x1, y1, x2, y2, **kwargs):
        y = y1
        if x2 < x1:
            x1, x2 = x2, x1

        for x in sorted(self.x_cross.get(y, [])):
            if x1 < x and x < x2:
                arckwargs = dict(kwargs)
                del arckwargs['jump']

                x += self.jump_shift
                r = self.jump_radius
                line = (XY(x1, y), XY(x - r, y))
                self.target.line(line, **kwargs)
                box = Box(x - r, y - r, x + r, y + r)
                self.target.arc(box, 180, 0, **arckwargs)
                x1 = x + r

        self.target.line((XY(x1, y), XY(x2, y)), **kwargs)

    def _vertical_jumpline(self, x1, y1, x2, y2, **kwargs):
        x = x1
        if y2 < y1:
            y1, y2 = y2, y1

        for y in sorted(self.y_cross.get(x, [])):
            if y1 < y and y < y2:
                arckwargs = dict(kwargs)
                del arckwargs['jump']

                y += self.jump_shift
                r = self.jump_radius
                line = (XY(x, y1), XY(x, y - r))
                self.target.line(line, **kwargs)
                box = Box(x - r, y - r, x + r, y + r)
                self.target.arc(box, 270, 90, **arckwargs)
                y1 = y + r

        self.target.line((XY(x, y1), XY(x, y2)), **kwargs)

    def line(self, xy, **kwargs):
        from bisect import insort
        for st, ed in zip(xy[:-1], xy[1:]):
            self.get_lazy_method("line")((st, ed), **kwargs)

            if 'jump' in kwargs and kwargs['jump'] is True:
                if st.y == ed.y:    # horizonal
                    insort(self.ytree, (st.y, 0, (st, ed)))
                elif st.x == ed.x:  # vertical
                    insort(self.ytree, (max(st.y, ed.y), -1, (st, ed)))
                    insort(self.ytree, (min(st.y, ed.y), +1, (st, ed)))

    def save(self, *args, **kwargs):
        # Search crosspoints
        from bisect import insort, bisect_left, bisect_right
        xtree = []
        for y, _, ((x1, y1), (x2, y2)) in self.ytree:
            if x2 < x1:
                x1, x2 = x2, x1
            if y2 < y1:
                y1, y2 = y2, y1

            if y == y1:
                insort(xtree, x1)

            if y == y2:
                del xtree[bisect_left(xtree, x1)]
                for x in xtree[bisect_right(xtree, x1):bisect_left(xtree, x2)]:
                    self.x_cross.setdefault(y, set()).add(x)
                    self.y_cross.setdefault(x, set()).add(y)

        self._run()
        return self.target.save(*args, **kwargs)
<|endoftext|>


<commit_before>import scipy as sp
import numpy as np

class MultivariateNormal(object):
		# TODO: make n_params general
		n_params = 5

    def __init__(self, params, temp_scale = 1.0):
        self.N, p = params.shape
        self.p = int(0.5 * (np.sqrt(8 * p - 3))

        self.loc = params[0:p, :].T

        self.L = np.zeros((self.p, self.p, self.N))
        self.L[np.tril_indices(self.p)] = params[self.p:, :]
        self.L = np.transpose(self.L, (2, 0, 1))
        self.cov = self.L @ np.transpose(self.L, (0, 2, 1))
        self.cov_inv = np.linalg.inv(self.cov)
        self.dCovdL = self.D_cov_D_L()

    def __getattr__(self, name):
        if name in dir(self.dist):
            return getattr(self.dist, name)
        return None

    def D_cov_D_L(self):
        # create commutation matrix
        commutation = np.zeros((self.p**2, self.p**2))
        ind = np.arange(self.p**2).reshape(self.p, self.p).T.flatten()
        commutation[np.arange(self.p**2), ind] = 1.

        # compute Jacobian
        dCovdL = (np.identity(self.p**2) + commutation) @\
                 np.kron(self.L, np.identity(self.p))
        dCovdL = dCovdL.reshape(-1,2,2,2,2).swapaxes(-2,-1)
        return dCovdL

		def nll(self, Y):
        diff = Y - self.loc
        M = diff[:,None,:] @ self.cov_inv @ diff[:,:,None]
        half_log_det = np.log(np.diagonal(self.L, axis1=1, axis2=2)).sum(-1)
        const = self.p / 2 * np.log(2*np.pi)
        logpdf = - const - half_log_det - 0.5 * M.flatten()
        return -logpdf

		def D_nll(self, Y_):
        # reshape Jacobian
        tril_indices = np.tril_indices(self.p)
        J = self.dCovdL[:,:,:,tril_indices[0],tril_indices[1]]
        J = np.transpose(J, (0,3,1,2)).reshape(self.N, -1, self.p**2)
        J = J.swapaxes(-2,-1)

        # compute grad mu
        D = np.zeros((self.N, J.shape[-1] + self.p))
        sigma_inv = np.linalg.inv(self.cov)
        diff = self.loc - Y_
        D[:, :self.p] = (sigma_inv @ diff[:,:,None])[...,0]

        # compute grad sigma
        D_sigma = 0.5*(sigma_inv - sigma_inv @ (diff[:,:,None]*diff[:,None,:]) @ sigma_inv)
        D_sigma = D_sigma.reshape(self.N, -1)
        D_L = J.swapaxes(-2,-1) @ D_sigma[:,:,None]
        D[:, self.p:] = D_L[..., 0]

        return D

	  def fisher_info(self):
        # reshape Jacobian
        tril_indices = np.tril_indices(self.p)
        J = self.dCovdL[:,:,:,tril_indices[0],tril_indices[1]]

        FI = np.zeros((self.N,self.n_params,self.n_params))

        # compute FI mu
        FI[:,:self.p,:self.p] = self.cov_inv

        # compute FI sigma
        M = np.einsum('nij,njkl->nikl', self.cov_inv, J)
        M = np.einsum('nijx,njky->nikxy', M, M)
        FI[:,self.p:,self.p:] = 0.5 * np.trace(M, axis1=1, axis2=2)

        return FI

		def fit(Y):
				pass
<commit_msg>add fit<commit_after>import scipy as sp
import numpy as np

class MultivariateNormal(object):
		# TODO: make n_params general
		n_params = 5

    def __init__(self, params, temp_scale = 1.0):
        self.N, p = params.shape
        self.p = int(0.5 * (np.sqrt(8 * p - 3))

        self.loc = params[0:p, :].T

        self.L = np.zeros((self.p, self.p, self.N))
        self.L[np.tril_indices(self.p)] = params[self.p:, :]
        self.L = np.transpose(self.L, (2, 0, 1))
        self.cov = self.L @ np.transpose(self.L, (0, 2, 1))
        self.cov_inv = np.linalg.inv(self.cov)
        self.dCovdL = self.D_cov_D_L()

    def __getattr__(self, name):
        if name in dir(self.dist):
            return getattr(self.dist, name)
        return None

    def D_cov_D_L(self):
        # create commutation matrix
        commutation = np.zeros((self.p**2, self.p**2))
        ind = np.arange(self.p**2).reshape(self.p, self.p).T.flatten()
        commutation[np.arange(self.p**2), ind] = 1.

        # compute Jacobian
        dCovdL = (np.identity(self.p**2) + commutation) @\
                 np.kron(self.L, np.identity(self.p))
        dCovdL = dCovdL.reshape(-1,2,2,2,2).swapaxes(-2,-1)
        return dCovdL

		def nll(self, Y):
        diff = Y - self.loc
        M = diff[:,None,:] @ self.cov_inv @ diff[:,:,None]
        half_log_det = np.log(np.diagonal(self.L, axis1=1, axis2=2)).sum(-1)
        const = self.p / 2 * np.log(2*np.pi)
        logpdf = - const - half_log_det - 0.5 * M.flatten()
        return -logpdf

		def D_nll(self, Y_):
        # reshape Jacobian
        tril_indices = np.tril_indices(self.p)
        J = self.dCovdL[:,:,:,tril_indices[0],tril_indices[1]]
        J = np.transpose(J, (0,3,1,2)).reshape(self.N, -1, self.p**2)
        J = J.swapaxes(-2,-1)

        # compute grad mu
        D = np.zeros((self.N, J.shape[-1] + self.p))
        sigma_inv = np.linalg.inv(self.cov)
        diff = self.loc - Y_
        D[:, :self.p] = (sigma_inv @ diff[:,:,None])[...,0]

        # compute grad sigma
        D_sigma = 0.5*(sigma_inv - sigma_inv @ (diff[:,:,None]*diff[:,None,:]) @ sigma_inv)
        D_sigma = D_sigma.reshape(self.N, -1)
        D_L = J.swapaxes(-2,-1) @ D_sigma[:,:,None]
        D[:, self.p:] = D_L[..., 0]

        return D

	  def fisher_info(self):
        # reshape Jacobian
        tril_indices = np.tril_indices(self.p)
        J = self.dCovdL[:,:,:,tril_indices[0],tril_indices[1]]

        FI = np.zeros((self.N,self.n_params,self.n_params))

        # compute FI mu
        FI[:,:self.p,:self.p] = self.cov_inv

        # compute FI sigma
        M = np.einsum('nij,njkl->nikl', self.cov_inv, J)
        M = np.einsum('nijx,njky->nikxy', M, M)
        FI[:,self.p:,self.p:] = 0.5 * np.trace(M, axis1=1, axis2=2)

        return FI

		def fit(Y):
        N, p = Y.shape
        m = Y.mean(axis=0)
        diff = Y - m
        sigma = 1 / N * (diff[:,:,None] @ diff[:,None,:]).sum(0)
        L = sp.linalg.cholesky(sigma, lower=True)
        return np.concatenate([m, L[np.tril_indices(p)]])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2014-2022 Dontnod Entertainment

# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:

# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import json
import os
import re

import nimp.command

class CreateLoadlist(nimp.command.Command):
	''' Generates a list of modified files from a set of Perforce changelists '''

	def configure_arguments(self, env, parser):
		parser.add_argument('changelists', nargs = '+', help = 'select the changelists to list files from')
		parser.add_argument('-o', '--output', help = 'output file')
		parser.add_argument('-e', '--extensions', nargs = '*', help = 'file extensions to include', default = [ 'uasset', 'umap' ])
		parser.add_argument('--check-empty', action = 'store_true', help = 'Returns check empty in json format')
		nimp.utils.p4.add_arguments(parser)
		return True

	def is_available(self, env):
		return env.is_unreal, ''

	def sanitized_changelists(self, env):
		changelists = []
		# deal with possible nimp create-laodlist changelists '12 23 43' "23"
		for possible_cl_streak_string in env.changelists[1:]:
			changelists += re.sub(' +', ';', possible_cl_streak_string).split(';')
		return [cl for cl in changelists if cl]

	def run(self, env):
		p4 = nimp.utils.p4.get_client(env)

		modified_files = []
		for path, action in p4.get_modified_files(*self.sanitized_changelists(env)):
			file = os.path.basename(path)
			for extension in env.extensions:
				if file.endswith(extension):
					modified_files.append(file)
					break

		loadlist_path = env.output if env.output else f'{env.unreal_loadlist}'
		loadlist_path = os.path.abspath(env.format(nimp.system.sanitize_path(loadlist_path)))

		if env.check_empty:
			return self.check_empty_loadlist(modified_files)

		with open(env.format(loadlist_path), 'w+') as output:
			lines = output.readlines()
			for file in modified_files:
				if file not in lines:
					print(file)
					output.write(f'{file}\n')

		return True

	def check_empty_loadlist(self, modified_files):
		results = {'loadlist_is_empty': True}
		if modified_files:
			results['loadlist_is_empty'] = False

		json_content = json.dumps(results, indent=4)
		print('<loadlist_start>')
		print(json_content)
		print('<loadlist_end>')
		return True<commit_msg>CreateLoadList: Improve modified files listing<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2014-2022 Dontnod Entertainment

# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:

# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import json
import os
import re

import nimp.command
import nimp.utils.p4

class CreateLoadlist(nimp.command.Command):
	''' Generates a list of modified files from a set of Perforce changelists '''

	def configure_arguments(self, env, parser):
		parser.add_argument('changelists', nargs = '+', help = 'select the changelists to list files from')
		parser.add_argument('-o', '--output', help = 'output file')
		parser.add_argument('-e', '--extensions', nargs = '*', help = 'file extensions to include', default = [ 'uasset', 'umap' ])
		parser.add_argument('--check-empty', action = 'store_true', help = 'Returns check empty in json format')
		nimp.utils.p4.add_arguments(parser)
		return True

	def is_available(self, env):
		return env.is_unreal, ''

	def sanitized_changelists(self, env):
		changelists = []
		# deal with possible nimp create-loadlist changelists '12 23 43' "23"
		for possible_cl_streak_string in env.changelists[1:]:
			changelists += re.sub(' +', ';', possible_cl_streak_string).split(';')
		return [cl for cl in changelists if cl]

	def get_modified_files(self, env):
		changelists = self.sanitized_changelists(env)
		p4 = nimp.utils.p4.get_client(env)

		# Do not use '//...' which will also list files not mapped to workspace
		root = f"//{p4._client}/..."

		paths = []
		for cl in changelists:
			paths.append(f"{root}@{cl}")

		if len(paths) <= 0:
			paths.append(root)

		base_command = [
			"fstat",
			# Only list modified files currently accessible
			"-F", "^headAction=delete & ^headAction=move/delete"
		]

		modified_files = set()
		for (filepath, ) in p4._parse_command_output(base_command + paths, r"^\.\.\. clientFile(.*)$", hide_output=True):
			modified_files.add(os.path.normpath(filepath))

		return list(modified_files)

	def run(self, env):
		loadlist_files = []
		for filepath in self.get_modified_files(env):
			file = os.path.basename(filepath)
			for extension in env.extensions:
				if file.endswith(extension):
					loadlist_files.append(file)
					break

		loadlist_path = env.output if env.output else f'{env.unreal_loadlist}'
		loadlist_path = os.path.abspath(env.format(nimp.system.sanitize_path(loadlist_path)))

		if env.check_empty:
			return self.check_empty_loadlist(loadlist_files)

		with open(env.format(loadlist_path), 'w+') as output:
			lines = output.readlines()
			for file in loadlist_files:
				if file not in lines:
					print(file)
					output.write(f'{file}\n')

		return True

	def check_empty_loadlist(self, modified_files):
		results = {'loadlist_is_empty': True}
		if modified_files:
			results['loadlist_is_empty'] = False

		json_content = json.dumps(results, indent=4)
		print('<loadlist_start>')
		print(json_content)
		print('<loadlist_end>')
		return True<|endoftext|>


<commit_before>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

# for running shell commands
import subprocess
# for finding out the linux distro
import sys,os,platform
# for downloading rpm/ppc files
import urllib

# ctx is imported and used in operations
from cloudify import ctx
# put the operation decorator on any function that is a task
from cloudify.decorators import operation
# Import Cloudify exception
from cloudify.exceptions import NonRecoverableError

# Runs any Shell Command
def _run_shell_command(command):
  ctx.logger.info("Running shell command: {0}".format(command))
  try:
    run = subprocess.check_call(
      command)
  except subprocess.CalledProcessError:
    ctx.logger.error("Unable to run shell command: {0}".format(command))
    raise NonRecoverableError("Command failed: {0}".format(command))
  return run

# Returns the Package Manager for the distrobution
def _get_package_manager():
  package_manager = "apt-get"
  distro,version = _get_distro_version()
  if distro == "Ubuntu":
    ctx.logger.info("{0} is the Linux distribution.".format(distro))
    package_manager = "apt-get"
  elif distro == "Centos":
    ctx.logger.info("{0} is the Linux distribution.".format(distro))
    package_manager = "yum"
  else:
    ctx.logger.error("Currently Operating System {0} is not supported".format(distro))
    exit(1)
  return package_manager

# Upgrade the Package Manager
def _upgrade_package_manager(package_manager):
  q,y = _install_args(package_manager)
  ctx.logger.info("Upgrading {0}".format(package_manager))
  command = ["sudo",package_manager,"upgrade",y,q]
  _run_shell_command(command)

# Installs a Package
def _install_package(package):
  package_manager = _get_package_manager()
  ctx.logger.info("Installing {0}".format(package))
  _update_package_manager(package_manager)
  _upgrade_package_manager(package_manager)
  q,y = _install_args(package_manager)
  command = ["sudo",package_manager,"install",package,q,y]
  _run_shell_command(command)
  _validate_installation(package)

# Returns Install Arguments for a Package Manager
def _install_args(package_manager):
  if package_manager == "apt-get":
    quiet_output = "-qq"
    assume_yes = "-y"
  if package_manager == "yum":
    quiet_output = "-q"
    assume_yes = "-y"
  return quiet_output,assume_yes

 # Gets the Distrobution
def _get_distro_version():
  info = platform.dist()
  distro = info[0]
  version = info[1]
  return distro,version

# Decides which repos to install
def _add_repo(package_manager):
  ctx.logger.info("Installing Additional Repositories to {0}".format(package_manager))
  if package_manager == "yum":
    _install_epel_repo(package_manager)
  elif package_manager == "apt-get":
    _install_ppa_repo(package_manager)
  else:
    ctx.logger.error("Not yum or apt-get.")

# Installs the EPEL Repo
def _install_epel_repo(package_manager):
  wget_url = "http://dl.fedoraproject.org/pub/epel/5/x86_64/"
  distro,version = _get_distro_version()
  if version == "5*":
    rpm_version = "epel-release-5-4.noarch.rpm"
  elif version == "6*":
    rpm_version = "epel-release-6-8.noarch.rpm"
  elif version == "7*":
    rpm_version = "epel-release-7-1.noarch.rpm"
  else:
    rpm_version = ""
    ctx.logger.error("invalid version of CentOs: {0}".format(version))
  filename = os.path.join("/tmp",rpm_version)
  if not os.path.exists(filename):
    ctx.logger.info("Deleting Existing RPM file: {0}".format(filename))
    ctx.logger.info("Downloading EPEL RPM: {0}{1}".format(wget_url,rpm_version))
    urllib.urlretrieve(wget_url + rpm_version, filename)
  command = ["sudo","rpm","-Uvh","/tmp/" + rpm_version]
  _run_shell_command(command)

# Installs the PPA repos
def _install_ppa_repo(package_manager):
  # Installs the Software Properties Common Dependency
  def _install_dependency(package_manager):
    command = ["sudo","apt-get","-f","install","-y"]
    _run_shell_command(command)
    q,y = _install_args(package_manager)
    command = ["sudo","apt-get","install","software-properties-common",q,y]
    _run_shell_command(command)
  _install_dependency(package_manager)
  command = ["sudo","apt-add-repository","ppa:ansible/ansible","-y"]
  _run_shell_command(command)

# Updates a Package Manager
def _update_package_manager(package_manager):
  command = ["sudo","apt-get","clean"]
  _run_shell_command(command)
  _add_repo(package_manager)
  ctx.logger.info("Updating {0}".format(package_manager))
  command = ["sudo",package_manager,"update"]
  _run_shell_command(command)

# validate the installation
def _validate_installation(package):
  ctx.logger.info("Validating {0}: ".format(package))
  command = [package,"--version"]
  code = _run_shell_command(command)
  if code > 0:
    ctx.logger.info("Installation was unsuccessful")
  else:
    ctx.logger.info("Installation was successful")

# Wraps _install_package()
@operation
def install(**kwargs):
  if "package_name" in kwargs.iteritems():
    package = package_name
  else:
    package = "ansible"
  _install_package(package)

<commit_msg>All Ubuntu Tests Successful with this one.<commit_after>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

# for running shell commands
import subprocess
# for finding out the linux distro
import sys,os,platform
# for downloading rpm/ppc files
import urllib

# ctx is imported and used in operations
from cloudify import ctx
# put the operation decorator on any function that is a task
from cloudify.decorators import operation
# Import Cloudify exception
from cloudify.exceptions import NonRecoverableError

# Runs any Shell Command
def _run_shell_command(command):
  ctx.logger.info("Running shell command: {0}".format(command))
  try:
    run = subprocess.check_call(
      command)
  except subprocess.CalledProcessError:
    ctx.logger.error("Unable to run shell command: {0}".format(command))
    raise NonRecoverableError("Command failed: {0}".format(command))
  return run

# Returns the Package Manager for the distrobution
def _get_package_manager():
  package_manager = "apt-get"
  distro,version = _get_distro_version()
  if distro == "Ubuntu":
    ctx.logger.info("{0} is the Linux distribution.".format(distro))
    package_manager = "apt-get"
  elif distro == "Centos":
    ctx.logger.info("{0} is the Linux distribution.".format(distro))
    package_manager = "yum"
  else:
    ctx.logger.error("Currently Operating System {0} is not supported".format(distro))
    exit(1)
  return package_manager

# Upgrade the Package Manager
def _upgrade_package_manager(package_manager):
  q,y = _install_args(package_manager)
  ctx.logger.info("Upgrading {0}".format(package_manager))
  command = ["sudo",package_manager,"upgrade",y,q]
  _run_shell_command(command)

# Installs a Package
def _install_package(package):
  package_manager = _get_package_manager()
  ctx.logger.info("Installing {0}".format(package))
  _update_package_manager(package_manager)
  _upgrade_package_manager(package_manager)
  q,y = _install_args(package_manager)
  command = ["sudo",package_manager,"install",package,q,y]
  _run_shell_command(command)
  _validate_installation(package)

# Returns Install Arguments for a Package Manager
def _install_args(package_manager):
  if package_manager == "apt-get":
    quiet_output = "-qq"
    assume_yes = "-y"
  if package_manager == "yum":
    quiet_output = "-q"
    assume_yes = "-y"
  return quiet_output,assume_yes

 # Gets the Distrobution
def _get_distro_version():
  info = platform.dist()
  distro = info[0]
  version = info[1]
  return distro,version

# Decides which repos to install
def _add_repo(package_manager):
  ctx.logger.info("Installing Additional Repositories to {0}".format(package_manager))
  if package_manager == "yum":
    _install_epel_repo(package_manager)
  elif package_manager == "apt-get":
    _install_ppa_repo(package_manager)
  else:
    ctx.logger.error("Not yum or apt-get.")

# Installs the EPEL Repo
def _install_epel_repo(package_manager):
  wget_url = "http://dl.fedoraproject.org/pub/epel/5/x86_64/"
  distro,version = _get_distro_version()
  if version == "5*":
    rpm_version = "epel-release-5-4.noarch.rpm"
  elif version == "6*":
    rpm_version = "epel-release-6-8.noarch.rpm"
  elif version == "7*":
    rpm_version = "epel-release-7-1.noarch.rpm"
  else:
    rpm_version = ""
    ctx.logger.error("invalid version of CentOs: {0}".format(version))
  filename = os.path.join("/tmp",rpm_version)
  if not os.path.exists(filename):
    ctx.logger.info("Deleting Existing RPM file: {0}".format(filename))
    ctx.logger.info("Downloading EPEL RPM: {0}{1}".format(wget_url,rpm_version))
    urllib.urlretrieve(wget_url + rpm_version, filename)
  command = ["sudo","rpm","-Uvh","/tmp/" + rpm_version]
  _run_shell_command(command)

# Installs the PPA repos
def _install_ppa_repo(package_manager):
  # Installs the Software Properties Common Dependency
  def _install_dependency(package_manager):
    command = ["sudo","apt-get","-f","install","-y"]
    _run_shell_command(command)
    q,y = _install_args(package_manager)
    command = ["sudo","apt-get","install","software-properties-common",q,y]
    _run_shell_command(command)
  _install_dependency(package_manager)
  command = ["sudo","apt-add-repository","ppa:ansible/ansible","-y"]
  _run_shell_command(command)

# Updates a Package Manager
def _update_package_manager(package_manager):
  command = ["sudo","apt-get","clean"]
  _run_shell_command(command)
  _add_repo(package_manager)
  ctx.logger.info("Updating {0}".format(package_manager))
  command = ["sudo",package_manager,"update"]
  _run_shell_command(command)

# validate the installation
def _validate_installation(package):
  ctx.logger.info("Validating {0}: ".format(package))
  command = [package,"--version"]
  code = _run_shell_command(command)
  if code > 0:
    ctx.logger.info("Installation was unsuccessful")
  else:
    ctx.logger.info("Installation was successful")

# Wraps _install_package()
@operation
def install(**kwargs):
  if "package_name" in kwargs.iteritems():
    package = package_name
  else:
    package = "ansible"
  _install_package(package)

<|endoftext|>