<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import numpy as np
import pytest

from .make_test_data import make_test_image
from ..harmonics import (fit_first_and_second_harmonics, fit_upper_harmonic,
                         first_and_second_harmonic_function)
from ..sample import EllipseSample

try:
    from scipy.optimize import leastsq    # noqa
    HAS_SCIPY = True
except ImportError:
    HAS_SCIPY = False

@pytest.mark.skipif('not HAS_SCIPY')
def test_harmonics_1():
    # this is an almost as-is example taken from stackoverflow
    N = 100  # number of data points
    t = np.linspace(0, 4*np.pi, N)

    # create artificial data with noise:
    # mean = 0.5, amplitude = 3., phase = 0.1, noise-std = 0.01
    data = 3.0 * np.sin(t + 0.1) + 0.5 + 0.01 * np.random.randn(N)

    # first guesses for harmonic parameters
    guess_mean = np.mean(data)
    guess_std = 3 * np.std(data) / 2**0.5
    guess_phase = 0

    # Minimize the difference between the actual data and our "guessed"
    # parameters
    # optimize_func = lambda x: x[0] * np.sin(t + x[1]) + x[2] - data
    def optimize_func(x):
        return x[0] * np.sin(t + x[1]) + x[2] - data

    est_std, est_phase, est_mean = leastsq(
        optimize_func, [guess_std, guess_phase, guess_mean])[0]

    # recreate the fitted curve using the optimized parameters
    data_fit = est_std * np.sin(t + est_phase) + est_mean
    residual = data - data_fit

    assert np.mean(residual) == pytest.approx(0.00, abs=0.001)
    assert np.std(residual) == pytest.approx(0.01, abs=0.01)

@pytest.mark.skipif('not HAS_SCIPY')
def test_harmonics_2():
    # this uses the actual functional form used for fitting ellipses
    N = 100
    E = np.linspace(0, 4*np.pi, N)

    y0_0 = 100.
    a1_0 = 10.
    b1_0 = 5.
    a2_0 = 8.
    b2_0 = 2.
    data = (y0_0 + a1_0*np.sin(E) + b1_0*np.cos(E) + a2_0*np.sin(2*E) +
            b2_0*np.cos(2*E) + 0.01*np.random.randn(N))

    harmonics = fit_first_and_second_harmonics(E, data)
    y0, a1, b1, a2, b2 = harmonics[0]
    data_fit = (y0 + a1*np.sin(E) + b1*np.cos(E) + a2*np.sin(2*E) +
                b2*np.cos(2*E) + 0.01*np.random.randn(N))
    residual = data - data_fit

    assert np.mean(residual) == pytest.approx(0.00, abs=0.01)
    assert np.std(residual) == pytest.approx(0.015, abs=0.01)

@pytest.mark.skipif('not HAS_SCIPY')
def test_harmonics_3():
    """Tests an upper harmonic fit."""

    N = 100
    E = np.linspace(0, 4*np.pi, N)
    y0_0 = 100.
    a1_0 = 10.
    b1_0 = 5.
    order = 3
    data = (y0_0 + a1_0*np.sin(order*E) + b1_0*np.cos(order*E) +
            0.01*np.random.randn(N))

    harmonic = fit_upper_harmonic(E, data, order)
    y0, a1, b1 = harmonic[0]
    data_fit = (y0 + a1*np.sin(order*E) + b1*np.cos(order*E) +
                0.01*np.random.randn(N))
    residual = data - data_fit

    assert np.mean(residual) == pytest.approx(0.00, abs=0.01)
    assert np.std(residual) == pytest.approx(0.015, abs=0.01)

@pytest.mark.skipif('not HAS_SCIPY')
class TestFitEllipseSamples(object):
    def setup_class(self):
        # major axis parallel to X image axis
        self.data1 = make_test_image(random_state=123)

        # major axis tilted 45 deg wrt X image axis
        self.data2 = make_test_image(pa=np.pi/4, random_state=123)

    def test_fit_ellipsesample_1(self):
        sample = EllipseSample(self.data1, 40.)
        s = sample.extract()

        harmonics = fit_first_and_second_harmonics(s[0], s[2])
        y0, a1, b1, a2, b2 = harmonics[0]

        assert np.mean(y0) == pytest.approx(200.019, abs=0.001)
        assert np.mean(a1) == pytest.approx(-0.000138, abs=0.001)
        assert np.mean(b1) == pytest.approx(0.000254, abs=0.001)
        assert np.mean(a2) == pytest.approx(-5.658e-05, abs=0.001)
        assert np.mean(b2) == pytest.approx(-0.00911, abs=0.001)

        # check that harmonics subtract nicely
        model = first_and_second_harmonic_function(
            s[0], np.array([y0, a1, b1, a2, b2]))
        residual = s[2] - model

        assert np.mean(residual) == pytest.approx(0.00, abs=0.001)
        assert np.std(residual) == pytest.approx(0.015, abs=0.01)

    def test_fit_ellipsesample_2(self):
        # initial guess is rounder than actual image
        sample = EllipseSample(self.data1, 40., eps=0.1)
        s = sample.extract()

        harmonics = fit_first_and_second_harmonics(s[0], s[2])
        y0, a1, b1, a2, b2 = harmonics[0]

        assert np.mean(y0) == pytest.approx(188.686, abs=0.001)
        assert np.mean(a1) == pytest.approx(0.000283, abs=0.001)
        assert np.mean(b1) == pytest.approx(0.00692, abs=0.001)
        assert np.mean(a2) == pytest.approx(-0.000215, abs=0.001)
        assert np.mean(b2) == pytest.approx(10.153, abs=0.001)

    def test_fit_ellipsesample_3(self):
        # initial guess for center is offset
        sample = EllipseSample(self.data1, x0=220., y0=210., sma=40.)
        s = sample.extract()

        harmonics = fit_first_and_second_harmonics(s[0], s[2])
        y0, a1, b1, a2, b2 = harmonics[0]

        assert np.mean(y0) == pytest.approx(152.660, abs=0.001)
        assert np.mean(a1) == pytest.approx(55.338, abs=0.001)
        assert np.mean(b1) == pytest.approx(33.091, abs=0.001)
        assert np.mean(a2) == pytest.approx(33.036, abs=0.001)
        assert np.mean(b2) == pytest.approx(-14.306, abs=0.001)

    def test_fit_ellipsesample_4(self):
        sample = EllipseSample(self.data2, 40., eps=0.4)
        s = sample.extract()

        harmonics = fit_first_and_second_harmonics(s[0], s[2])
        y0, a1, b1, a2, b2 = harmonics[0]

        assert np.mean(y0) == pytest.approx(245.102, abs=0.001)
        assert np.mean(a1) == pytest.approx(-0.003108, abs=0.001)
        assert np.mean(b1) == pytest.approx(-0.0578, abs=0.001)
        assert np.mean(a2) == pytest.approx(28.781, abs=0.001)
        assert np.mean(b2) == pytest.approx(-63.184, abs=0.001)
<commit_msg>Use approx_allclose in test_harmonics<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import numpy as np
from numpy.testing import assert_allclose
import pytest

from .make_test_data import make_test_image
from ..harmonics import (fit_first_and_second_harmonics, fit_upper_harmonic,
                         first_and_second_harmonic_function)
from ..sample import EllipseSample

try:
    from scipy.optimize import leastsq    # noqa
    HAS_SCIPY = True
except ImportError:
    HAS_SCIPY = False

@pytest.mark.skipif('not HAS_SCIPY')
def test_harmonics_1():
    # this is an almost as-is example taken from stackoverflow
    N = 100  # number of data points
    t = np.linspace(0, 4*np.pi, N)

    # create artificial data with noise:
    # mean = 0.5, amplitude = 3., phase = 0.1, noise-std = 0.01
    data = 3.0 * np.sin(t + 0.1) + 0.5 + 0.01 * np.random.randn(N)

    # first guesses for harmonic parameters
    guess_mean = np.mean(data)
    guess_std = 3 * np.std(data) / 2**0.5
    guess_phase = 0

    # Minimize the difference between the actual data and our "guessed"
    # parameters
    # optimize_func = lambda x: x[0] * np.sin(t + x[1]) + x[2] - data
    def optimize_func(x):
        return x[0] * np.sin(t + x[1]) + x[2] - data

    est_std, est_phase, est_mean = leastsq(
        optimize_func, [guess_std, guess_phase, guess_mean])[0]

    # recreate the fitted curve using the optimized parameters
    data_fit = est_std * np.sin(t + est_phase) + est_mean
    residual = data - data_fit

    assert_allclose(np.mean(residual), 0., atol=0.001)
    assert_allclose(np.std(residual), 0.01, atol=0.01)

@pytest.mark.skipif('not HAS_SCIPY')
def test_harmonics_2():
    # this uses the actual functional form used for fitting ellipses
    N = 100
    E = np.linspace(0, 4*np.pi, N)

    y0_0 = 100.
    a1_0 = 10.
    b1_0 = 5.
    a2_0 = 8.
    b2_0 = 2.
    data = (y0_0 + a1_0*np.sin(E) + b1_0*np.cos(E) + a2_0*np.sin(2*E) +
            b2_0*np.cos(2*E) + 0.01*np.random.randn(N))

    harmonics = fit_first_and_second_harmonics(E, data)
    y0, a1, b1, a2, b2 = harmonics[0]
    data_fit = (y0 + a1*np.sin(E) + b1*np.cos(E) + a2*np.sin(2*E) +
                b2*np.cos(2*E) + 0.01*np.random.randn(N))
    residual = data - data_fit

    assert_allclose(np.mean(residual), 0., atol=0.01)
    assert_allclose(np.std(residual), 0.015, atol=0.01)

@pytest.mark.skipif('not HAS_SCIPY')
def test_harmonics_3():
    """Tests an upper harmonic fit."""

    N = 100
    E = np.linspace(0, 4*np.pi, N)
    y0_0 = 100.
    a1_0 = 10.
    b1_0 = 5.
    order = 3
    data = (y0_0 + a1_0*np.sin(order*E) + b1_0*np.cos(order*E) +
            0.01*np.random.randn(N))

    harmonic = fit_upper_harmonic(E, data, order)
    y0, a1, b1 = harmonic[0]
    data_fit = (y0 + a1*np.sin(order*E) + b1*np.cos(order*E) +
                0.01*np.random.randn(N))
    residual = data - data_fit

    assert_allclose(np.mean(residual), 0., atol=0.01)
    assert_allclose(np.std(residual), 0.015, atol=0.01)

@pytest.mark.skipif('not HAS_SCIPY')
class TestFitEllipseSamples(object):
    def setup_class(self):
        # major axis parallel to X image axis
        self.data1 = make_test_image(random_state=123)

        # major axis tilted 45 deg wrt X image axis
        self.data2 = make_test_image(pa=np.pi/4, random_state=123)

    def test_fit_ellipsesample_1(self):
        sample = EllipseSample(self.data1, 40.)
        s = sample.extract()

        harmonics = fit_first_and_second_harmonics(s[0], s[2])
        y0, a1, b1, a2, b2 = harmonics[0]

        assert_allclose(np.mean(y0), 200.019, atol=0.001)
        assert_allclose(np.mean(a1), -0.000138, atol=0.001)
        assert_allclose(np.mean(b1), 0.000254, atol=0.001)
        assert_allclose(np.mean(a2), -5.658e-05, atol=0.001)
        assert_allclose(np.mean(b2), -0.00911, atol=0.001)

        # check that harmonics subtract nicely
        model = first_and_second_harmonic_function(
            s[0], np.array([y0, a1, b1, a2, b2]))
        residual = s[2] - model

        assert_allclose(np.mean(residual), 0., atol=0.001)
        assert_allclose(np.std(residual), 0.015, atol=0.01)

    def test_fit_ellipsesample_2(self):
        # initial guess is rounder than actual image
        sample = EllipseSample(self.data1, 40., eps=0.1)
        s = sample.extract()

        harmonics = fit_first_and_second_harmonics(s[0], s[2])
        y0, a1, b1, a2, b2 = harmonics[0]

        assert_allclose(np.mean(y0), 188.686, atol=0.001)
        assert_allclose(np.mean(a1), 0.000283, atol=0.001)
        assert_allclose(np.mean(b1), 0.00692, atol=0.001)
        assert_allclose(np.mean(a2), -0.000215, atol=0.001)
        assert_allclose(np.mean(b2), 10.153, atol=0.001)

    def test_fit_ellipsesample_3(self):
        # initial guess for center is offset
        sample = EllipseSample(self.data1, x0=220., y0=210., sma=40.)
        s = sample.extract()

        harmonics = fit_first_and_second_harmonics(s[0], s[2])
        y0, a1, b1, a2, b2 = harmonics[0]

        assert_allclose(np.mean(y0), 152.660, atol=0.001)
        assert_allclose(np.mean(a1), 55.338, atol=0.001)
        assert_allclose(np.mean(b1), 33.091, atol=0.001)
        assert_allclose(np.mean(a2), 33.036, atol=0.001)
        assert_allclose(np.mean(b2), -14.306, atol=0.001)

    def test_fit_ellipsesample_4(self):
        sample = EllipseSample(self.data2, 40., eps=0.4)
        s = sample.extract()

        harmonics = fit_first_and_second_harmonics(s[0], s[2])
        y0, a1, b1, a2, b2 = harmonics[0]

        assert_allclose(np.mean(y0), 245.102, atol=0.001)
        assert_allclose(np.mean(a1), -0.003108, atol=0.001)
        assert_allclose(np.mean(b1), -0.0578, atol=0.001)
        assert_allclose(np.mean(a2), 28.781, atol=0.001)
        assert_allclose(np.mean(b2), -63.184, atol=0.001)
<|endoftext|>


<commit_before>

"""
Test Cython and weave mixing code.
"""<commit_msg>Starting implementing tests for mixing<commit_after>

"""
Test Cython and weave mixing code.
"""

import scipy.stats as stats
import Yank.mixing as mixing
import numpy as np

def mix_replicas(n_swaps=100, n_states=16, u_kl=None):
    """
    Utility function to generate "replicas" and call the mixing function a certain number of times

    Arguments
    ---------
    n_swaps : int (optional)
        The number of times to call the mixing code (default 100)
    n_states : int (optional)
        The number of replica states to include (default 16)
    u_kl : ndarray of floats (optional)
        Energies for each state. If None, will be initialized to 0s

    Returns
    -------
    replica_swap_results : n_states x n_swaps ndarray
       Contains the result of swapping replicas for each state in each swap attempt
    """

def test_even_mixing():
    pass

def test_general_mixing()<|endoftext|>


<commit_before>import unicodecsv as csv
import sys

lowestulmparents = ('/metadaten/', '/single/')

csvoutfile = open(sys.argv[2], 'wb')
citywriter = csv.writer(csvoutfile, delimiter=',')
cities = dict()

with open(sys.argv[1], 'rb') as csvfile:
    cityreader = csv.reader(csvfile, delimiter=',')
    #Skip headings
    headings = next(cityreader, None)
    #Write headings
    citywriter.writerow(headings)

    #Divide up data into cities to reduce searching
    for row in cityreader:
        cityurl = row[0]
        if cityurl not in cities:
            cities[cityurl] = dict()

csvfile.close()

with open(sys.argv[1], 'rb') as csvfile:
    cityreader = csv.reader(csvfile, delimiter=',')
    #Skip headings
    headings = next(cityreader, None)

    for inrow in cityreader:
        try:
            cityurl = inrow[0]
            searchurl = inrow[1]
            parenturl = inrow[6]
        except:
            print u'Data format error: ' + u', '.join(inrow) + u'\nStopping.'
        found = False

        print "Searching for " + searchurl

        if searchurl in cities[cityurl]:
            #Prioritize records with all data
            #In general this shouldn't happen, as the file is downloaded
            #after the link is found
            if ((('ermittelt' in cities[cityurl][searchurl][6]) and ('ermittelt' not in parenturl)) or
            #Ulm special handling to prevent using links from summary pages
            ((cityurl == 'ulm.de') and any(x in parenturl for x in lowestulmparents))):
                cities[cityurl][searchurl] = inrow
                print "Replacing row"
            else:
                print "Row is already there: duplicate"
        else:
            print "Appending as unique entry"
            cities[cityurl][searchurl] = inrow

for city in cities.values():
    for outputrow in city.values():
        citywriter.writerow(outputrow)

csvfile.close();
csvoutfile.close();
<commit_msg>Search for data for entries with missing parents<commit_after>import unicodecsv as csv
import sys

if (len(sys.argv)<3):
    print 'Usage: cleanDuplicates filetoclean.csv outputfile.csv [supportingitemsfulllist.csv]'
    exit()

lowestulmparents = ('/metadaten/', '/single/')

csvoutfile = open(sys.argv[2], 'wb')
citywriter = csv.writer(csvoutfile, delimiter=',')
cities = dict()

with open(sys.argv[1], 'rb') as csvfile:
    cityreader = csv.reader(csvfile, delimiter=',')
    #Skip headings
    headings = next(cityreader, None)
    #Write headings
    citywriter.writerow(headings)

    #Divide up data into cities to reduce searching
    for row in cityreader:
        cityurl = row[0]
        if cityurl not in cities:
            cities[cityurl] = dict()

csvfile.close()

with open(sys.argv[1], 'rb') as csvfile:
    cityreader = csv.reader(csvfile, delimiter=',')
    #Skip headings
    headings = next(cityreader, None)

    for inrow in cityreader:
        try:
            cityurl = inrow[0]
            searchurl = inrow[1]
            parenturl = inrow[6]
        except:
            print u'Data format error: ' + u', '.join(inrow) + u'\nStopping.'
        found = False

        print "Searching for " + searchurl

        if searchurl in cities[cityurl]:
            #Prioritize records with all data
            #In general this shouldn't happen, as the file is downloaded
            #after the link is found
            if ((('ermittelt' in cities[cityurl][searchurl][6]) and ('ermittelt' not in parenturl)) or
            #Ulm special handling to prevent using links from summary pages
            ((cityurl == 'ulm.de') and any(x in parenturl for x in lowestulmparents))):
                cities[cityurl][searchurl] = inrow
                print "Replacing row"
            else:
                print "Row is already there: duplicate"
        else:
            print "Appending as unique entry"
            cities[cityurl][searchurl] = inrow

findparentsource = False

if (len(sys.argv)>3):
    findparentsource = True
    parentsfile = sys.argv[3]

for city in cities.values():
    for outputrow in city.values():
        if 'ermittelt' in outputrow[6] and findparentsource:
            #These links always have http etc. in them, the stored links from the A elements may not
            missinglink = outputrow[1].split('/')[3]
            #Unfortunately it won't work if the URL arguments get rearranged :(
            #If it becomes a problem we should parse the URLs
            with open(sys.argv[1], 'rb') as csvfile:
                parentreader = csv.reader(csvfile, delimiter=',')
                #Skip headings
                headings = next(cityreader, None)

                for inrow in cityreader:
                    try:
                        fileurl = inrow[1]
                        if missinglink in fileurl:
                            outputrow[2] = inrow[2]
                            outputrow[6] = inrow[6]
                            outputrow[6] = inrow[7]
                            break
                    except:
                        print u'Data format error: ' + u', '.join(inrow) + u'\nStopping.'

        citywriter.writerow(outputrow)

csvfile.close();
csvoutfile.close();
<|endoftext|>


<commit_before>#! /usr/bin/env python3
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2016 Pi-Yueh Chuang <pychuang@gwu.edu>
#
# Distributed under terms of the MIT license.

"""Definition of the Polynomial class"""

import numpy
from utils.poly.poly_operations import eval_poly, der_poly, find_roots

class Polynomial(object):
    """The base class for all kinds of polynomial.

    Attrs:
        n: the order of the polynomial
        coeffs: coefficient vector of the polynomial
        defined: a boolean indicating whether the polynomial is defined
        roots: the roots of the polynomial
    """

    def __init__(self, _coeffs=None):
        """__init__

        Args:
            _coeffs: coefficient vector of the polynomial

        Returns: None
        """

        if _coeffs is not None:
            self.set(_coeffs)
        else:
            self.n = None
            self.coeffs = None
            self.defined = False
            self.roots = None

    def __call__(self, x):
        """__call__

        Args:
            x: the location at where the value will be evaluated

        Returns:
            the value at x of the polynomial
        """
        return eval_poly(x, self.coeffs)

    def __find_roots(self):
        """calculate the roots and store them in self.root"""
        self.roots = find_roots(self.coeffs)

    def set(self, _coeffs):
        """set

        Args:
            _coeffs: coefficient vector of the polynomial

        Returns:
        """
        assert isinstance(_coeffs, numpy.ndarray), \
            "coeffs is not a numpy.ndarray"
        assert len(_coeffs.shape) == 1, "_coeffs is not a 1D array"

        self.n = _coeffs.size
        self.coeffs = _coeffs.copy()
        self.defined = True
        self.__find_roots()

    def derive(self, o=1):
        assert o >= 1, "The order of derivative should >= 1"
        assert o <= (self.n + 1), "The order of derivative should <= n+1"

        if o == 1:
            return Polynomial(der_poly(self.coeffs))
        else:
            return Polynomial(der_poly(self.coeffs)).derive(o-1)
<commit_msg>Add __str__ and __repr__ to Polynomial, and fix n<commit_after>#! /usr/bin/env python3
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2016 Pi-Yueh Chuang <pychuang@gwu.edu>
#
# Distributed under terms of the MIT license.

"""Definition of the Polynomial class"""

import numpy
from utils.poly.poly_operations import eval_poly, der_poly, find_roots

class Polynomial(object):
    """The base class for all kinds of polynomial.

    Attrs:
        n: the order of the polynomial
        coeffs: coefficient vector of the polynomial
        defined: a boolean indicating whether the polynomial is defined
        roots: the roots of the polynomial
    """

    def __init__(self, _coeffs=None):
        """__init__

        Args:
            _coeffs: coefficient vector of the polynomial

        Returns: None
        """

        if _coeffs is not None:
            self.set(_coeffs)
        else:
            self.n = None
            self.coeffs = None
            self.defined = False
            self.roots = None

    def __call__(self, x):
        """__call__

        Args:
            x: the location at where the value will be evaluated

        Returns:
            the value at x of the polynomial
        """
        return eval_poly(x, self.coeffs)

    def __repr__(self):
        """__repr__"""

        return "{0}({1})".format(self.__class__, self.coeffs)

    def __str__(self):
        """__str__"""

        s = "{0}".format(self.coeffs[0])
        for i, c in enumerate(self.coeffs[1:]):
            s += " + "
            s += "{0} x^{1}".format(c, i)

        return s

    def __find_roots(self):
        """calculate the roots and store them in self.root"""
        self.roots = find_roots(self.coeffs)

    def set(self, _coeffs):
        """set

        Args:
            _coeffs: coefficient vector of the polynomial

        Returns:
        """
        assert isinstance(_coeffs, numpy.ndarray), \
            "coeffs is not a numpy.ndarray"
        assert len(_coeffs.shape) == 1, "_coeffs is not a 1D array"

        self.n = _coeffs.size - 1
        self.coeffs = _coeffs.copy()
        self.defined = True
        self.__find_roots()

    def derive(self, o=1):
        assert o >= 1, "The order of derivative should >= 1"
        assert o <= (self.n + 1), "The order of derivative should <= n+1"

        if o == 1:
            return Polynomial(der_poly(self.coeffs))
        else:
            return Polynomial(der_poly(self.coeffs)).derive(o-1)
<|endoftext|>


<commit_before>
from igf_data.utils.tools.picard_util import Picard_tools
from ehive.runnable.IGFBaseProcess import IGFBaseProcess
from igf_data.utils.tools.reference_genome_utils import Reference_genome_utils
from igf_data.utils.fileutils import get_temp_dir,remove_dir,move_file

class RunPicard(IGFBaseProcess):
  def param_defaults(self):
    params_dict=super(RunPicard,self).param_defaults()
    params_dict.update({
        'reference_type':'GENOME_FASTA',
        'reference_refFlat':'GENE_REFFLAT',
      })
    return params_dict

  def run(self):
    try:
      project_igf_id=self.param_required('project_igf_id')
      experiment_igf_id=self.param_required('experiment_igf_id')
      sample_igf_id=self.param_required('sample_igf_id')
      java_exe=self.param_required('java_exe')
      java_param=self.param_required('java_param')
      picard_jar=self.param_required('picard_jar')
      input_file=self.param_required('input_file')
      picard_command=self.param_required('picard_command')
      igf_session_class=self.param_required('igf_session_class')
      species_name=self.param('species_name')
      reference_type=self.param('reference_type')
      reference_refFlat=self.param('reference_refFlat')
      base_work_dir=self.param_required('base_work_dir')
      work_dir_prefix=os.path.join(base_work_dir,
                                   project_igf_id,
                                   sample_igf_id,
                                   experiment_igf_id)
      work_dir=self.get_job_work_dir(work_dir=work_dir_prefix)                  # get a run work dir
      temp_output_dir=get_temp_dir()                                            # get temp work dir
      ref_genome=Reference_genome_utils(genome_tag=species_name,
                                        dbsession_class=igf_session_class,
                                        genome_fasta_type=reference_type,
                                        gene_reflat_type=reference_refFlat
                                       )                                        # setup ref genome utils
      genome_fasta=ref_genome.get_genome_fasta()                                # get genome fasta
      ref_flat_file=ref_genome.get_gene_reflat()                                # get refFlat file
      picard=Picard_tools(java_exe=java_exe,
                          java_param=java_param,
                          picard_jar=picard_jar,
                          input_file=input_file,
                          output_dir=temp_output_dir,
                          ref_fasta=genome_fasta,
                          ref_flat_file=ref_flat_file)                          # get genome fasta)                               # setup picard tool
      temp_output_files=picard.run_picard_command(command_name=picard_command)  # run picard command
      output_file_list=list()
      for source_path in temp_output_files:
        dest_path=os.path.join(work_dir,
                               os.path.basename(source_path))                   # get destination filepath
        move_file(source_path=source_path,
                  destinationa_path=dest_path,
                  force=True)                                                   # move files to work dir
        output_file_list.append(dest_path)

      self.param('dataflow_params',{picard_command:output_file_list})           # pass on picard output list
    except Exception as e:
      message='project: {2}, sample:{3}, Error in {0}: {1}'.format(self.__class__.__name__, \
                                                      e, \
                                                      project_igf_id,
                                                      sample_igf_id)
      self.warning(message)
      self.post_message_to_slack(message,reaction='fail')                       # post msg to slack for failed jobs
      raise<commit_msg>removing temp dir<commit_after>
from igf_data.utils.tools.picard_util import Picard_tools
from ehive.runnable.IGFBaseProcess import IGFBaseProcess
from igf_data.utils.tools.reference_genome_utils import Reference_genome_utils
from igf_data.utils.fileutils import get_temp_dir,remove_dir,move_file

class RunPicard(IGFBaseProcess):
  def param_defaults(self):
    params_dict=super(RunPicard,self).param_defaults()
    params_dict.update({
        'reference_type':'GENOME_FASTA',
        'reference_refFlat':'GENE_REFFLAT',
      })
    return params_dict

  def run(self):
    try:
      project_igf_id=self.param_required('project_igf_id')
      experiment_igf_id=self.param_required('experiment_igf_id')
      sample_igf_id=self.param_required('sample_igf_id')
      java_exe=self.param_required('java_exe')
      java_param=self.param_required('java_param')
      picard_jar=self.param_required('picard_jar')
      input_file=self.param_required('input_file')
      picard_command=self.param_required('picard_command')
      igf_session_class=self.param_required('igf_session_class')
      species_name=self.param('species_name')
      reference_type=self.param('reference_type')
      reference_refFlat=self.param('reference_refFlat')
      base_work_dir=self.param_required('base_work_dir')
      work_dir_prefix=os.path.join(base_work_dir,
                                   project_igf_id,
                                   sample_igf_id,
                                   experiment_igf_id)
      work_dir=self.get_job_work_dir(work_dir=work_dir_prefix)                  # get a run work dir
      temp_output_dir=get_temp_dir()                                            # get temp work dir
      ref_genome=Reference_genome_utils(genome_tag=species_name,
                                        dbsession_class=igf_session_class,
                                        genome_fasta_type=reference_type,
                                        gene_reflat_type=reference_refFlat
                                       )                                        # setup ref genome utils
      genome_fasta=ref_genome.get_genome_fasta()                                # get genome fasta
      ref_flat_file=ref_genome.get_gene_reflat()                                # get refFlat file
      picard=Picard_tools(java_exe=java_exe,
                          java_param=java_param,
                          picard_jar=picard_jar,
                          input_file=input_file,
                          output_dir=temp_output_dir,
                          ref_fasta=genome_fasta,
                          ref_flat_file=ref_flat_file)                          # get genome fasta)                               # setup picard tool
      temp_output_files=picard.run_picard_command(command_name=picard_command)  # run picard command
      output_file_list=list()
      for source_path in temp_output_files:
        dest_path=os.path.join(work_dir,
                               os.path.basename(source_path))                   # get destination filepath
        move_file(source_path=source_path,
                  destinationa_path=dest_path,
                  force=True)                                                   # move files to work dir
        output_file_list.append(dest_path)
      remove_dir(temp_output_dir)
      self.param('dataflow_params',{picard_command:output_file_list})           # pass on picard output list
    except Exception as e:
      message='project: {2}, sample:{3}, Error in {0}: {1}'.format(self.__class__.__name__, \
                                                      e, \
                                                      project_igf_id,
                                                      sample_igf_id)
      self.warning(message)
      self.post_message_to_slack(message,reaction='fail')                       # post msg to slack for failed jobs
      raise<|endoftext|>


<commit_before>import logging
import boto3

from simple_container_runtime.modules.Module import SignalModule
from simple_container_runtime.util import get_instance_id

class AwsCfnSignal(SignalModule):
    def __init__(self, config: dict):
        super().__init__(config)
        self.region = config["region"]
        self.stack_name = config["stack_name"]
        self.logical_resource_id = config["logical_resource_id"]
        self.instance_id = get_instance_id()

    def run(self, successful: bool):
        if successful:
            status = "SUCCESS"
        else:
            status = "FAILURE"

        boto3.client("cloudformation").signal_resource(
            StackName=self.stack_name,
            LogicalResourceId=self.logical_resource_id,
            UniqueId=self.instance_id,
            Status=status
        )
<commit_msg>region must be configured explicitly<commit_after>import logging
import boto3

from simple_container_runtime.modules.Module import SignalModule
from simple_container_runtime.util import get_instance_id

class AwsCfnSignal(SignalModule):
    def __init__(self, config: dict):
        super().__init__(config)
        self.region = config["region"]
        self.stack_name = config["stack_name"]
        self.logical_resource_id = config["logical_resource_id"]
        self.instance_id = get_instance_id()

    def run(self, successful: bool):
        if successful:
            status = "SUCCESS"
        else:
            status = "FAILURE"

        boto3.client("cloudformation", region_name=self.region).signal_resource(
            StackName=self.stack_name,
            LogicalResourceId=self.logical_resource_id,
            UniqueId=self.instance_id,
            Status=status
        )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Convert AWS account information from internal to external format"""

from __future__ import print_function, absolute_import, division
import yaml
import json
import logging

FILENAME = "accounts"

def get_converted_aws_accounts(accounts):
    """Return converted AWS account data.

    Produces a dictionary of "file name" -> "file content" pairs. When
    uploading to S3, these become "S3 key" -> "S3 value" pairs.
    """

    try:
        result = {
            "{filename}.yaml".format(filename=FILENAME): yaml.dump(accounts),
            "{filename}.json".format(filename=FILENAME): json.dumps(accounts)}
    except Exception as exc:
        raise Exception("Failed to convert to yaml and json: {0}".format(exc))

    logging.debug("Data successfully converted to yaml and json")
    return result
<commit_msg>CLOUD-311 make yaml and json more readable for humans<commit_after># -*- coding: utf-8 -*-
"""Convert AWS account information from internal to external format"""

from __future__ import print_function, absolute_import, division
import yaml
import json
import logging

FILENAME = "accounts"

def get_converted_aws_accounts(accounts):
    """Return converted AWS account data.

    Produces a dictionary of "file name" -> "file content" pairs. When
    uploading to S3, these become "S3 key" -> "S3 value" pairs.
    """
    try:
        yaml_data = yaml.dump(accounts, indent=2, default_flow_style=False)
        json_data = json.dumps(accounts, sort_keys=True, indent=2)
    except Exception as exc:
        raise Exception("Failed to convert to yaml and json: {0}".format(exc))
    logging.debug("Data successfully converted to yaml and json")

    return {FILENAME + ".yaml": yaml_data,
            FILENAME + ".json": json_data}
<|endoftext|>


<commit_before>'''
Idea from tensorflow translation example:
https://github.com/tensorflow/tensorflow/blob/e39d8feebb9666a331345cd8d960f5ade4652bba/tensorflow/models/rnn/translate/seq2seq_model.py

And neural chatbot example:
https://github.com/inikdom/neural-chatbot/blob/master/models/chatbot.py
'''

import random

import numpy as np
from six.moves import xrange
import tensorflow as tf
from tensorflow.contrib import rnn, seq2seq
from tensorflow.python.layers.core import Dense
import special_vocab as config
import util.vocabutils as vocab_utils

class ChatbotModel(object):
    def __init__(self, vocab_size, hidden_size, dropout,
                 num_layers, max_gradient_norm, batch_size, learning_rate,
                 lr_decay_factor, max_target_length,
                 max_source_length, decoder_mode=False):
        '''
        vocab_size: number of vocab tokens
        buckets: buckets of max sequence lengths
        hidden_size: dimension of hidden layers
        num_layers: number of hidden layers
        max_gradient_norm: maximum gradient magnitude
        batch_size: number of training examples fed to network at once
        learning_rate: starting learning rate of network
        lr_decay_factor: amount by which to decay learning rate
        num_samples: number of samples for sampled softmax
        decoder_mode: Whether to build backpass nodes or not
        '''
        GO_ID = config.GO_ID
        EOS_ID = config.EOS_ID
        self.max_source_length = max_source_length
        self.max_target_length = max_target_length
        self.vocab_size = vocab_size
        self.batch_size = batch_size
        self.global_step = tf.Variable(0, trainable=False)
        self.learning_rate = learning_rate
        self.encoder_inputs = tf.placeholder(shape=(None, None), dtype=tf.int32, name='encoder_inputs')
        self.source_lengths = tf.placeholder(shape=(None,), dtype=tf.int32, name='source_lengths')

        self.decoder_targets = tf.placeholder(shape=(None, None), dtype=tf.int32, name='decoder_targets')
        self.target_lengths = tf.placeholder(shape=(None,), dtype=tf.int32, name="target_lengths")

        with tf.variable_scope('embeddings') as scope:
            embeddings = tf.Variable(tf.random_uniform([vocab_size, hidden_size], -1.0, 1.0), dtype=tf.float32)
            encoder_inputs_embedded = tf.nn.embedding_lookup(embeddings, self.encoder_inputs)
            targets_embedding = tf.nn.embedding_lookup(embeddings, self.decoder_targets)

        with tf.variable_scope('encoder') as scope:
            encoder_cell = rnn.LSTMCell(hidden_size)
            encoder_cell = rnn.DropoutWrapper(encoder_cell,
                                              input_keep_prob=dropout)

            encoder_cell = rnn.MultiRNNCell([encoder_cell] * num_layers)

            encoder_outputs, encoder_state = tf.nn.bidirectional_dynamic_rnn(cell_fw=encoder_cell,
                                                               cell_bw=encoder_cell,
                                                               sequence_length=self.source_lengths,
                                                               inputs=encoder_inputs_embedded,
                                                               dtype=tf.float32,
                                                               time_major=False) #BiLSTM encoder
            encoder_outputs = tf.concat(encoder_outputs,2)
        with tf.variable_scope('decoder') as scope:
            decoder_cell = rnn.LSTMCell(hidden_size)
            decoder_cell = rnn.DropoutWrapper(decoder_cell,
                                              input_keep_prob=dropout)

            decoder_cell = rnn.MultiRNNCell([decoder_cell] * num_layers)

            #TODO add attention
            #attention_mechanism= seq2seq.BahdanauAttention(num_units=hidden_size,memory=encoder_outputs)

            #decoder_cell = seq2seq.AttentionWrapper(cell=decoder_cell,
            #                                        attention_mechanism=)
            attn_mech=seq2seq.BahdanauAttention(
                      num_units=hidden_size,#depth of query mechanism
                      memory=encoder_outputs, #out of RNN hidden states
                      memory_sequence_length=None,
                      normalize=False,
                      name='BahdanauAttentiion'
                      )
            attn_cell=seq2seq.AttentionWrapper(
                      cell=decoder_cell, #same as encoder
                      attention_mechanism=attn_mech,
                      attention_layer_size=hidden_size, #depth of attention ( output ) tensor
                      output_attention=False,
                      name='attention_wrapper'
                      )#attention layer
            batch_size=tf.shape(encoder_outputs)[0]
            attn_zero=attn_cell.zero_state(batch_size=batch_size,dtype=tf.float32)
            init_state=attn_zero.clone(cell_state=encoder_state)
        if decoder_mode:
            decoder = seq2seq.BeamSearchDecoder(cell=decoder_cell,embedding=embeddings,
                                                start_tokens=tf.tile([GO_ID], [1]),
                                                end_token=EOS_ID,
                                                initial_state=encoder_state[-1],
                                                beam_width=1)#BeamSearch in Decoder
            final_outputs, final_state, final_sequence_lengths =\
                            seq2seq.dynamic_decode(decoder=decoder)
            self.logits = final_outputs.predicted_ids
        else:
            helper = seq2seq.TrainingHelper(inputs=targets_embedding,
                                            sequence_length=self.target_lengths)

            decoder = seq2seq.BasicDecoder(cell=attn_cell,
                                           helper=helper,
                                           initial_state=init_state,
                                           output_layer=Dense(vocab_size))
            final_outputs, final_state, final_sequence_lengths =\
                            seq2seq.dynamic_decode(decoder=decoder)

            self.logits = final_outputs.rnn_output

        if not decoder_mode:
            with tf.variable_scope("loss") as scope:
                #have to pad logits, dynamic decode produces results not consistent
                #in shape with targets
                pad_size = self.max_target_length - tf.reduce_max(final_sequence_lengths)
                self.logits = tf.pad(self.logits, [[0, 0], [0,pad_size], [0, 0]])

                weights = tf.sequence_mask(lengths=final_sequence_lengths,
                                           maxlen=self.max_target_length,
                                           dtype=tf.float32,
                                           name='weights')

                x_entropy_loss = seq2seq.sequence_loss(logits=self.logits,
                                                       targets=self.decoder_targets,
                                                       weights=weights)#cross-entropy loss function

                self.loss = tf.reduce_mean(x_entropy_loss)

            optimizer = tf.train.AdamOptimizer()#Adam optimization algorithm
            gradients = optimizer.compute_gradients(x_entropy_loss)
            capped_grads = [(tf.clip_by_value(grad, -max_gradient_norm, max_gradient_norm), var) for grad, var in gradients]
            self.train_op = optimizer.apply_gradients(capped_grads,
                                                      global_step=self.global_step)
            self.saver = tf.train.Saver(tf.global_variables())

    def step(self, sess, inputs,
             targets, source_lengths,
             target_lengths, test_mode=False):
        '''
        '''
        if test_mode:
            loss = sess.run([self.loss],
                    {self.encoder_inputs : inputs,
                     self.source_lengths : source_lengths,
                     self.decoder_targets : targets,
                     self.target_lengths : target_lengths})
        else:
            _, loss = sess.run([self.train_op, self.loss],
                    {self.encoder_inputs : inputs,
                     self.source_lengths : source_lengths,
                     self.decoder_targets : targets,
                     self.target_lengths : target_lengths})
        return loss

    def test(self, sess, inputs,
             targets, source_lengths,
             target_lengths):

        logits=sess.run([self.logits],
                    {self.encoder_inputs : inputs,
                     self.source_lengths : source_lengths,
                     self.decoder_targets : targets,
                     self.target_lengths : target_lengths})

        return logits

    def get_batch(self, dataset):
        '''
        Obtains batch from dataset
        Inputs: dataset - list of [input, target] sentence pairs

        Outputs:
        source_batch_major- [batch_size x max_sequence_length] inputs
        target_batch_major- [batch_size x max_sequence_length] targets
        source_seq_lengths- list of input seq lengths
        target_seq_lengths- list of target seq lengths
        '''
        source_seq_lengths = []
        target_seq_lengths =[]

        for seq in dataset:
            source_seq_lengths.append(len(seq[0]))
            target_seq_lengths.append(len(seq[1]))

        #max_target_length = max(target_seq_lengths)

        source_batch_major = np.zeros(shape=[len(dataset), self.max_source_length], dtype=np.int32)

        target_batch_major = np.zeros(shape=[len(dataset), self.max_target_length], dtype=np.int32)

        for i, seq in enumerate(dataset):
            for j, element in enumerate(seq[0]):
                source_batch_major[i, j] = element
            for j, element in enumerate(seq[1]):
                target_batch_major[i,j] = element

        return source_batch_major, target_batch_major, source_seq_lengths, target_seq_lengths
<commit_msg>Update chatbot.py<commit_after>'''
Idea from tensorflow translation example:
https://github.com/tensorflow/tensorflow/blob/e39d8feebb9666a331345cd8d960f5ade4652bba/tensorflow/models/rnn/translate/seq2seq_model.py

And neural chatbot example:
https://github.com/inikdom/neural-chatbot/blob/master/models/chatbot.py
'''

import random

import numpy as np
from six.moves import xrange
import tensorflow as tf
from tensorflow.contrib import rnn, seq2seq
from tensorflow.python.layers.core import Dense
import special_vocab as config
import util.vocabutils as vocab_utils

class ChatbotModel(object):
    def __init__(self, vocab_size, hidden_size, dropout,
                 num_layers, max_gradient_norm, batch_size, learning_rate,
                 lr_decay_factor, max_target_length,
                 max_source_length, decoder_mode=False):
        '''
        vocab_size: number of vocab tokens
        buckets: buckets of max sequence lengths
        hidden_size: dimension of hidden layers
        num_layers: number of hidden layers
        max_gradient_norm: maximum gradient magnitude
        batch_size: number of training examples fed to network at once
        learning_rate: starting learning rate of network
        lr_decay_factor: amount by which to decay learning rate
        num_samples: number of samples for sampled softmax
        decoder_mode: Whether to build backpass nodes or not
        '''
        GO_ID = config.GO_ID
        EOS_ID = config.EOS_ID
        self.max_source_length = max_source_length
        self.max_target_length = max_target_length
        self.vocab_size = vocab_size
        self.batch_size = batch_size
        self.global_step = tf.Variable(0, trainable=False)
        self.learning_rate = learning_rate
        self.encoder_inputs = tf.placeholder(shape=(None, None), dtype=tf.int32, name='encoder_inputs')
        self.source_lengths = tf.placeholder(shape=(None,), dtype=tf.int32, name='source_lengths')

        self.decoder_targets = tf.placeholder(shape=(None, None), dtype=tf.int32, name='decoder_targets')
        self.target_lengths = tf.placeholder(shape=(None,), dtype=tf.int32, name="target_lengths")

        with tf.variable_scope('embeddings') as scope:
            embeddings = tf.Variable(tf.random_uniform([vocab_size, hidden_size], -1.0, 1.0), dtype=tf.float32)
            encoder_inputs_embedded = tf.nn.embedding_lookup(embeddings, self.encoder_inputs)
            targets_embedding = tf.nn.embedding_lookup(embeddings, self.decoder_targets)

        with tf.variable_scope('encoder') as scope:
            encoder_cell = rnn.LSTMCell(hidden_size)
            encoder_cell = rnn.DropoutWrapper(encoder_cell,
                                              input_keep_prob=dropout)

            encoder_cell = rnn.MultiRNNCell([encoder_cell] * num_layers)

            encoder_outputs, encoder_state = tf.nn.bidirectional_dynamic_rnn(cell_fw=encoder_cell,
                                                               cell_bw=encoder_cell,
                                                               sequence_length=self.source_lengths,
                                                               inputs=encoder_inputs_embedded,
                                                               dtype=tf.float32,
                                                               time_major=False) #BiLSTM encoder
            encoder_outputs = tf.concat(encoder_outputs,2)
        with tf.variable_scope('decoder') as scope:
            decoder_cell = rnn.LSTMCell(hidden_size)
            decoder_cell = rnn.DropoutWrapper(decoder_cell,
                                              input_keep_prob=dropout)

            decoder_cell = rnn.MultiRNNCell([decoder_cell] * num_layers)

            #TODO add attention
            #attention_mechanism= seq2seq.BahdanauAttention(num_units=hidden_size,memory=encoder_outputs)

            #decoder_cell = seq2seq.AttentionWrapper(cell=decoder_cell,
            #                                        attention_mechanism=)
            attn_mech=seq2seq.BahdanauAttention(
                      num_units=hidden_size,#depth of query mechanism
                      memory=encoder_outputs, #out of RNN hidden states
                      memory_sequence_length=None,
                      normalize=False,
                      name='BahdanauAttentiion'
                      )
            attn_cell=seq2seq.AttentionWrapper(
                      cell=decoder_cell, #same as encoder
                      attention_mechanism=attn_mech,
                      attention_layer_size=hidden_size, #depth of attention ( output ) tensor
                      output_attention=False,
                      name='attention_wrapper'
                      )#attention layer
        if decoder_mode:
            beam_width=2
            batch_size=tf.shape(encoder_outputs)[0]
            attn_zero=attn_cell.zero_state(batch_size=batch_size*beam_width,tf.float32)
            init_state=attn_zero.clone(cell_state=encoder_state)
            decoder = seq2seq.BeamSearchDecoder(cell=decoder_cell,embedding=embeddings,
                                                start_tokens=tf.tile([GO_ID], [1]),
                                                end_token=EOS_ID,
                                                initial_state=encoder_state[-1],
                                                beam_width=beam_width)#BeamSearch in Decoder
            final_outputs, final_state, final_sequence_lengths =\
                            seq2seq.dynamic_decode(decoder=decoder)
            self.logits = final_outputs.predicted_ids
        else:
            batch_size=tf.shape(encoder_outputs)[0]
            attn_zero=attn_cell.zero_state(batch_size=batch_size,dtype=tf.float32)
            init_state=attn_zero.clone(cell_state=encoder_state)
            helper = seq2seq.TrainingHelper(inputs=targets_embedding,
                                            sequence_length=self.target_lengths)

            decoder = seq2seq.BasicDecoder(cell=attn_cell,
                                           helper=helper,
                                           initial_state=init_state,
                                           output_layer=Dense(vocab_size))
            final_outputs, final_state, final_sequence_lengths =\
                            seq2seq.dynamic_decode(decoder=decoder)

            self.logits = final_outputs.rnn_output

        if not decoder_mode:
            with tf.variable_scope("loss") as scope:
                #have to pad logits, dynamic decode produces results not consistent
                #in shape with targets
                pad_size = self.max_target_length - tf.reduce_max(final_sequence_lengths)
                self.logits = tf.pad(self.logits, [[0, 0], [0,pad_size], [0, 0]])

                weights = tf.sequence_mask(lengths=final_sequence_lengths,
                                           maxlen=self.max_target_length,
                                           dtype=tf.float32,
                                           name='weights')

                x_entropy_loss = seq2seq.sequence_loss(logits=self.logits,
                                                       targets=self.decoder_targets,
                                                       weights=weights)#cross-entropy loss function

                self.loss = tf.reduce_mean(x_entropy_loss)

            optimizer = tf.train.AdamOptimizer()#Adam optimization algorithm
            gradients = optimizer.compute_gradients(x_entropy_loss)
            capped_grads = [(tf.clip_by_value(grad, -max_gradient_norm, max_gradient_norm), var) for grad, var in gradients]
            self.train_op = optimizer.apply_gradients(capped_grads,
                                                      global_step=self.global_step)
            self.saver = tf.train.Saver(tf.global_variables())

    def step(self, sess, inputs,
             targets, source_lengths,
             target_lengths, test_mode=False):
        '''
        '''
        if test_mode:
            loss = sess.run([self.loss],
                    {self.encoder_inputs : inputs,
                     self.source_lengths : source_lengths,
                     self.decoder_targets : targets,
                     self.target_lengths : target_lengths})
        else:
            _, loss = sess.run([self.train_op, self.loss],
                    {self.encoder_inputs : inputs,
                     self.source_lengths : source_lengths,
                     self.decoder_targets : targets,
                     self.target_lengths : target_lengths})
        return loss

    def test(self, sess, inputs,
             targets, source_lengths,
             target_lengths):

        logits=sess.run([self.logits],
                    {self.encoder_inputs : inputs,
                     self.source_lengths : source_lengths,
                     self.decoder_targets : targets,
                     self.target_lengths : target_lengths})

        return logits

    def get_batch(self, dataset):
        '''
        Obtains batch from dataset
        Inputs: dataset - list of [input, target] sentence pairs

        Outputs:
        source_batch_major- [batch_size x max_sequence_length] inputs
        target_batch_major- [batch_size x max_sequence_length] targets
        source_seq_lengths- list of input seq lengths
        target_seq_lengths- list of target seq lengths
        '''
        source_seq_lengths = []
        target_seq_lengths =[]

        for seq in dataset:
            source_seq_lengths.append(len(seq[0]))
            target_seq_lengths.append(len(seq[1]))

        #max_target_length = max(target_seq_lengths)

        source_batch_major = np.zeros(shape=[len(dataset), self.max_source_length], dtype=np.int32)

        target_batch_major = np.zeros(shape=[len(dataset), self.max_target_length], dtype=np.int32)

        for i, seq in enumerate(dataset):
            for j, element in enumerate(seq[0]):
                source_batch_major[i, j] = element
            for j, element in enumerate(seq[1]):
                target_batch_major[i,j] = element

        return source_batch_major, target_batch_major, source_seq_lengths, target_seq_lengths
<|endoftext|>


<commit_before># float * speedType[6] = {normalVelFast, stopVelFast, standVelFast, normalVelSlow, stopVelSlow, standVelSlow }
CMD_LIST = {
    'dutyCycle': 0x03,    #expected parameters: int[1], float32[1]
    'setDirectionBit': 0x04, #expected parameters: int[1]
    'removeDirectionBit': 0x05,   #expected parameters: int[1]
    'setMotorVoltage': 0x06,  #expected parameters: int[1], float32[1]
    'setPidParameters': 0x08, #expected parameters: float32[3]
    'setRotatoinSpeed': 0x09, #expected parameters: float32[4]
    'switchOnKinematicCalculation': 0xB,
    'switchOffKinematicCalculation': 0xC,
    'setMovementSpeed': 0xD, #expected parameters: float32[3]
    'switchOnTrajectoryRegulator': 0xE,
    'switchOffTrajectoryRegulator': 0xF,
    'cleanPointsStack': 0x10,
    'getStackState': 0x12,
    'setMovementParameters': 0x15,    #expected parameters: float32[5]
    'setADCPinMode': 0x16,    #expected parameters: int[1], int[1]
    'getADCPinState': 0x17,   #expected parameters: int[1]
    'getAllADCPinsStet': 0x18,
    'getDigitalPinState': 0x19,   #expected parameters: int[1]
    'getAllDigitalPinState': 0x1a,
    'setOutputState': 0x1b,   #expected parameters: int[1]
    'getPinMode': 0x1c,   #expected parameters: int[1]
    'setPinModeExit': 0x1d,   #expected parameters: int[2]
    'getDiscretePinState': 0x1e,  #expected parameters: int[1]
    'getAllDiscretePinStates': 0x1f,
    'setDiscreteOutputState': 0x20,   #expected parameters: int[1]
    'determineCurrentPinMode': 0x21,  #expected parameters: int[1]
    'set12VState': 0x22, #expected parameters: int[1]
    'switchOffPid': 0x23,
    'switchOnPid': 0x24,
    'setCorrectCoordinates': 0x25,

    # TODO': laying field side
    # TODO: beginning of the competition sign
    # TODO: implement commands listed below
    #getManipulatorState = 0x26
    #changeSuckerState = 0x27   #expected parameters: int[1]
    #uploadPuck = 0x28
    #unloadAllPucks = 0x29  #expected parameters: int[1]
    #changeFishingRodState = 0x30   #expected parameters: int[1]
    #changeFishingLatchState = 0x2A #expected parameters: int[1]
    'setManipulatorAngle': 0x31, # expected parameter: float[1]
    'switchOffBelts': 0x33,
    'startGame': 0x34,

    # Small Robot
    'on_sucker':0x3c,
    'off_sucker':0x3d,
    'rotate_cylinder_horizonal':0x42,
    'rotate_cylinder_vertical':0x41,
    'take_cylinder_inside':0x4A,
    'take_cylinder_outside':0x49,
    'lift_up':0x44,
    'store':0x45,
    'out_cylinders':0x48,
    'in+store':0x47,
    #'drop':0x48,
    #Big Robot

    #balls
    'left_ball_down':0x64,
    'left_ball_up':0x65,
    'left_ball_drop':0x66,
    'right_ball_down':0x467,
    'right_ball_up':0x68,
    'right_ball_drop':0x69,
    # cylinders

    # face

    'funny_action':0x77,
    'funny_action_start':0x78,

    # General
    'echo': 0x01,  # expected parameters: char[4] = 'ECHO'
    'setCoordinates': 0x02,  # expected parameters: float32[3]
    'go_to_with_corrections': 0x43,# expected parameters: float32[6], int[1]
    'is_point_was_reached': 0x32,  # no parameters, returns 0 or 1
    'sensors_data': 0x3a,  # no parameters, returns integer with first 6 bits sensor data
    'getCurrentCoordinates': 0x13,
    'getCurrentSpeed': 0x14,
    'addPointToStack': 0x11,  # expected parameters: float32[3], int[1]
    'stopAllMotors': 0x29,

}

REVERSED_CMD_LIST = dict((v,k) for k, v in  CMD_LIST.items())
<commit_msg>Smth deleted<commit_after># float * speedType[6] = {normalVelFast, stopVelFast, standVelFast, normalVelSlow, stopVelSlow, standVelSlow }
CMD_LIST = {
    'dutyCycle': 0x03,    #expected parameters: int[1], float32[1]
    'setDirectionBit': 0x04, #expected parameters: int[1]
    'removeDirectionBit': 0x05,   #expected parameters: int[1]
    'setMotorVoltage': 0x06,  #expected parameters: int[1], float32[1]
    'setPidParameters': 0x08, #expected parameters: float32[3]
    'setRotatoinSpeed': 0x09, #expected parameters: float32[4]
    'switchOnKinematicCalculation': 0xB,
    'switchOffKinematicCalculation': 0xC,
    'setMovementSpeed': 0xD, #expected parameters: float32[3]
    'switchOnTrajectoryRegulator': 0xE,
    'switchOffTrajectoryRegulator': 0xF,
    'cleanPointsStack': 0x10,
    'getStackState': 0x12,
    'setMovementParameters': 0x15,    #expected parameters: float32[5]
    'setADCPinMode': 0x16,    #expected parameters: int[1], int[1]
    'getADCPinState': 0x17,   #expected parameters: int[1]
    'getAllADCPinsStet': 0x18,
    'getDigitalPinState': 0x19,   #expected parameters: int[1]
    'getAllDigitalPinState': 0x1a,
    'setOutputState': 0x1b,   #expected parameters: int[1]
    'getPinMode': 0x1c,   #expected parameters: int[1]
    'setPinModeExit': 0x1d,   #expected parameters: int[2]
    'getDiscretePinState': 0x1e,  #expected parameters: int[1]
    'getAllDiscretePinStates': 0x1f,
    'setDiscreteOutputState': 0x20,   #expected parameters: int[1]
    'determineCurrentPinMode': 0x21,  #expected parameters: int[1]
    'set12VState': 0x22, #expected parameters: int[1]
    'switchOffPid': 0x23,
    'switchOnPid': 0x24,
    'setCorrectCoordinates': 0x25,

    # TODO': laying field side
    # TODO: beginning of the competition sign
    # TODO: implement commands listed below
    #getManipulatorState = 0x26    #changeSuckerState = 0x27   #expected parameters: int[1]
    #uploadPuck = 0x28
    #unloadAllPucks = 0x29  #expected parameters: int[1]
    #changeFishingRodState = 0x30   #expected parameters: int[1]
    #changeFishingLatchState = 0x2A #expected parameters: int[1]
    'setManipulatorAngle': 0x31, # expected parameter: float[1]
    'switchOffBelts': 0x33,
    'startGame': 0x34,

    ## Small Robot
    'on_sucker':0x3c,
    'off_sucker':0x3d,
    'rotate_cylinder_horizonal':0x42,
    'rotate_cylinder_vertical':0x41,
    'take_cylinder_inside':0x4A,
    'take_cylinder_outside':0x49,
    'lift_up':0x44,
    'store':0x45,
    'out_cylinders':0x48,
    'in+store':0x47,
    #'drop':0x48,

    ## Big Robot

    #balls
    'right_ball_down':0x64,
    'right_ball_up':0x65,
    'right_ball_drop':0x66,
    'left_ball_down':0x67,
    'left_ball_up':0x68,
    'left_ball_drop':0x69,
    # cylinders
    'front_down_cylinder_no':0x6A,
    'front_up_cylinder_yes':0x6B,
    'front_drop_cylinder_yes':0x6C,
    'back_down_cylinder_no':0x6D,
    'back_up_cylinder_yes':0x6E,
    'back_drop_cylinder_yes':0x6F,
    'both_sticks_open':0x70,
    'both_sticks_close':0x71,
    # face
    'funny_action':0x77,
    'funny_action_start':0x78,
    # sensors
    'ir_sensors':0x76,
    'us_sencsors':0x75,

    ## General
    'echo': 0x01,  # expected parameters: char[4] = 'ECHO'
    'setCoordinates': 0x02,  # expected parameters: float32[3]
    'go_to_with_corrections': 0x43,# expected parameters: float32[6], int[1]
    'is_point_was_reached': 0x32,  # no parameters, returns 0 or 1
    'sensors_data': 0x3a,  # no parameters, returns integer with first 6 bits sensor data
    'getCurrentCoordinates': 0x13,
    'getCurrentSpeed': 0x14,
    'addPointToStack': 0x11,  # expected parameters: float32[3], int[1]
    'stopAllMotors': 0x29,

}

REVERSED_CMD_LIST = dict((v,k) for k, v in  CMD_LIST.items())
<|endoftext|>


<commit_before>"""
Operator objects that perform some kind of operation using Container objects
as input.

"""
import numpy as np

import MDAnalysis
from MDAnalysis.core.log import ProgressMeter

from Core import *

class Analysis(OperatorCore):
    """Base class for analysis on individual Sim objects.

    """

    def __init__(self, *args, **kwargs):
        """Generate analysis object.

        Analysis objects perform an analysis operation on individual Sims,
        looping through the trajectory to look at each timestep.

        When multiple Sims are loaded, each Analysis method operates on them
        in parallel using separate processes.

        Important methods::

        :method:`run()` - perform time-consuming trajectory-loop data
                          collection

        :method:`analyze()` - post-collection operations that do not directly
                              require trajectory information

        This class is meant to be a parent class to a specific analysis for
        specific Sim-derived objects. When inheriting, one need only replace::

        :method:`_run_container_pre()`
        :method:`_run_container_loop()`
        :method:`_run_container_post()`
        :method:`analyze()`

        at minimum to implement specific results.

        :Arguments:
            *args
                Sim (or Sim-derived) objects to analyze

        """
        super(Analysis, self).__init__(*args, **kwargs)

    def _run_container(self, container, **kwargs):
        """Run timeseries collection for single container.

        :Arguments:
            *container*
                Sim (or Sim-derived) object

        :Keywords:
            *universe*
                key of universe to use for analysis [``'main'``]

            **kwargs passed to :method:`_run_container_pre()`
                               :method:`_run_container_loop()`
                               :method:`_run_container_post()`

        """
        ukey = kwargs.pop('universe', 'main')
        container._logger.info("Running {} analysis on '{}'...".format(self.__class__.__name__, container.metadata['name']))

        # set up data storage structure
        con_results = {'time': np.zeros((len(container.universes[ukey].trajectory),), dtype=float),
                      }

        self._run_container_pre(container, con_results, **kwargs)

        # iterate through trajectory; collect raw data
        container._logger.info("Collecting timeseries...")
        pm = ProgressMeter(container.universes[ukey].trajectory.numframes, interval=100)
        container.universes[ukey].trajectory[0]
        for ts in container.universes[ukey].trajectory:
            pm.echo(ts.frame)
            con_results['time'][container.universes[ukey].trajectory.frame - 1] = container.universes[ukey].trajectory.time
            self._run_container_loop(container, con_results, **kwargs)

        self._run_container_post(container, con_results, **kwargs)
        self._save(container, con_results)

    def _run_container_pre(self, container, con_results, **kwargs):
        """Operations to be performed before run loop.

        :Arguments:
            *container*
                Sim (or Sim-derived) object
            *con_results*
                dict storing results for *container*

        :Keywords:

        """
        return

    def _run_container_loop(self, container, con_results, **kwargs):
        """Operations to be performed inside of run loop.

        :Arguments:
            *container*
                Sim (or Sim-derived) object
            *con_results*
                dict storing results for *container*

        :Keywords:

        """
        return

    def _run_container_post(self, container, con_results, **kwargs):
        """Operations to be performed after run loop.

        :Arguments:
            *container*
                Sim (or Sim-derived) object
            *con_results*
                dict storing results for *container*

        :Keywords:

        """
        return

class MetaAnalysis(OperatorCore):
    """Base class for analysis on Group objects.

    """
    def __init__(self, *args, **kwargs):
        """Generate meta-analysis object.

        MetaAnalysis objects perform analysis operations on Groups, allowing
        for direct comparison of trajectories to distill larger patterns.

        When multiple Groups are loaded, each MetaAnalysis method operates on
        them in parallel using separate processes.

        Important methods::

        :method:`run()` - perform time-consuming trajectory-loop data
                          collection

        :method:`analyze()` - post-collection operations that do not directly
                              require trajectory information

        This class is meant to be a parent class to a specific analysis for
        specific Group-derived objects. When inheriting, one need only replace::

        :method:`_run_container_pre()`
        :method:`_run_container_loop()`
        :method:`_run_container_post()`
        :method:`analyze()`

        at minimum to implement specific results.

        :Arguments:
            *args
                Group (or Group-derived) objects to analyze
        """
        super(MetaAnalysis, self).__init__(*args, **kwargs)
<commit_msg>Needed to attach a universe before running Analysis.<commit_after>"""
Operator objects that perform some kind of operation using Container objects
as input.

"""
import numpy as np

import MDAnalysis
from MDAnalysis.core.log import ProgressMeter

from Core import *

class Analysis(OperatorCore):
    """Base class for analysis on individual Sim objects.

    """

    def __init__(self, *args, **kwargs):
        """Generate analysis object.

        Analysis objects perform an analysis operation on individual Sims,
        looping through the trajectory to look at each timestep.

        When multiple Sims are loaded, each Analysis method operates on them
        in parallel using separate processes.

        Important methods::

        :method:`run()` - perform time-consuming trajectory-loop data
                          collection

        :method:`analyze()` - post-collection operations that do not directly
                              require trajectory information

        This class is meant to be a parent class to a specific analysis for
        specific Sim-derived objects. When inheriting, one need only replace::

        :method:`_run_container_pre()`
        :method:`_run_container_loop()`
        :method:`_run_container_post()`
        :method:`analyze()`

        at minimum to implement specific results.

        :Arguments:
            *args
                Sim (or Sim-derived) objects to analyze

        """
        super(Analysis, self).__init__(*args, **kwargs)

    def _run_container(self, container, **kwargs):
        """Run timeseries collection for single container.

        :Arguments:
            *container*
                Sim (or Sim-derived) object

        :Keywords:
            *universe*
                key of universe to use for analysis [``'main'``]

            **kwargs passed to :method:`_run_container_pre()`
                               :method:`_run_container_loop()`
                               :method:`_run_container_post()`

        """
        ukey = kwargs.pop('universe', 'main')
        container._logger.info("Running {} analysis on '{}'...".format(self.__class__.__name__, container.metadata['name']))

        container.attach(ukey)

        # set up data storage structure
        con_results = {'time': np.zeros((len(container.universes[ukey].trajectory),), dtype=float),
                      }

        self._run_container_pre(container, con_results, **kwargs)

        # iterate through trajectory; collect raw data
        container._logger.info("Collecting timeseries...")
        pm = ProgressMeter(container.universes[ukey].trajectory.numframes, interval=100)
        container.universes[ukey].trajectory[0]
        for ts in container.universes[ukey].trajectory:
            pm.echo(ts.frame)
            con_results['time'][container.universes[ukey].trajectory.frame - 1] = container.universes[ukey].trajectory.time
            self._run_container_loop(container, con_results, **kwargs)

        self._run_container_post(container, con_results, **kwargs)
        self._save(container, con_results)

    def _run_container_pre(self, container, con_results, **kwargs):
        """Operations to be performed before run loop.

        :Arguments:
            *container*
                Sim (or Sim-derived) object
            *con_results*
                dict storing results for *container*

        :Keywords:

        """
        return

    def _run_container_loop(self, container, con_results, **kwargs):
        """Operations to be performed inside of run loop.

        :Arguments:
            *container*
                Sim (or Sim-derived) object
            *con_results*
                dict storing results for *container*

        :Keywords:

        """
        return

    def _run_container_post(self, container, con_results, **kwargs):
        """Operations to be performed after run loop.

        :Arguments:
            *container*
                Sim (or Sim-derived) object
            *con_results*
                dict storing results for *container*

        :Keywords:

        """
        return

class MetaAnalysis(OperatorCore):
    """Base class for analysis on Group objects.

    """
    def __init__(self, *args, **kwargs):
        """Generate meta-analysis object.

        MetaAnalysis objects perform analysis operations on Groups, allowing
        for direct comparison of trajectories to distill larger patterns.

        When multiple Groups are loaded, each MetaAnalysis method operates on
        them in parallel using separate processes.

        Important methods::

        :method:`run()` - perform time-consuming trajectory-loop data
                          collection

        :method:`analyze()` - post-collection operations that do not directly
                              require trajectory information

        This class is meant to be a parent class to a specific analysis for
        specific Group-derived objects. When inheriting, one need only replace::

        :method:`_run_container_pre()`
        :method:`_run_container_loop()`
        :method:`_run_container_post()`
        :method:`analyze()`

        at minimum to implement specific results.

        :Arguments:
            *args
                Group (or Group-derived) objects to analyze
        """
        super(MetaAnalysis, self).__init__(*args, **kwargs)
<|endoftext|>