<commit_before># Copyright (c) 2011, Enthought, Ltd.
# Author: Pietro Berkes <pberkes@enthought.com>
# License: Modified BSD license (2-clause)

import unittest
import numpy as np
from numpy import testing
from pyanno import ModelBt
from pyanno.util import MISSING_VALUE as MV, is_valid, PyannoValueError

class TestModelBt(unittest.TestCase):

    def test_mle_estimation(self):
        # test simple model, check that we get to global optimum
        nclasses, nitems = 3, 500*8
        # create random model and data (this is our ground truth model)
        true_model = ModelBt.create_initial_state(nclasses)
        annotations = true_model.generate_annotations(nitems)

        # create a new, empty model and infer back the parameters
        model = ModelBt.create_initial_state(nclasses)
        before_llhood = model.log_likelihood(annotations)
        model.mle(annotations)
        after_llhood = model.log_likelihood(annotations)

        testing.assert_allclose(model.gamma, true_model.gamma, atol=1e-1, rtol=0.)
        testing.assert_allclose(model.theta, true_model.theta, atol=1e-1, rtol=0.)
        self.assertGreater(after_llhood, before_llhood)

    def test_map_estimation(self):
        # test simple model, check that we get to global optimum
        nclasses, nitems = 3, 500*8
        # create random model and data (this is our ground truth model)
        true_model = ModelBt.create_initial_state(nclasses)
        annotations = true_model.generate_annotations(nitems)

        # create a new, empty model and infer back the parameters
        model = ModelBt.create_initial_state(nclasses)
        before_obj = model.log_likelihood(annotations) + model._log_prior()
        model.map(annotations)
        after_obj = model.log_likelihood(annotations) + model._log_prior()

        testing.assert_allclose(model.gamma, true_model.gamma, atol=1e-1, rtol=0.)
        testing.assert_allclose(model.theta, true_model.theta, atol=1e-1, rtol=0.)
        self.assertGreater(after_obj, before_obj)

    def test_log_likelihood(self):
        # check that log likelihood is maximal at true parameters
        nclasses, nitems = 3, 1500*8
        # create random model and data (this is our ground truth model)
        true_model = ModelBt.create_initial_state(nclasses)
        annotations = true_model.generate_annotations(nitems)

        max_llhood = true_model.log_likelihood(annotations)
        # perturb gamma
        for _ in xrange(20):
            theta = true_model.theta
            gamma = np.random.normal(loc=true_model.gamma, scale=0.1)
            gamma = np.clip(gamma, 0., 1.)
            gamma /= gamma.sum()
            model = ModelBt(nclasses, gamma, theta)
            llhood = model.log_likelihood(annotations)
            self.assertGreater(max_llhood, llhood)

        # perturb theta
        for _ in xrange(20):
            gamma = true_model.gamma
            theta = np.random.normal(loc=true_model.theta, scale=0.1)
            theta = np.clip(theta, 0., 1.)
            model = ModelBt(nclasses, gamma, theta)
            llhood = model.log_likelihood(annotations)
            self.assertGreater(max_llhood, llhood)

    def test_sampling_theta(self):
        nclasses, nitems = 3, 500*8
        nsamples = 1000

        # create random model (this is our ground truth model)
        true_model = ModelBt.create_initial_state(nclasses)
        # create random data
        annotations = true_model.generate_annotations(nitems)

        # create a new model
        model = ModelBt.create_initial_state(nclasses)
        # get optimal parameters (to make sure we're at the optimum)
        model.mle(annotations)

        # modify parameters, to give false start to sampler
        real_theta = model.theta.copy()
        model.theta = model._random_theta(model.nannotators)
        # save current parameters
        gamma_before, theta_before = model.gamma.copy(), model.theta.copy()
        samples = model.sample_posterior_over_accuracy(
            annotations,
            nsamples,
            burn_in_samples=100,
            thin_samples=2
        )
        # test: the mean of the sampled parameters is the same as the MLE one
        # (up to 3 standard deviations of the estimate sample distribution)
        testing.assert_array_less(np.absolute(samples.mean(0)-real_theta),
                                  3.*samples.std(0))

        # check that original parameters are intact
        testing.assert_equal(model.gamma, gamma_before)
        testing.assert_equal(model.theta, theta_before)

    def test_inference(self):
        # perfect annotation, check that inferred label is correct
        nclasses, nitems = 3, 50*8

        # create random model (this is our ground truth model)
        gamma = np.ones((nclasses,)) / float(nclasses)
        theta = np.ones((8,)) * 0.999
        true_model = ModelBt(nclasses, gamma, theta)
        # create random data
        labels = true_model.generate_labels(nitems)
        annotations = true_model.generate_annotations_from_labels(labels)

        posterior = true_model.infer_labels(annotations)
        testing.assert_allclose(posterior.sum(1), 1., atol=1e-6, rtol=0.)
        inferred = posterior.argmax(1)

        testing.assert_equal(inferred, labels)
        self.assertTrue(np.all(posterior[np.arange(nitems),inferred] > 0.999))

        # at chance annotation, disagreeing annotators: get back prior
        gamma = ModelBt._random_gamma(nclasses)
        theta = np.ones((8,)) / float(nclasses)
        model = ModelBt(nclasses, gamma, theta)

        data = np.array([[MV, 0, 1, 2, MV, MV, MV, MV,]])
        testing.assert_almost_equal(np.squeeze(model.infer_labels(data)),
                                    model.gamma, 6)

    def test_generate_annotations(self):
        # test to check that annotations are masked correctly when the number
        # of items is not divisible by the number of annotators
        nclasses, nitems = 5, 8*30+3

        model = ModelBt.create_initial_state(nclasses)
        annotations = model.generate_annotations(nitems)

        valid = is_valid(annotations)
        # check that on every row there are exactly 3 annotations
        self.assertTrue(np.all(valid.sum(1) == 3))

    def test_annotations_compatibility(self):
        nclasses = 3
        model = ModelBt.create_initial_state(nclasses)

        # test method that checks annotations compatibility
        anno = np.array([[MV, MV, 0, 0, 1, MV, MV, MV]])
        self.assertTrue(model.are_annotations_compatible(anno))

        anno = np.array([[MV, MV, 0, 0, 1, MV, MV, MV, MV]])
        self.assertFalse(model.are_annotations_compatible(anno))

        anno = np.array([[MV, MV, 0, 0, 3, MV, MV, MV]])
        self.assertFalse(model.are_annotations_compatible(anno))

        anno = np.array([[MV, MV, 0, 0, 2, 1, MV, MV]])
        self.assertFalse(model.are_annotations_compatible(anno))

        anno = np.array([[0, 0, MV, -2, MV, MV, MV, MV]])
        self.assertFalse(model.are_annotations_compatible(anno))

    def test_raise_error_on_incompatible_annotation(self):
        nclasses = 3
        model = ModelBt.create_initial_state(nclasses)
        anno = np.array([[MV, MV, 0, 0, 7, MV, MV, MV]])

        with self.assertRaises(PyannoValueError):
            model.mle(anno)

        with self.assertRaises(PyannoValueError):
            model.map(anno)

        with self.assertRaises(PyannoValueError):
            model.sample_posterior_over_accuracy(anno, 10)

        with self.assertRaises(PyannoValueError):
            model.infer_labels(anno)

        with self.assertRaises(PyannoValueError):
            model.log_likelihood(anno)

if __name__ == '__main__':
    unittest.main()
<commit_msg>NEW:tests for model Bt (general)<commit_after># Copyright (c) 2011, Enthought, Ltd.
# Author: Pietro Berkes <pberkes@enthought.com>
# License: Modified BSD license (2-clause)

import unittest
import numpy as np
from numpy import testing
from pyanno import ModelBt
from pyanno.modelBt_loopdesign import ModelBtLoopDesign
from pyanno.util import MISSING_VALUE as MV, is_valid, PyannoValueError, labels_frequency

class TestModelBt(unittest.TestCase):

    def test_create_model(self):
        nclasses = 8
        nannotators = 32
        model = ModelBt.create_initial_state(nclasses, nannotators)

        self.assertEqual(model.nannotators, nannotators)
        self.assertEqual(model.gamma.shape[0], nclasses)
        self.assertEqual(model.theta.shape[0], nannotators)

    def test_mle_estimation(self):
        # test simple model, check that we get to global optimum
        nclasses, nannotators, nitems = 3, 5, 5000
        # create random model and data (this is our ground truth model)
        true_model = ModelBt.create_initial_state(nclasses, nannotators)
        annotations = true_model.generate_annotations(nitems)

        # create a new, empty model and infer back the parameters
        model = ModelBt.create_initial_state(nclasses, nannotators)
        before_llhood = model.log_likelihood(annotations)
        model.mle(annotations)
        after_llhood = model.log_likelihood(annotations)

        testing.assert_allclose(model.gamma, true_model.gamma,
                                atol=0.05, rtol=0.)
        testing.assert_allclose(model.theta, true_model.theta,
                                atol=0.05, rtol=0.)
        self.assertGreater(after_llhood, before_llhood)

    def test_map_estimation(self):
        # test simple model, check that we get to global optimum
        nclasses, nannotators, nitems = 3, 5, 5000
        # create random model and data (this is our ground truth model)
        true_model = ModelBt.create_initial_state(nclasses, nannotators)
        annotations = true_model.generate_annotations(nitems)

        # create a new, empty model and infer back the parameters
        model = ModelBt.create_initial_state(nclasses, nannotators)
        before_obj = model.log_likelihood(annotations) + model._log_prior()
        model.map(annotations)
        after_obj = model.log_likelihood(annotations) + model._log_prior()

        testing.assert_allclose(model.gamma, true_model.gamma,
                                atol=0.05, rtol=0.)
        testing.assert_allclose(model.theta, true_model.theta,
                                atol=0.05, rtol=0.)
        self.assertGreater(after_obj, before_obj)

    def test_log_likelihood_loop_design(self):
        # behavior: the log likelihood of the new class should match the one
        # of the more specialized class
        nclasses, nannotators, nitems = 4, 8, 100

        # create specialized model, draw data
        true_model = ModelBtLoopDesign.create_initial_state(nclasses)
        annotations = true_model.generate_annotations(nitems)
        expect = true_model.log_likelihood(annotations)

        model = ModelBt(nclasses, nannotators,
                        gamma=true_model.gamma, theta=true_model.theta)
        llhood = model.log_likelihood(annotations)

        np.testing.assert_almost_equal(llhood, expect, 10)

    def test_log_likelihood(self):
        # check that log likelihood is maximal at true parameters
        nclasses, nannotators, nitems = 3, 5, 1000
        # create random model and data (this is our ground truth model)
        true_model = ModelBt.create_initial_state(nclasses, nannotators)
        annotations = true_model.generate_annotations(nitems)

        max_llhood = true_model.log_likelihood(annotations)

        # perturb gamma
        for _ in xrange(20):
            theta = true_model.theta
            gamma = np.random.normal(loc=true_model.gamma, scale=0.1)
            gamma = np.clip(gamma, 0., 1.)
            gamma /= gamma.sum()
            model = ModelBt(nclasses, nannotators, gamma, theta)
            llhood = model.log_likelihood(annotations)
            self.assertGreater(max_llhood, llhood)

        # perturb theta
        for _ in xrange(20):
            gamma = true_model.gamma
            theta = np.random.normal(loc=true_model.theta, scale=0.1)
            theta = np.clip(theta, 0., 1.)
            model = ModelBt(nclasses, nannotators, gamma, theta)
            llhood = model.log_likelihood(annotations)
            self.assertGreater(max_llhood, llhood)

    def test_sampling_theta(self):
        nclasses, nannotators, nitems = 3, 5, 5000
        nsamples = 1000

        # create random model (this is our ground truth model)
        true_model = ModelBt.create_initial_state(nclasses, nannotators)
        # create random data
        annotations = true_model.generate_annotations(nitems)

        # create a new model
        model = ModelBt.create_initial_state(nclasses, nannotators)
        # get optimal parameters (to make sure we're at the optimum)
        model.map(annotations)

        # modify parameters, to give false start to sampler
        real_theta = model.theta.copy()
        model.theta = model._random_theta(model.nannotators)
        # save current parameters
        gamma_before, theta_before = model.gamma.copy(), model.theta.copy()
        samples = model.sample_posterior_over_accuracy(
            annotations,
            nsamples,
            burn_in_samples=100,
            thin_samples=2
        )
        # test: the mean of the sampled parameters is the same as the MLE one
        # (up to 3 standard deviations of the estimate sample distribution)
        testing.assert_array_less(np.absolute(samples.mean(0)-real_theta),
                                  3.*samples.std(0))

        # check that original parameters are intact
        testing.assert_equal(model.gamma, gamma_before)
        testing.assert_equal(model.theta, theta_before)

    def test_inference(self):
        # perfect annotation, check that inferred label is correct
        nclasses, nannotators, nitems = 3, 5, 50*8

        # create random model (this is our ground truth model)
        gamma = np.ones((nclasses,)) / float(nclasses)
        theta = np.ones((8,)) * 0.999
        true_model = ModelBt(nclasses, nannotators, gamma, theta)
        # create random data
        labels = true_model.generate_labels(nitems)
        annotations = true_model.generate_annotations_from_labels(labels)

        posterior = true_model.infer_labels(annotations)
        testing.assert_allclose(posterior.sum(1), 1., atol=1e-6, rtol=0.)
        inferred = posterior.argmax(1)

        testing.assert_equal(inferred, labels)
        self.assertTrue(np.all(posterior[np.arange(nitems),inferred] > 0.999))

        # at chance annotation, disagreeing annotators: get back prior
        gamma = ModelBt._random_gamma(nclasses)
        theta = np.ones((nannotators,)) / float(nclasses)
        model = ModelBt(nclasses, nannotators, gamma, theta)

        data = np.array([[MV, 0, 1, 2, MV]])
        testing.assert_almost_equal(np.squeeze(model.infer_labels(data)),
                                    model.gamma, 6)

    def test_generate_annotations(self):
        # test to check that annotations are masked correctly when the number
        # of items is not divisible by the number of annotators
        nclasses, nannotators, nitems = 5, 7, 201

        model = ModelBt.create_initial_state(nclasses, nannotators)
        annotations = model.generate_annotations(nitems)

        valid = is_valid(annotations)
        self.assertEqual(annotations.shape, (nitems, nannotators))
        model.are_annotations_compatible(annotations)

        # perfect annotators, annotations correspond to prior
        nitems = 20000
        model.theta[:] = 1.
        annotations = model.generate_annotations(nitems)
        freq = labels_frequency(annotations, nclasses)
        np.testing.assert_almost_equal(freq, model.gamma, 2)

    def test_annotations_compatibility(self):
        nclasses = 3
        nannotators = 5
        model = ModelBt.create_initial_state(nclasses, nannotators)

        # test method that checks annotations compatibility
        anno = np.array([[0, 1, MV, MV, MV]])
        self.assertTrue(model.are_annotations_compatible(anno))

        anno = np.array([[0, 0, 0, 0]])
        self.assertFalse(model.are_annotations_compatible(anno))

        anno = np.array([[4, 0, 0, 0, 0]])
        self.assertFalse(model.are_annotations_compatible(anno))

        anno = np.array([[-2, MV, MV, MV, MV]])
        self.assertFalse(model.are_annotations_compatible(anno))

    def test_raise_error_on_incompatible_annotation(self):
        nclasses, nannotators = 3, 7
        model = ModelBt.create_initial_state(nclasses, nannotators)
        anno = np.array([[MV, MV, 0, 0, 7, MV, MV]])

        with self.assertRaises(PyannoValueError):
            model.mle(anno)

        with self.assertRaises(PyannoValueError):
            model.map(anno)

        with self.assertRaises(PyannoValueError):
            model.sample_posterior_over_accuracy(anno, 10)

        with self.assertRaises(PyannoValueError):
            model.infer_labels(anno)

        with self.assertRaises(PyannoValueError):
            model.log_likelihood(anno)

    def test_missing_annotations(self):
        # test simple model, check that we get to global optimum

        nclasses, nannotators, nitems = 2, 3, 10000
        # create random model and data (this is our ground truth model)
        true_model = ModelBt.create_initial_state(nclasses, nannotators)
        annotations = true_model.generate_annotations(nitems)
        # remove about 10% of the annotations
        for _ in range(nitems*nannotators//10):
            i = np.random.randint(nitems)
            j = np.random.randint(nannotators)
            annotations[i,j] = MV

        # create a new, empty model and infer back the parameters
        model = ModelBt.create_initial_state(nclasses, nannotators)
        before_llhood = (model.log_likelihood(annotations)
                         + model._log_prior(model.theta))
        model.map(annotations)
        after_llhood = (model.log_likelihood(annotations)
                        + model._log_prior(model.theta))

        testing.assert_allclose(model.gamma, true_model.gamma,
                                atol=1e-1, rtol=0.)
        testing.assert_allclose(model.theta, true_model.theta,
                                atol=1e-1, rtol=0.)
        self.assertGreater(after_llhood, before_llhood)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>class StateMask:

    def __init__(self, bit, doc=None):
        self.bit = bit
        if doc is not None:
            self.__doc__ = 'bit {}.  {}'.format(bit, doc)

    def __get__(self, obj, type_=None):
        if obj is None:
            return self
        else:
            return bool(obj._data >> self.bit & 1)

    def __set__(self, obj, value):
        raise AttributeError('{!r} of {!r} is not settable'.format(self, obj))

class DroneState:

    def __init__(self, data):
        self._data = data

    fly_mask = StateMask(
        0,
        'FLY MASK : (0) ardrone is landed, (1) ardrone is flying'
    )

    video_mask = StateMask(
        1,
        'VIDEO MASK : (0) video disable, (1) video enable'
    )

    vision_mask = StateMask(
        2,
        'VISION MASK : (0) vision disable, (1) vision enable'
    )

    control_mask = StateMask(
        3,
        'CONTROL ALGO : (0) euler angles control, (1) angular speed control'
    )

    altitude_mask = StateMask(
        4,
        'ALTITUDE CONTROL ALGO : '
        '(0) altitude control inactive (1) altitude control active'
    )

    user_feedback_start = StateMask(
        5,
        'USER feedback : Start button state'
    )

    command_mask = StateMask(
        6,
        'Control command ACK : (0) None, (1) one received'
    )

    camera_mask = StateMask(
        7,
        'CAMERA MASK : (0) camera not ready, (1) Camera ready'
    )

    travelling_mask = StateMask(
        8,
        'Travelling mask : (0) disable, (1) enable'
    )

    usb_mask = StateMask(
        9,
        'USB key : (0) usb key not ready, (1) usb key ready'
    )

    navdata_demo_mask = StateMask(
        10,
        'Navdata demo : (0) All navdata, (1) only navdata demo'
    )

    navdata_bootstrap = StateMask(
        11,
        'Navdata bootstrap : '
        '(0) options sent in all or demo mode, (1) no navdata options sent'
    )

    motors_mask = StateMask(
        12,
        'Motors status : (0) Ok, (1) Motors problem'
    )

    com_lost_mask = StateMask(
        13,
        'Communication Lost : (1) com problem, (0) Com is ok'
    )

    software_fault = StateMask(
        14,
        'Software fault detected - user should land as quick as possible (1)'
    )

    vbat_low = StateMask(
        15,
        'VBat low : (1) too low, (0) Ok'
    )

    user_el = StateMask(
        16,
        'User Emergency Landing : (1) User EL is ON, (0) User EL is OFF'
    )

    timer_elapsed = StateMask(
        17,
        'Timer elapsed : (1) elapsed, (0) not elapsed'
    )

    magneto_needs_calib = StateMask(
        18,
        'Magnetometer calibration state : '
        '(0) Ok, no calibration needed, (1) not ok, calibration needed'
    )

    angles_out_of_range = StateMask(
        19,
        'Angles : (0) Ok, (1) out of range'
    )

    wind_mask = StateMask(
        20,
        'WIND MASK: (0) ok, (1) Too much wind'
    )

    ultrasound_mask = StateMask(
        21,
        'Ultrasonic sensor : (0) Ok, (1) deaf'
    )

    cutout_mask = StateMask(
        22,
        'Cutout system detection : (0) Not detected, (1) detected'
    )

    pic_version_mask = StateMask(
        23,
        'PIC Version number OK : '
        '(0) a bad version number, (1) version number is OK'
    )

    atcodec_thread_on = StateMask(
        24,
        'ATCodec thread ON : (0) thread OFF (1) thread ON'
    )

    navdata_thread_on = StateMask(
        25,
        'Navdata thread ON : (0) thread OFF (1) thread ON'
    )

    video_thread_on = StateMask(
        26,
        'Video thread ON : (0) thread OFF (1) thread ON'
    )

    acq_thread_on = StateMask(
        27,
        'Acquisition thread ON : (0) thread OFF (1) thread ON'
    )

    ctrl_watchdog_mask = StateMask(
        28,
        'CTRL watchdog : '
        '(1) delay in control execution (> 5ms), (0) control is well scheduled'
    )

    adc_watchdog_mask = StateMask(
        29,
        'ADC Watchdog : (1) delay in uart2 dsr (> 5ms), (0) uart2 is good'
    )

    com_watchdog_mask = StateMask(
        30,
        'Communication Watchdog : (1) com problem, (0) Com is ok'
    )

    emergency_mask = StateMask(
        31,
        'Emergency landing : (0) no emergency, (1) emergency'
    )
<commit_msg>improve DroneState.__repr__<commit_after>class StateMask:

    def __init__(self, bit, doc=None):
        self.bit = bit
        if doc is not None:
            self.__doc__ = 'bit {}.  {}'.format(bit, doc)

    def __get__(self, obj, type_=None):
        if obj is None:
            return self
        else:
            return bool(obj._data >> self.bit & 1)

    def __set__(self, obj, value):
        raise AttributeError('{!r} of {!r} is not settable'.format(self, obj))

class DroneState:

    def __init__(self, data):
        self._data = data

    def __repr__(self):
        return '{self.__class__.__name__}(0b{self._data:b})'.format(self=self)

    fly_mask = StateMask(
        0,
        'FLY MASK : (0) ardrone is landed, (1) ardrone is flying'
    )

    video_mask = StateMask(
        1,
        'VIDEO MASK : (0) video disable, (1) video enable'
    )

    vision_mask = StateMask(
        2,
        'VISION MASK : (0) vision disable, (1) vision enable'
    )

    control_mask = StateMask(
        3,
        'CONTROL ALGO : (0) euler angles control, (1) angular speed control'
    )

    altitude_mask = StateMask(
        4,
        'ALTITUDE CONTROL ALGO : '
        '(0) altitude control inactive (1) altitude control active'
    )

    user_feedback_start = StateMask(
        5,
        'USER feedback : Start button state'
    )

    command_mask = StateMask(
        6,
        'Control command ACK : (0) None, (1) one received'
    )

    camera_mask = StateMask(
        7,
        'CAMERA MASK : (0) camera not ready, (1) Camera ready'
    )

    travelling_mask = StateMask(
        8,
        'Travelling mask : (0) disable, (1) enable'
    )

    usb_mask = StateMask(
        9,
        'USB key : (0) usb key not ready, (1) usb key ready'
    )

    navdata_demo_mask = StateMask(
        10,
        'Navdata demo : (0) All navdata, (1) only navdata demo'
    )

    navdata_bootstrap = StateMask(
        11,
        'Navdata bootstrap : '
        '(0) options sent in all or demo mode, (1) no navdata options sent'
    )

    motors_mask = StateMask(
        12,
        'Motors status : (0) Ok, (1) Motors problem'
    )

    com_lost_mask = StateMask(
        13,
        'Communication Lost : (1) com problem, (0) Com is ok'
    )

    software_fault = StateMask(
        14,
        'Software fault detected - user should land as quick as possible (1)'
    )

    vbat_low = StateMask(
        15,
        'VBat low : (1) too low, (0) Ok'
    )

    user_el = StateMask(
        16,
        'User Emergency Landing : (1) User EL is ON, (0) User EL is OFF'
    )

    timer_elapsed = StateMask(
        17,
        'Timer elapsed : (1) elapsed, (0) not elapsed'
    )

    magneto_needs_calib = StateMask(
        18,
        'Magnetometer calibration state : '
        '(0) Ok, no calibration needed, (1) not ok, calibration needed'
    )

    angles_out_of_range = StateMask(
        19,
        'Angles : (0) Ok, (1) out of range'
    )

    wind_mask = StateMask(
        20,
        'WIND MASK: (0) ok, (1) Too much wind'
    )

    ultrasound_mask = StateMask(
        21,
        'Ultrasonic sensor : (0) Ok, (1) deaf'
    )

    cutout_mask = StateMask(
        22,
        'Cutout system detection : (0) Not detected, (1) detected'
    )

    pic_version_mask = StateMask(
        23,
        'PIC Version number OK : '
        '(0) a bad version number, (1) version number is OK'
    )

    atcodec_thread_on = StateMask(
        24,
        'ATCodec thread ON : (0) thread OFF (1) thread ON'
    )

    navdata_thread_on = StateMask(
        25,
        'Navdata thread ON : (0) thread OFF (1) thread ON'
    )

    video_thread_on = StateMask(
        26,
        'Video thread ON : (0) thread OFF (1) thread ON'
    )

    acq_thread_on = StateMask(
        27,
        'Acquisition thread ON : (0) thread OFF (1) thread ON'
    )

    ctrl_watchdog_mask = StateMask(
        28,
        'CTRL watchdog : '
        '(1) delay in control execution (> 5ms), (0) control is well scheduled'
    )

    adc_watchdog_mask = StateMask(
        29,
        'ADC Watchdog : (1) delay in uart2 dsr (> 5ms), (0) uart2 is good'
    )

    com_watchdog_mask = StateMask(
        30,
        'Communication Watchdog : (1) com problem, (0) Com is ok'
    )

    emergency_mask = StateMask(
        31,
        'Emergency landing : (0) no emergency, (1) emergency'
    )
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import os
import fsquery

def puaq():
    print("Usage: %s base_path." % os.path.basename(__file__))
    sys.exit(1)

def __filter_git_only(thelist):
    ret = []
    for d in thelist:
        if os.path.basename(d) == ".git":
            ret.append(d)
    return ret

def make_repo_list(path):
    ret = fsquery.makecontentlist(path, True, False, True, False, True, [])
    ret = __filter_git_only(ret)
    return ret

def get_remotes(repo):
    out = check_output(["git", "--git-dir=%s" % repo, "--work-tree=%s" % os.path.dirname(repo), "remote"])
    ret_list = out.split()
    return ret_list

if __name__ == "__main__":

    if len(sys.argv) < 2:
        puaq()

    basepath = sys.argv[1]
    if not os.path.exists(basepath):
        print("%s does not exist. Aborting." % basepath)
        sys.exit(1)

    for r in make_repo_list(basepath):
        print(r)

<commit_msg>adding dependency<commit_after>#!/usr/bin/env python

import sys
import os
import fsquery
from subprocess import check_output

def puaq():
    print("Usage: %s base_path." % os.path.basename(__file__))
    sys.exit(1)

def __filter_git_only(thelist):
    ret = []
    for d in thelist:
        if os.path.basename(d) == ".git":
            ret.append(d)
    return ret

def make_repo_list(path):
    ret = fsquery.makecontentlist(path, True, False, True, False, True, [])
    ret = __filter_git_only(ret)
    return ret

def get_remotes(repo):
    out = check_output(["git", "--git-dir=%s" % repo, "--work-tree=%s" % os.path.dirname(repo), "remote"])
    ret_list = out.split()
    return ret_list

if __name__ == "__main__":

    if len(sys.argv) < 2:
        puaq()

    basepath = sys.argv[1]
    if not os.path.exists(basepath):
        print("%s does not exist. Aborting." % basepath)
        sys.exit(1)

    for r in make_repo_list(basepath):
        print(r)

<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import os
import subprocess

import git_visitor_base
import git_repo_query

def visitor_push(repos):

    ORIGINAL_COLOR = "\033[0m" # mvtodo: would be better to try to detect the terminal's current standard color

    report = []
    for rp in repos:
        remotes = git_repo_query.get_remotes(rp)
        for rm in remotes:
            branches = git_repo_query.get_branches(rp)
            for bn in branches:

                try:
                    out = subprocess.check_output(["git", "--git-dir=%s" % rp, "--work-tree=%s" % os.path.dirname(rp), "push", rm, bn])
                    out = "OK."
                    color = "\033[32m" # green
                except OSError as oser:
                    out = "Failed."
                    color = "\033[31m" # red
                except subprocess.CalledProcessError as cper:
                    out = "Failed."
                    color = "\033[31m" # red

                report.append("%s%s (remote=%s, branch=%s): %s%s" % (color, rp, rm, bn, out, ORIGINAL_COLOR))

    print("\nRESULTS:")
    for p in report:
        print(p)
    print("%s\n" % ORIGINAL_COLOR) # reset terminal color

if __name__ == "__main__":
    # mvtodo: get the remotes by param here, then pass to visitor_push. inside visitor_push, only use all remotes if this parameter is None
    git_visitor_base.do_visit(None, visitor_push)

<commit_msg>visitor push now uses the git push plugin<commit_after>#!/usr/bin/env python

import sys
import os

import git_visitor_base
import git_repo_query
import git_push

def visitor_push(repos):

    ORIGINAL_COLOR = "\033[0m" # mvtodo: would be better to try to detect the terminal's current standard color

    report = []
    for rp in repos:
        remotes = git_repo_query.get_remotes(rp)
        branches = git_repo_query.get_branches(rp)
        report_piece = git_push.do_push(rp, remotes, branches)
        for ri in report_piece:
            report.append(ri)

    print("\nRESULTS:")
    for p in report:
        print(p)
    print("%s\n" % ORIGINAL_COLOR) # reset terminal color

if __name__ == "__main__":
    # mvtodo: get the remotes by param here, then pass to visitor_push. inside visitor_push, only use all remotes if this parameter is None
    git_visitor_base.do_visit(None, visitor_push)

<|endoftext|>


<commit_before>"""
The equilibrium module defines routines for interacting with
calculated phase equilibria.
"""

import pycalphad.variables as v
from pycalphad.log import logger
from pycalphad.eq.utils import make_callable, generate_dof
from pycalphad.eq.utils import check_degenerate_phases
from pycalphad.eq.utils import unpack_kwarg
from pycalphad.constraints import sitefrac_cons, sitefrac_jac
from pycalphad.constraints import molefrac_ast
from pycalphad import Model
from pycalphad.eq.energy_surf import energy_surf
from pycalphad.eq.geometry import lower_convex_hull
from pycalphad.eq.eqresult import EquilibriumResult
from sympy import Symbol
import pandas as pd
import numpy as np
import scipy.spatial
import scipy.optimize
from collections import Counter
import copy

try:
    set
except NameError:
    from sets import Set as set #pylint: disable=W0622

class EquilibriumError(Exception):
    "Exception related to calculation of equilibrium"
    pass

class Equilibrium(object):
    """
    Calculate the equilibrium state of a system containing the specified
    components and phases, under the specified conditions.
    Model parameters are taken from 'db' and any state variables (T, P, etc.)
    can be specified as keyword arguments.

    Parameters
    ----------
    dbf : Database
        Thermodynamic database containing the relevant parameters.
    comps : list
        Names (case-sensitive) of components to consider in the calculation.
    phases : list
        Names (case-sensitive) of phases to consider in the calculation.
    conditions : dict
        StateVariables and their corresponding value.

    Returns
    -------
    Structured equilibrium calculation.

    Examples
    --------
    None yet.
    """
    def __init__(self, dbf, comps, phases, conditions, **kwargs):
        self.conditions = conditions
        self.components = set(comps)
        self.phases = dict()
        self.statevars = dict()
        self.data = pd.DataFrame()

        self._phases = dict([[name, dbf.phases[name]] for name in phases])
        self._phase_callables = dict()
        self._gradient_callables = dict()
        self._molefrac_callables = dict()
        self._molefrac_jac_callables = dict()
        self._variables = dict()
        self._sublattice_dof = dict()
        self.statevars = dict()
        for key in ['T', 'P']:
            try:
                self.statevars[v.StateVariable(key)] = kwargs[key]
            except KeyError:
                pass

        # Construct models for each phase; prioritize user models
        self._models = unpack_kwarg(kwargs.pop('model', Model), \
            default_arg=Model)
        for name in phases:
            mod = self._models[name]
            if isinstance(mod, type):
                # Initialize the model
                self._models[name] = mod(dbf, self.components, name)

        self._build_objective_functions()

        self.data = energy_surf(dbf, comps, phases, model=self._models, \
            **kwargs)

        # self.data now contains energy surface information for the system
        # find simplex for a starting point; refine with optimization
        estimates = self.get_starting_simplex()
        logger.debug(estimates)
        self.result = self.minimize(estimates[0], estimates[1])

    def __str__(self):
        return str(self.result)

    def get_starting_simplex(self):
        """
        Calculate convex hull and find a suitable starting point.
        Returns (DataFrame of phase compositions, ndarray of phase fractions)
        """
        phase_compositions, phase_fracs, pots = \
            lower_convex_hull(self.data, self.components, self.conditions)
        phase_compositions = phase_compositions[0]
        phase_fracs = phase_fracs[0]
        pots = pots[0]
        if phase_compositions is None:
            logger.error('Unable to find starting point for calculation')
            raise EquilibriumError('Unable to find starting point for calculation')
        logger.debug(self.data.iloc[phase_compositions])
        independent_indices = \
            check_degenerate_phases(self.data.iloc[phase_compositions],
                                    mindist=0.1)
        logger.debug('phase_fracs: %s', phase_fracs)
        logger.debug('independent_indices: %s', independent_indices)
        # renormalize phase fractions to 1 after eliminating redundant phases
        phase_fracs = phase_fracs[independent_indices]
        phase_fracs /= np.sum(phase_fracs)
        return [self.data.iloc[phase_compositions[independent_indices]],
                phase_fracs]

    def minimize(self, simplex, phase_fractions=None):
        """
        Accept a list of simplex vertices and return the values of the
        variables that minimize the energy under the constraints.
        """
        # Generate phase fraction variables
        # Track the multiplicity of phases with a Counter object
        composition_sets = Counter()
        all_variables = []
        # starting point
        x_0 = []
        # scaling factor -- set to minimum energy of starting simplex
        # Scaling the objective to be of order '10' seems to result in
        # sufficient precision (at least 5 significant figures).
        scaling_factor = abs(simplex['GM'].min()) / 10.0
        # a list of tuples for where each phase's variable indices
        # start and end
        index_ranges = []
        #print(list(enumerate(simplex.iterrows())))
        #print((simplex.iterrows()))
        #print('END')
        for m_idx, vertex in enumerate(simplex.iterrows()):
            vertex = vertex[1]
            # increase multiplicity by one
            composition_sets[vertex['Phase']] += 1
            # create new phase fraction variable
            all_variables.append(
                v.PhaseFraction(vertex['Phase'],
                                composition_sets[vertex['Phase']])
                )
            start = len(x_0)
            # default position is centroid of the simplex
            if phase_fractions is None:
                x_0.append(1.0/len(list(simplex.iterrows())))
            else:
                # use the provided guess for the phase fraction
                x_0.append(phase_fractions[m_idx])

            # add site fraction variables
            all_variables.extend(self._variables[vertex['Phase']])
            # add starting point for variable
            for varname in self._variables[vertex['Phase']]:
                x_0.append(vertex[str(varname)])
            index_ranges.append([start, len(x_0)])

        # Create master objective function
        def obj(input_x):
            "Objective function. Takes x vector as input. Returns scalar."
            objective = 0.0
            for idx, vertex in enumerate(simplex.iterrows()):
                vertex = vertex[1]
                cur_x = input_x[index_ranges[idx][0]+1:index_ranges[idx][1]]
                #print('Phase: '+vertex['Phase']+' '+str(cur_x))
                # phase fraction times value of objective for that phase
                objective += input_x[index_ranges[idx][0]] * \
                    self._phase_callables[vertex['Phase']](
                        *list(cur_x))
            return objective / scaling_factor

        # Create master gradient function
        def gradient(input_x):
            "Accepts input vector and returns gradient vector."
            gradient = np.zeros(len(input_x))
            for idx, vertex in enumerate(simplex.iterrows()):
                vertex = vertex[1]
                cur_x = input_x[index_ranges[idx][0]+1:index_ranges[idx][1]]
                #print('grad cur_x: '+str(cur_x))
                # phase fraction derivative is just the phase energy
                gradient[index_ranges[idx][0]] = \
                    self._phase_callables[vertex['Phase']](
                        *list(cur_x))
                # gradient for particular phase's variables
                # NOTE: We assume all phase d.o.f are independent here,
                # and we handle any coupling through the constraints
                for g_idx, grad in \
                    enumerate(self._gradient_callables[vertex['Phase']]):
                    gradient[index_ranges[idx][0]+1+g_idx] = \
                        input_x[index_ranges[idx][0]] * \
                            grad(*list(cur_x))
            #print('grad: '+str(gradient / scaling_factor))
            return gradient / scaling_factor

        # Generate constraint sequence
        constraints = []

        # phase fraction constraint
        def phasefrac_cons(input_x):
            "Accepts input vector and returns phase fraction constraint."
            output = 1.0 - sum([input_x[i[0]] for i in index_ranges])#** 2
            return output
        def phasefrac_jac(input_x):
            "Accepts input vector and returns Jacobian of constraint."
            output_x = np.zeros(len(input_x))
            for idx_range in index_ranges:
                output_x[idx_range[0]] = -1.0 #\
                #    -2.0*sum([input_x[i[0]] for i in index_ranges])
            return output_x
        phasefrac_dict = dict()
        phasefrac_dict['type'] = 'eq'
        phasefrac_dict['fun'] = phasefrac_cons
        phasefrac_dict['jac'] = phasefrac_jac
        constraints.append(phasefrac_dict)

        # bounds constraint
        def nonneg_cons(input_x, idx):
            "Accepts input vector and returns non-negativity constraint."
            output = input_x[idx]
            #print('nonneg_cons: '+str(output))
            return output

        def nonneg_jac(input_x, idx):
            "Accepts input vector and returns Jacobian of constraint."
            output_x = np.zeros(len(input_x))
            output_x[idx] = 1.0
            return output_x

        for idx in range(len(all_variables)):
            nonneg_dict = dict()
            nonneg_dict['type'] = 'ineq'
            nonneg_dict['fun'] = nonneg_cons
            nonneg_dict['jac'] = nonneg_jac
            nonneg_dict['args'] = [idx]
            constraints.append(nonneg_dict)

        # Generate all site fraction constraints
        for idx_range in index_ranges:
            # need to generate constraint for each sublattice
            dofs = self._sublattice_dof[all_variables[idx_range[0]].phase_name]
            cur_idx = idx_range[0]+1
            for dof in dofs:
                sitefrac_dict = dict()
                sitefrac_dict['type'] = 'eq'
                sitefrac_dict['fun'] = sitefrac_cons
                sitefrac_dict['jac'] = sitefrac_jac
                sitefrac_dict['args'] = [[cur_idx, cur_idx+dof]]
                cur_idx += dof
                if dof > 0:
                    constraints.append(sitefrac_dict)

        # All other constraints, e.g., mass balance
        def molefrac_cons(input_x, species, fix_val, all_variables, phases):
            """
            Accept input vector, species and fixed value.
            Returns constraint.
            """
            output = -fix_val
            for idx, vertex in enumerate(simplex.iterrows()):
                vertex = vertex[1]
                cur_x = input_x[index_ranges[idx][0]+1:index_ranges[idx][1]]
                res = self._molefrac_callables[vertex['Phase']][species](*cur_x)
                output += input_x[index_ranges[idx][0]] * res
            #print('molefrac_cons: '+str(output))
            return output
        def molefrac_jac(input_x, species, fix_val, all_variables, phases):
            "Accepts input vector and returns Jacobian vector."
            output_x = np.zeros(len(input_x))
            for idx, vertex in enumerate(simplex.iterrows()):
                vertex = vertex[1]
                cur_x = input_x[index_ranges[idx][0]+1:index_ranges[idx][1]]
                output_x[index_ranges[idx][0]] = \
                    self._molefrac_callables[vertex['Phase']][species](*cur_x)
                for g_idx, grad in \
                    enumerate(self._molefrac_jac_callables[vertex['Phase']][species]):
                    output_x[index_ranges[idx][0]+1+g_idx] = \
                        input_x[index_ranges[idx][0]] * \
                            grad(*list(cur_x))
            #print('molefrac_jac '+str(output_x))
            return output_x

        eqs = len([x for x in constraints if x['type'] == 'eq'])
        if eqs < len(x_0):
            for condition, value in self.conditions.items():
                if isinstance(condition, v.Composition):
                    # mass balance constraint for mole fraction
                    molefrac_dict = dict()
                    molefrac_dict['type'] = 'eq'
                    molefrac_dict['fun'] = molefrac_cons
                    molefrac_dict['jac'] = molefrac_jac
                    molefrac_dict['args'] = \
                        [condition.species, value, all_variables,
                         self._phases]
                    constraints.append(molefrac_dict)
        else:
            logger.warning("""Dropping mass balance constraints due to
                zero internal degrees of freedom""")

        # Run optimization
        res = scipy.optimize.minimize(obj, x_0, method='slsqp', jac=gradient,\
            constraints=constraints, options={'maxiter': 1000})
        # rescale final values back to original
        res['raw_fun'] = copy.deepcopy(res['fun'])
        res['raw_jac'] = copy.deepcopy(res['jac'])
        res['raw_x'] = copy.deepcopy(res['x'])
        res['fun'] *= scaling_factor
        res['jac'] *= scaling_factor
        # force tiny numerical values to be positive
        res['x'] = np.maximum(res['x'], np.zeros(1, dtype=np.float64))
        logger.debug(res)
        if not res['success']:
            logger.error('Energy minimization failed')
            return None

        # Build result object
        eq_res = EquilibriumResult(self._phases, self.components,
                                   self.statevars, res['fun'],
                                   zip(all_variables, res['x']))
        return eq_res

    def _build_objective_functions(self):
        "Construct objective function callables for each phase."
        for phase_name, phase_obj in self._phases.items():
            # Get the symbolic representation of the energy
            mod = self._models[phase_name]
            undefs = list(mod.ast.atoms(Symbol) - mod.ast.atoms(v.StateVariable))
            for undef in undefs:
                mod.ast = mod.ast.xreplace({undef: float(0)})
                logger.warning('Setting undefined symbol %s for phase %s to zero',
                               undef, phase_name)
            # Construct an ordered list of the variables
            self._variables[phase_name], self._sublattice_dof[phase_name] = \
                generate_dof(phase_obj, self.components)
            molefrac_dict = dict([(x, molefrac_ast(phase_obj, x)) \
                for x in self.components if x != 'VA'])
            molefrac_jac_dict = dict()

            # Generate callables for the mole fractions
            for comp in self.components:
                if comp == 'VA':
                    continue
                molefrac_jac_dict[comp] = [ \
                    make_callable(molefrac_dict[comp].diff(vx), \
                    self._variables[phase_name], \
                    ) for vx in self._variables[phase_name]]
                molefrac_dict[comp] = make_callable(molefrac_dict[comp], \
                    self._variables[phase_name])

            # Build the "fast" representation of energy model
            subbed_ast = mod.ast.subs(self.statevars)
            self._phase_callables[phase_name] = \
                make_callable(subbed_ast, \
                self._variables[phase_name])
            self._gradient_callables[phase_name] = [ \
                make_callable(subbed_ast.diff(vx), \
                    self._variables[phase_name], \
                    ) for vx in self._variables[phase_name]]
            self._molefrac_callables[phase_name] = molefrac_dict
            self._molefrac_jac_callables[phase_name] = molefrac_jac_dict
<commit_msg>BUG: Don't check for undefs during model construction. Let energy_surf deal with it<commit_after>"""
The equilibrium module defines routines for interacting with
calculated phase equilibria.
"""

import pycalphad.variables as v
from pycalphad.log import logger
from pycalphad.eq.utils import make_callable, generate_dof
from pycalphad.eq.utils import check_degenerate_phases
from pycalphad.eq.utils import unpack_kwarg
from pycalphad.constraints import sitefrac_cons, sitefrac_jac
from pycalphad.constraints import molefrac_ast
from pycalphad import Model
from pycalphad.eq.energy_surf import energy_surf
from pycalphad.eq.geometry import lower_convex_hull
from pycalphad.eq.eqresult import EquilibriumResult
from sympy import Symbol
import pandas as pd
import numpy as np
import scipy.spatial
import scipy.optimize
from collections import Counter
import copy

try:
    set
except NameError:
    from sets import Set as set #pylint: disable=W0622

class EquilibriumError(Exception):
    "Exception related to calculation of equilibrium"
    pass

class Equilibrium(object):
    """
    Calculate the equilibrium state of a system containing the specified
    components and phases, under the specified conditions.
    Model parameters are taken from 'db' and any state variables (T, P, etc.)
    can be specified as keyword arguments.

    Parameters
    ----------
    dbf : Database
        Thermodynamic database containing the relevant parameters.
    comps : list
        Names (case-sensitive) of components to consider in the calculation.
    phases : list
        Names (case-sensitive) of phases to consider in the calculation.
    conditions : dict
        StateVariables and their corresponding value.

    Returns
    -------
    Structured equilibrium calculation.

    Examples
    --------
    None yet.
    """
    def __init__(self, dbf, comps, phases, conditions, **kwargs):
        self.conditions = conditions
        self.components = set(comps)
        self.phases = dict()
        self.statevars = dict()
        self.data = pd.DataFrame()

        self._phases = dict([[name, dbf.phases[name]] for name in phases])
        self._phase_callables = dict()
        self._gradient_callables = dict()
        self._molefrac_callables = dict()
        self._molefrac_jac_callables = dict()
        self._variables = dict()
        self._sublattice_dof = dict()
        self.statevars = dict()
        for key in ['T', 'P']:
            try:
                self.statevars[v.StateVariable(key)] = kwargs[key]
            except KeyError:
                pass

        # Construct models for each phase; prioritize user models
        self._models = unpack_kwarg(kwargs.pop('model', Model), \
            default_arg=Model)
        for name in phases:
            mod = self._models[name]
            if isinstance(mod, type):
                # Initialize the model
                self._models[name] = mod(dbf, self.components, name)

        self._build_objective_functions()

        self.data = energy_surf(dbf, comps, phases, model=self._models, \
            **kwargs)

        # self.data now contains energy surface information for the system
        # find simplex for a starting point; refine with optimization
        estimates = self.get_starting_simplex()
        logger.debug(estimates)
        self.result = self.minimize(estimates[0], estimates[1])

    def __str__(self):
        return str(self.result)

    def get_starting_simplex(self):
        """
        Calculate convex hull and find a suitable starting point.
        Returns (DataFrame of phase compositions, ndarray of phase fractions)
        """
        phase_compositions, phase_fracs, pots = \
            lower_convex_hull(self.data, self.components, self.conditions)
        phase_compositions = phase_compositions[0]
        phase_fracs = phase_fracs[0]
        pots = pots[0]
        if phase_compositions is None:
            logger.error('Unable to find starting point for calculation')
            raise EquilibriumError('Unable to find starting point for calculation')
        logger.debug(self.data.iloc[phase_compositions])
        independent_indices = \
            check_degenerate_phases(self.data.iloc[phase_compositions],
                                    mindist=0.1)
        logger.debug('phase_fracs: %s', phase_fracs)
        logger.debug('independent_indices: %s', independent_indices)
        # renormalize phase fractions to 1 after eliminating redundant phases
        phase_fracs = phase_fracs[independent_indices]
        phase_fracs /= np.sum(phase_fracs)
        return [self.data.iloc[phase_compositions[independent_indices]],
                phase_fracs]

    def minimize(self, simplex, phase_fractions=None):
        """
        Accept a list of simplex vertices and return the values of the
        variables that minimize the energy under the constraints.
        """
        # Generate phase fraction variables
        # Track the multiplicity of phases with a Counter object
        composition_sets = Counter()
        all_variables = []
        # starting point
        x_0 = []
        # scaling factor -- set to minimum energy of starting simplex
        # Scaling the objective to be of order '10' seems to result in
        # sufficient precision (at least 5 significant figures).
        scaling_factor = abs(simplex['GM'].min()) / 10.0
        # a list of tuples for where each phase's variable indices
        # start and end
        index_ranges = []
        #print(list(enumerate(simplex.iterrows())))
        #print((simplex.iterrows()))
        #print('END')
        for m_idx, vertex in enumerate(simplex.iterrows()):
            vertex = vertex[1]
            # increase multiplicity by one
            composition_sets[vertex['Phase']] += 1
            # create new phase fraction variable
            all_variables.append(
                v.PhaseFraction(vertex['Phase'],
                                composition_sets[vertex['Phase']])
                )
            start = len(x_0)
            # default position is centroid of the simplex
            if phase_fractions is None:
                x_0.append(1.0/len(list(simplex.iterrows())))
            else:
                # use the provided guess for the phase fraction
                x_0.append(phase_fractions[m_idx])

            # add site fraction variables
            all_variables.extend(self._variables[vertex['Phase']])
            # add starting point for variable
            for varname in self._variables[vertex['Phase']]:
                x_0.append(vertex[str(varname)])
            index_ranges.append([start, len(x_0)])

        # Create master objective function
        def obj(input_x):
            "Objective function. Takes x vector as input. Returns scalar."
            objective = 0.0
            for idx, vertex in enumerate(simplex.iterrows()):
                vertex = vertex[1]
                cur_x = input_x[index_ranges[idx][0]+1:index_ranges[idx][1]]
                #print('Phase: '+vertex['Phase']+' '+str(cur_x))
                # phase fraction times value of objective for that phase
                objective += input_x[index_ranges[idx][0]] * \
                    self._phase_callables[vertex['Phase']](
                        *list(cur_x))
            return objective / scaling_factor

        # Create master gradient function
        def gradient(input_x):
            "Accepts input vector and returns gradient vector."
            gradient = np.zeros(len(input_x))
            for idx, vertex in enumerate(simplex.iterrows()):
                vertex = vertex[1]
                cur_x = input_x[index_ranges[idx][0]+1:index_ranges[idx][1]]
                #print('grad cur_x: '+str(cur_x))
                # phase fraction derivative is just the phase energy
                gradient[index_ranges[idx][0]] = \
                    self._phase_callables[vertex['Phase']](
                        *list(cur_x))
                # gradient for particular phase's variables
                # NOTE: We assume all phase d.o.f are independent here,
                # and we handle any coupling through the constraints
                for g_idx, grad in \
                    enumerate(self._gradient_callables[vertex['Phase']]):
                    gradient[index_ranges[idx][0]+1+g_idx] = \
                        input_x[index_ranges[idx][0]] * \
                            grad(*list(cur_x))
            #print('grad: '+str(gradient / scaling_factor))
            return gradient / scaling_factor

        # Generate constraint sequence
        constraints = []

        # phase fraction constraint
        def phasefrac_cons(input_x):
            "Accepts input vector and returns phase fraction constraint."
            output = 1.0 - sum([input_x[i[0]] for i in index_ranges])#** 2
            return output
        def phasefrac_jac(input_x):
            "Accepts input vector and returns Jacobian of constraint."
            output_x = np.zeros(len(input_x))
            for idx_range in index_ranges:
                output_x[idx_range[0]] = -1.0 #\
                #    -2.0*sum([input_x[i[0]] for i in index_ranges])
            return output_x
        phasefrac_dict = dict()
        phasefrac_dict['type'] = 'eq'
        phasefrac_dict['fun'] = phasefrac_cons
        phasefrac_dict['jac'] = phasefrac_jac
        constraints.append(phasefrac_dict)

        # bounds constraint
        def nonneg_cons(input_x, idx):
            "Accepts input vector and returns non-negativity constraint."
            output = input_x[idx]
            #print('nonneg_cons: '+str(output))
            return output

        def nonneg_jac(input_x, idx):
            "Accepts input vector and returns Jacobian of constraint."
            output_x = np.zeros(len(input_x))
            output_x[idx] = 1.0
            return output_x

        for idx in range(len(all_variables)):
            nonneg_dict = dict()
            nonneg_dict['type'] = 'ineq'
            nonneg_dict['fun'] = nonneg_cons
            nonneg_dict['jac'] = nonneg_jac
            nonneg_dict['args'] = [idx]
            constraints.append(nonneg_dict)

        # Generate all site fraction constraints
        for idx_range in index_ranges:
            # need to generate constraint for each sublattice
            dofs = self._sublattice_dof[all_variables[idx_range[0]].phase_name]
            cur_idx = idx_range[0]+1
            for dof in dofs:
                sitefrac_dict = dict()
                sitefrac_dict['type'] = 'eq'
                sitefrac_dict['fun'] = sitefrac_cons
                sitefrac_dict['jac'] = sitefrac_jac
                sitefrac_dict['args'] = [[cur_idx, cur_idx+dof]]
                cur_idx += dof
                if dof > 0:
                    constraints.append(sitefrac_dict)

        # All other constraints, e.g., mass balance
        def molefrac_cons(input_x, species, fix_val, all_variables, phases):
            """
            Accept input vector, species and fixed value.
            Returns constraint.
            """
            output = -fix_val
            for idx, vertex in enumerate(simplex.iterrows()):
                vertex = vertex[1]
                cur_x = input_x[index_ranges[idx][0]+1:index_ranges[idx][1]]
                res = self._molefrac_callables[vertex['Phase']][species](*cur_x)
                output += input_x[index_ranges[idx][0]] * res
            #print('molefrac_cons: '+str(output))
            return output
        def molefrac_jac(input_x, species, fix_val, all_variables, phases):
            "Accepts input vector and returns Jacobian vector."
            output_x = np.zeros(len(input_x))
            for idx, vertex in enumerate(simplex.iterrows()):
                vertex = vertex[1]
                cur_x = input_x[index_ranges[idx][0]+1:index_ranges[idx][1]]
                output_x[index_ranges[idx][0]] = \
                    self._molefrac_callables[vertex['Phase']][species](*cur_x)
                for g_idx, grad in \
                    enumerate(self._molefrac_jac_callables[vertex['Phase']][species]):
                    output_x[index_ranges[idx][0]+1+g_idx] = \
                        input_x[index_ranges[idx][0]] * \
                            grad(*list(cur_x))
            #print('molefrac_jac '+str(output_x))
            return output_x

        eqs = len([x for x in constraints if x['type'] == 'eq'])
        if eqs < len(x_0):
            for condition, value in self.conditions.items():
                if isinstance(condition, v.Composition):
                    # mass balance constraint for mole fraction
                    molefrac_dict = dict()
                    molefrac_dict['type'] = 'eq'
                    molefrac_dict['fun'] = molefrac_cons
                    molefrac_dict['jac'] = molefrac_jac
                    molefrac_dict['args'] = \
                        [condition.species, value, all_variables,
                         self._phases]
                    constraints.append(molefrac_dict)
        else:
            logger.warning("""Dropping mass balance constraints due to
                zero internal degrees of freedom""")

        # Run optimization
        res = scipy.optimize.minimize(obj, x_0, method='slsqp', jac=gradient,\
            constraints=constraints, options={'maxiter': 1000})
        # rescale final values back to original
        res['raw_fun'] = copy.deepcopy(res['fun'])
        res['raw_jac'] = copy.deepcopy(res['jac'])
        res['raw_x'] = copy.deepcopy(res['x'])
        res['fun'] *= scaling_factor
        res['jac'] *= scaling_factor
        # force tiny numerical values to be positive
        res['x'] = np.maximum(res['x'], np.zeros(1, dtype=np.float64))
        logger.debug(res)
        if not res['success']:
            logger.error('Energy minimization failed')
            return None

        # Build result object
        eq_res = EquilibriumResult(self._phases, self.components,
                                   self.statevars, res['fun'],
                                   zip(all_variables, res['x']))
        return eq_res

    def _build_objective_functions(self):
        "Construct objective function callables for each phase."
        for phase_name, phase_obj in self._phases.items():
            mod = self._models[phase_name]
            # Construct an ordered list of the variables
            self._variables[phase_name], self._sublattice_dof[phase_name] = \
                generate_dof(phase_obj, self.components)
            molefrac_dict = dict([(x, molefrac_ast(phase_obj, x)) \
                for x in self.components if x != 'VA'])
            molefrac_jac_dict = dict()

            # Generate callables for the mole fractions
            for comp in self.components:
                if comp == 'VA':
                    continue
                molefrac_jac_dict[comp] = [ \
                    make_callable(molefrac_dict[comp].diff(vx), \
                    self._variables[phase_name], \
                    ) for vx in self._variables[phase_name]]
                molefrac_dict[comp] = make_callable(molefrac_dict[comp], \
                    self._variables[phase_name])

            # Build the "fast" representation of energy model
            subbed_ast = mod.ast.subs(self.statevars)
            self._phase_callables[phase_name] = \
                make_callable(subbed_ast, \
                self._variables[phase_name])
            self._gradient_callables[phase_name] = [ \
                make_callable(subbed_ast.diff(vx), \
                    self._variables[phase_name], \
                    ) for vx in self._variables[phase_name]]
            self._molefrac_callables[phase_name] = molefrac_dict
            self._molefrac_jac_callables[phase_name] = molefrac_jac_dict
<|endoftext|>


<commit_before>from _blockedarray import *
import numpy
import vigra
import time
import copy

from lazyflow.operators import OpBlockedArrayCache, OpArrayPiper, OpSlicedBlockedArrayCache
from lazyflow.graph import Graph
from lazyflow.operators.opCxxBlockedArrayCache import OpCxxBlockedArrayCache

def generateSlicings(n=100, twoD = False):
    sl = []
    while(True):
        start = numpy.zeros(len(dataShape), numpy.uint32)
        stop  = numpy.zeros(len(dataShape), numpy.uint32)
        for i in range(len(dataShape)):
            start[i] = copy.copy(numpy.random.randint(low=0, high=dataShape[i])) # inclusive, exclusive
            stop[i] = copy.copy(numpy.random.randint(low=start[i]+1, high=dataShape[i]+1))
            assert stop[i] >= start[i]+1
            assert stop[i] < dataShape[i]+1
        assert numpy.all(stop > start), "start = %r, stop=%r" % (start, stop)
        assert numpy.all(stop <= dataShape)
        assert numpy.all(start >= 0)
        assert numpy.all(stop >= 0)

        slicing = tuple([slice(a,b) for a,b in zip(start, stop)])

        if twoD:
            x = numpy.random.randint(1,4)
            slicing = list(slicing)
            slicing[x] = slice(slicing[x].start, slicing[x].start+1)
            slicing = tuple(slicing)

        sl.append(slicing)
        if(len(sl) >= n):
            break
    return sl

if __name__ == "__main__":
    numpy.random.seed(0)

    dataShape =  (1,200,200,200,1)
    blockShape = (1,32 ,32, 32, 1)
    outerBlockShape = blockShape

    sl = generateSlicings(100, twoD=True)
    for slicing in sl:
        print slicing

    g = Graph()

    ba = BlockedArray5uint8(blockShape)

    print "generate random data"
    data = (255*numpy.random.random(dataShape)).astype(numpy.uint8)

    data = data.view(vigra.VigraArray)
    data.axistags = vigra.defaultAxistags('txyzc')

    graph = Graph()

    opProvider = OpArrayPiper(graph=graph)
    opProvider.Input.setValue(data)
    opProvider = opProvider

    #python cache
    opCache = OpBlockedArrayCache(graph=graph)
    opCache.Input.meta.shape = dataShape
    opCache.Input.connect(opProvider.Output)
    opCache.innerBlockShape.setValue(blockShape)
    opCache.outerBlockShape.setValue(outerBlockShape)
    opCache.fixAtCurrent.setValue(False)

    #python sliced array cache
    opCacheSliced = OpSlicedBlockedArrayCache(graph=graph)
    opCacheSliced.Input.meta.shape = dataShape
    opCacheSliced.Input.connect(opProvider.Output)

    #these are the actual values in pixel classification
    #blockDimsX = { 't' : (1,1), 'z' : (128,256), 'y' : (128,256), 'x' : (1,1), 'c' : (100, 100) }
    #blockDimsY = { 't' : (1,1), 'z' : (128,256), 'y' : (1,1), 'x' : (128,256), 'c' : (100,100) }
    #blockDimsZ = { 't' : (1,1), 'z' : (1,1), 'y' : (128,256), 'x' : (128,256), 'c' : (100,100) }

    #blockDimsX = { 't' : (1,1), 'z' : (32,32), 'y' : (32,32), 'x' : (1,1), 'c' : (32,32) }
    #blockDimsY = { 't' : (1,1), 'z' : (32,32), 'y' : (1,1), 'x' : (32,32), 'c' : (32,32) }
    #blockDimsZ = { 't' : (1,1), 'z' : (1,1), 'y' : (32,32), 'x' : (32,32), 'c' : (32,32) }

    blockDimsX = { 't' : (1,1), 'z' : (32,32), 'y' : (32,32), 'x' : (32,32), 'c' : (32,32) }
    blockDimsY = { 't' : (1,1), 'z' : (32,32), 'y' : (32,32), 'x' : (32,32), 'c' : (32,32) }
    blockDimsZ = { 't' : (1,1), 'z' : (32,32), 'y' : (32,32), 'x' : (32,32), 'c' : (32,32) }

    axisOrder=['t', 'x', 'y', 'z', 'c']
    innerBlockShapeX = tuple( blockDimsX[k][0] for k in axisOrder )
    outerBlockShapeX = tuple( blockDimsX[k][1] for k in axisOrder )
    innerBlockShapeY = tuple( blockDimsY[k][0] for k in axisOrder )
    outerBlockShapeY = tuple( blockDimsY[k][1] for k in axisOrder )
    innerBlockShapeZ = tuple( blockDimsZ[k][0] for k in axisOrder )
    outerBlockShapeZ = tuple( blockDimsZ[k][1] for k in axisOrder )

    opCacheSliced.innerBlockShape.setValue( (innerBlockShapeX, innerBlockShapeY, innerBlockShapeZ) )
    opCacheSliced.outerBlockShape.setValue( (outerBlockShapeX, outerBlockShapeY, outerBlockShapeZ) )

    opCacheSliced.fixAtCurrent.setValue(False)

    #C++ cache
    opCacheCpp = OpCxxBlockedArrayCache(graph=graph)
    opCacheCpp.Input.connect(opProvider.Output)
    opCacheCpp.innerBlockShape.setValue(blockShape)
    opCacheCpp.outerBlockShape.setValue(blockShape)

    #make sure that all blocks are in the cache

    print "request all from cpp",
    t = time.time()
    opCacheCpp.Output[tuple([slice(0, dataShape[i]) for i in range(len(dataShape))])].wait()
    #opCacheCpp.fixAtCurrent.setValue(True)
    print " took ", time.time()-t

    print "request all from py (array cached)",
    t = time.time()
    opCache.Output[tuple([slice(0, dataShape[i]) for i in range(len(dataShape))])].wait()
    #opCache.fixAtCurrent.setValue(True)
    print " took ", time.time()-t

    print "request all from py (sliced array cached)",
    t = time.time()
    opCacheSliced.Output[tuple([slice(0, dataShape[i]) for i in range(len(dataShape))])].wait()
    #opCacheSliced.fixAtCurrent.setValue(True)
    print " took ", time.time()-t

    slicings = []
    iter = 0

    ts = []
    ts2 = []

    #sl = [ (slice(0, 1, None), slice(77, 93, None), slice(56, 71, None), slice(138, 145, None), slice(0, 1, None)) ]

    #sl = [ (slice(0, 1, None), slice(48, 156, None), slice(113, 114, None), slice(188, 200, None), slice(0, 1, None)) ]
    #sl  = [ (slice(0, 1, None), slice(174, 192, None), slice(193, 200, None), slice(10, 11, None), slice(0, 1, None)) ]
    #sl = [ (slice(0, 1, None), slice(58, 95, None), slice(10, 97, None), slice(43, 44, None), slice(0, 1, None)) ]

    for slicing in sl:
        dataCachedSlice = None
        dataCpp         = None
        dataPy          = None

        print "size: %f^3" % numpy.power(numpy.prod([s.stop-s.start for s in slicing]), 1/3.0), "slicing =", slicing

        t0 = time.time()
        dataCacheSliced = opCacheSliced.Output( slicing ).wait()
        tPySliced = time.time()-t0
        if tPySliced*1000.0 > 10:
            print "&&&&&&&&&&&&&&&&&&&&&&&&&&&&"
        print "py1: %f msec." % (tPySliced*1000.0,)

        t1 = time.time()
        dataPy = opCache.Output( slicing ).wait()
        tPy = time.time()-t1
        print "py2: %f msec." % (tPy*1000.0,)

        t2 = time.time()
        dataCpp = opCacheCpp.Output( slicing ).wait()
        tCpp = time.time()-t2
        print "cpp  %f msec." % (tCpp*1000.0,)

        ts.append( tCpp/float(tPy) )
        ts2.append( tCpp/float(tPySliced) )

        assert numpy.all(dataPy == dataCpp)
        if dataPy is not None and dataCachedSlice is not None:
            assert numpy.all(dataPy == dataCacheSliced)

        iter += 1

    print "average ratio: cpp/py:         ", numpy.average(ts)
    print "average ratio: cpp/py(sliced): ", numpy.average(ts2)

<commit_msg>benchmark: less verbose<commit_after>from _blockedarray import *
import numpy
import vigra
import time
import copy

from lazyflow.operators import OpBlockedArrayCache, OpArrayPiper, OpSlicedBlockedArrayCache
from lazyflow.graph import Graph
from lazyflow.operators.opCxxBlockedArrayCache import OpCxxBlockedArrayCache

def generateSlicings(n=100, twoD = False):
    sl = []
    while(True):
        start = numpy.zeros(len(dataShape), numpy.uint32)
        stop  = numpy.zeros(len(dataShape), numpy.uint32)
        for i in range(len(dataShape)):
            start[i] = copy.copy(numpy.random.randint(low=0, high=dataShape[i])) # inclusive, exclusive
            stop[i] = copy.copy(numpy.random.randint(low=start[i]+1, high=dataShape[i]+1))
            assert stop[i] >= start[i]+1
            assert stop[i] < dataShape[i]+1
        assert numpy.all(stop > start), "start = %r, stop=%r" % (start, stop)
        assert numpy.all(stop <= dataShape)
        assert numpy.all(start >= 0)
        assert numpy.all(stop >= 0)

        slicing = tuple([slice(a,b) for a,b in zip(start, stop)])

        if twoD:
            x = numpy.random.randint(1,4)
            slicing = list(slicing)
            slicing[x] = slice(slicing[x].start, slicing[x].start+1)
            slicing = tuple(slicing)

        sl.append(slicing)
        if(len(sl) >= n):
            break
    return sl

if __name__ == "__main__":
    numpy.random.seed(0)

    dataShape =  (1,200,400,400,1)
    blockShape = (1,32 ,32, 32, 1)
    outerBlockShape = blockShape

    sl = generateSlicings(100, twoD=True)
    #for slicing in sl:
    #    print slicing

    g = Graph()

    ba = BlockedArray5uint8(blockShape)

    print "generate random data"
    data = (255*numpy.random.random(dataShape)).astype(numpy.uint8)

    data = data.view(vigra.VigraArray)
    data.axistags = vigra.defaultAxistags('txyzc')

    graph = Graph()

    opProvider = OpArrayPiper(graph=graph)
    opProvider.Input.setValue(data)
    opProvider = opProvider

    #python cache
    opCache = OpBlockedArrayCache(graph=graph)
    opCache.Input.meta.shape = dataShape
    opCache.Input.connect(opProvider.Output)
    opCache.innerBlockShape.setValue(blockShape)
    opCache.outerBlockShape.setValue(outerBlockShape)
    opCache.fixAtCurrent.setValue(False)

    #python sliced array cache
    opCacheSliced = OpSlicedBlockedArrayCache(graph=graph)
    opCacheSliced.Input.meta.shape = dataShape
    opCacheSliced.Input.connect(opProvider.Output)

    blockDimsX = { 't' : (1,1), 'z' : (32,32), 'y' : (32,32), 'x' : (32,32), 'c' : (32,32) }
    blockDimsY = { 't' : (1,1), 'z' : (32,32), 'y' : (32,32), 'x' : (32,32), 'c' : (32,32) }
    blockDimsZ = { 't' : (1,1), 'z' : (32,32), 'y' : (32,32), 'x' : (32,32), 'c' : (32,32) }

    axisOrder=['t', 'x', 'y', 'z', 'c']
    innerBlockShapeX = tuple( blockDimsX[k][0] for k in axisOrder )
    outerBlockShapeX = tuple( blockDimsX[k][1] for k in axisOrder )
    innerBlockShapeY = tuple( blockDimsY[k][0] for k in axisOrder )
    outerBlockShapeY = tuple( blockDimsY[k][1] for k in axisOrder )
    innerBlockShapeZ = tuple( blockDimsZ[k][0] for k in axisOrder )
    outerBlockShapeZ = tuple( blockDimsZ[k][1] for k in axisOrder )

    opCacheSliced.innerBlockShape.setValue( (innerBlockShapeX, innerBlockShapeY, innerBlockShapeZ) )
    opCacheSliced.outerBlockShape.setValue( (outerBlockShapeX, outerBlockShapeY, outerBlockShapeZ) )

    opCacheSliced.fixAtCurrent.setValue(False)

    #C++ cache
    opCacheCpp = OpCxxBlockedArrayCache(graph=graph)
    opCacheCpp.Input.connect(opProvider.Output)
    opCacheCpp.innerBlockShape.setValue(blockShape)
    opCacheCpp.outerBlockShape.setValue(blockShape)

    #make sure that all blocks are in the cache

    print "request all from cpp",
    t = time.time()
    opCacheCpp.Output[tuple([slice(0, dataShape[i]) for i in range(len(dataShape))])].wait()
    opCacheCpp.fixAtCurrent.setValue(True)
    print " took ", time.time()-t

    print "request all from py (array cached)",
    t = time.time()
    opCache.Output[tuple([slice(0, dataShape[i]) for i in range(len(dataShape))])].wait()
    #opCache.fixAtCurrent.setValue(True)
    print " took ", time.time()-t

    print "request all from py (sliced array cached)",
    t = time.time()
    opCacheSliced.Output[tuple([slice(0, dataShape[i]) for i in range(len(dataShape))])].wait()
    #opCacheSliced.fixAtCurrent.setValue(True)
    print " took ", time.time()-t

    slicings = []
    iter = 0

    ts = []
    ts2 = []

    for slicing in sl:
        dataCachedSlice = None
        dataCpp         = None
        dataPy          = None

        #print "size: %f^3" % numpy.power(numpy.prod([s.stop-s.start for s in slicing]), 1/3.0), "slicing =", slicing

        t0 = time.time()
        dataCacheSliced = opCacheSliced.Output( slicing ).wait()
        tPySliced = time.time()-t0
        if tPySliced*1000.0 > 10:
            print "&&&&&&&&&&&&&&&&&&&&&&&&&&&&"
        #print "py1: %f msec." % (tPySliced*1000.0,)

        t1 = time.time()
        dataPy = opCache.Output( slicing ).wait()
        tPy = time.time()-t1
        #print "py2: %f msec." % (tPy*1000.0,)

        t2 = time.time()
        dataCpp = opCacheCpp.Output( slicing ).wait()
        tCpp = time.time()-t2
        #print "cpp  %f msec." % (tCpp*1000.0,)

        ts.append( tCpp/float(tPy) )
        ts2.append( tCpp/float(tPySliced) )

        assert numpy.all(dataPy == dataCpp)
        if dataPy is not None and dataCachedSlice is not None:
            assert numpy.all(dataPy == dataCacheSliced)

        iter += 1

    print "average ratio: cpp/py:         ", numpy.average(ts)
    print "average ratio: cpp/py(sliced): ", numpy.average(ts2)

<|endoftext|>


<commit_before>#!/usr/bin/env python
import numpy as np
import matplotlib.pyplot as plt

def avg(vec):
    numeric_vec = [int(s.replace(",", "")) for s in vec]
    return np.average(numeric_vec)

def std(vec):
    numeric_vec = [int(s.replace(",", "")) for s in vec]
    return np.std(numeric_vec)

tcp = ("56,941,111,714", "57,425,424,643", "58,552,010,333")
tcpNc = ("50,950,205,711", "51,988,494,233", "52,604,911,053")
scrReads = ("26,925,647,069", "27,417,391,153", "27,086,041,030")
scrReadsNc = ("23,574,844,081", "23,546,252,040", "23,112,340,964")
zcrReads = ("12,602,014,986", "12,717,778,761", "12,808,630,928")

allVec = [ tcp, tcpNc, scrReads, scrReadsNc, zcrReads ]

N = 5
ind = np.arange(N)
barwidth = 0.35

fig, ax = plt.subplots()
rects1 = ax.bar(ind, [avg(v) for v in allVec], barwidth, color='r', yerr=[std(v) for v in allVec])

#twentyGbMeans = (tcpReadsAvg20G, tcpReadsNcAvg20G, scrReadsAvg20G, scrReadsNcAvg20G, zcrReadsAvg20G)
#twentyGbStd =   (tcpReadsStd20G, tcpReadsNcStd20G, scrReadsStd20G, scrReadsNcStd20G, zcrReadsStd20G)
#rects2 = ax.bar(ind+barwidth, twentyGbMeans, barwidth, color='y', yerr=twentyGbStd)

# add some
ax.set_ylabel('total cycles')
ax.set_title('vecsum2 total cycle count')
ax.set_xticks(ind+barwidth)
ax.set_xticklabels( ('TCP', 'TCP-no-csum', 'SCR', 'SCR-no-csum', 'zero-copy') )

#ax.legend( (rects1[0]), ('1 GB', '20 GB'), loc='left')

#ax.legend( (rects1[0], rects2[0]), ('1 GB', '20 GB'), loc='left')

plt.savefig("micro2.png")
<commit_msg>add 1 G x 20 comment to micro2.py<commit_after>#!/usr/bin/env python
import numpy as np
import matplotlib.pyplot as plt

# 1 G x 20

def avg(vec):
    numeric_vec = [int(s.replace(",", "")) for s in vec]
    return np.average(numeric_vec)

def std(vec):
    numeric_vec = [int(s.replace(",", "")) for s in vec]
    return np.std(numeric_vec)

tcp = ("56,941,111,714", "57,425,424,643", "58,552,010,333")
tcpNc = ("50,950,205,711", "51,988,494,233", "52,604,911,053")
scrReads = ("26,925,647,069", "27,417,391,153", "27,086,041,030")
scrReadsNc = ("23,574,844,081", "23,546,252,040", "23,112,340,964")
zcrReads = ("12,602,014,986", "12,717,778,761", "12,808,630,928")

allVec = [ tcp, tcpNc, scrReads, scrReadsNc, zcrReads ]

N = 5
ind = np.arange(N)
barwidth = 0.35

fig, ax = plt.subplots()
rects1 = ax.bar(ind, [avg(v) for v in allVec], barwidth, color='r', yerr=[std(v) for v in allVec])

#twentyGbMeans = (tcpReadsAvg20G, tcpReadsNcAvg20G, scrReadsAvg20G, scrReadsNcAvg20G, zcrReadsAvg20G)
#twentyGbStd =   (tcpReadsStd20G, tcpReadsNcStd20G, scrReadsStd20G, scrReadsNcStd20G, zcrReadsStd20G)
#rects2 = ax.bar(ind+barwidth, twentyGbMeans, barwidth, color='y', yerr=twentyGbStd)

# add some
ax.set_ylabel('total cycles')
ax.set_title('vecsum2 total cycle count')
ax.set_xticks(ind+barwidth)
ax.set_xticklabels( ('TCP', 'TCP-no-csum', 'SCR', 'SCR-no-csum', 'zero-copy') )

#ax.legend( (rects1[0]), ('1 GB', '20 GB'), loc='left')

#ax.legend( (rects1[0], rects2[0]), ('1 GB', '20 GB'), loc='left')

plt.savefig("micro2.png")
<|endoftext|>