<commit_before># -*- coding: utf-8 -*-
import re

_CAMEL_CASE_RE = re.compile(r'[A-Z]')
_LOWER_UNDERSCORE_CASE_RE = re.compile(r'_([a-z])')
_LOWER_DASH_CASE_RE = re.compile(r'-([a-z])')

def camel_to_lower_separated(s, sep):
    """
    Convert camel case representation into lower separated case ie:

      backgroundColor -> background_color

    Note any separator at the start or end is stripped.

    """
    return _CAMEL_CASE_RE.sub(lambda m: sep + m.group(0).lower(), s).strip(sep)

def camel_to_lower_underscore(s):
    """
    Convert camel case to lower underscore case.

        backgroundColor -> background_color
    """
    return camel_to_lower_separated(s, '_')

def camel_to_lower_dash(s):
    """
    Convert camel case to lower dash case.

      backgroundColor -> background-color
    """
    return camel_to_lower_separated(s, '-')

def lower_underscore_to_camel(value):
    """
    Convert lower underscore case to camel case

      background_color -> backgroundColor
    """
    return _LOWER_UNDERSCORE_CASE_RE.sub(
        lambda m: m.group(1).upper(),
        value.lower()
    )

def lower_dash_to_camel(value):
    """
    Convert lower dash case to camel case

      background-color -> backgroundColor
    """
    return _LOWER_DASH_CASE_RE.sub(
        lambda m: m.group(1).upper(),
        value.lower()
    )

class cached_property(object):
    """
    Acts like a standard class `property` except return values cached.
    """
    @staticmethod
    def clear_caches(instance):
        instance._cache = {}

    def __init__(self, func):
        self.func = func
        self.__doc__ = func.__doc__
        self.__name__ = func.__name__
        self.__module__ = func.__module__

    def __get__(self, instance, owner):
        try:
            value = instance._cache[self.__name__]
        except (KeyError, AttributeError):
            value = self.func(instance)
            try:
                cache = instance._cache
            except AttributeError:
                cache = instance._cache = {}
            cache[self.__name__] = value
        return value

def getmeta(resource):
    """
    Get meta object from a resource or resource instance.

    :param resource:
    :return: Meta options class

    """
    return getattr(resource, '_meta')

def field_iter(resource, include_virtual=True):
    """
    Return an iterator that yields fields from a resource.

    :param resource: Resource to iterate over.
    :param include_virtual: Include virtual fields.
    :returns: an iterator that returns fields.

    """
    meta = getmeta(resource)
    if include_virtual:
        return iter(meta.all_fields)
    else:
        return iter(meta.fields)

def field_iter_items(resource, fields=None):
    """
    Return an iterator that yields fields and their values from a resource.

    :param resource: Resource to iterate over.
    :param fields: Fields to use; if :const:`None` defaults to all of the resources fields.
    :returns: an iterator that returns (field, value) tuples.

    """
    meta = getmeta(resource)
    if fields is None:
        fields = meta.all_fields
    for f in fields:
        yield f, f.prepare(f.value_from_object(resource))

def virtual_field_iter_items(resource):
    """
    Return an iterator that yields virtual fields and their values from a resource.

    :param resource: Resource to iterate over.
    :returns: an iterator that returns (field, value) tuples.

    """
    meta = getmeta(resource)
    return field_iter_items(resource, meta.virtual_fields)

def attribute_field_iter_items(resource):
    """
    Return an iterator that yields fields and their values from a resource that have the attribute flag set.

    :param resource: Resource to iterate over.
    :returns: an iterator that returns (field, value) tuples.

    :note::
        This iterator is designed for codecs that have a distinction between attribute and element fields (eg XML).

    """
    return field_iter_items(resource, resource._meta.attribute_fields)

def element_field_iter_items(resource):
    """
    Return an iterator that yields fields and their values from a resource that do not have the attribute flag set.

    :param resource: Resource to iterate over.
    :returns: an iterator that returns (field, value) tuples.

    :note::
        This iterator is designed for codecs that have a distinction between attribute and element fields (eg XML).

    """
    return field_iter_items(resource, resource._meta.element_fields)

def extract_fields_from_dict(d, resource):
    """
    Extract values from a dict that are defined on a resource.

    Fields that are not found will not be included in the output dict.

    :param d: the source dictionary.
    :param resource: the resource that provides the fields.
    :returns: a dictionary of the resource fields that where found in the dict.

    """
    return dict((f.name, d[f.name]) for f in field_iter(resource) if f.name in d)

def value_in_choices(value, choices):
    """
    Check if the value appears in the choices list (a iterable of tuples, the first value of which is the choice value)

    :param value:
    :param choices:
    :return: True if value is in the choices iterable.

    """
    for choice in choices:
        if value == choice[0]:
            return True
    return False

def force_tuple(value):
    """
    Forces a value to be a tuple.

    Either by converting into a tuple (if is a list) or changing value to be a tuple.

    """
    if value is None:
        return tuple()
    if isinstance(value, tuple):
        return value
    if isinstance(value, list):
        return tuple(value)
    return value,

def chunk(iterable, n):
    """
    Return list of n items from an iterable.

    :param iterable: Iterable of items
    :param n: Size of iterable chunks to return.
    :return: List containing n items.

    """
    items = []
    for item in iterable:
        items.append(item)
        if len(items) == n:
            yield items
            items = []

    if items:
        yield items
<commit_msg>Improved docs and added type hints to several methods<commit_after># -*- coding: utf-8 -*-
import re

_CAMEL_CASE_RE = re.compile(r'[A-Z]')
_LOWER_UNDERSCORE_CASE_RE = re.compile(r'_([a-z])')
_LOWER_DASH_CASE_RE = re.compile(r'-([a-z])')

def camel_to_lower_separated(s, sep):
    """
    Convert camel case representation into lower separated case ie:

      backgroundColor -> background_color

    Note any separator at the start or end is stripped.

    """
    return _CAMEL_CASE_RE.sub(lambda m: sep + m.group(0).lower(), s).strip(sep)

def camel_to_lower_underscore(s):
    """
    Convert camel case to lower underscore case.

        backgroundColor -> background_color
    """
    return camel_to_lower_separated(s, '_')

def camel_to_lower_dash(s):
    """
    Convert camel case to lower dash case.

      backgroundColor -> background-color
    """
    return camel_to_lower_separated(s, '-')

def lower_underscore_to_camel(value):
    """
    Convert lower underscore case to camel case

      background_color -> backgroundColor
    """
    return _LOWER_UNDERSCORE_CASE_RE.sub(
        lambda m: m.group(1).upper(),
        value.lower()
    )

def lower_dash_to_camel(value):
    """
    Convert lower dash case to camel case

      background-color -> backgroundColor
    """
    return _LOWER_DASH_CASE_RE.sub(
        lambda m: m.group(1).upper(),
        value.lower()
    )

class cached_property(object):
    """
    Acts like a standard class `property` except return values cached.
    """
    @staticmethod
    def clear_caches(instance):
        instance._cache = {}

    def __init__(self, func):
        self.func = func
        self.__doc__ = func.__doc__
        self.__name__ = func.__name__
        self.__module__ = func.__module__

    def __get__(self, instance, owner):
        try:
            value = instance._cache[self.__name__]
        except (KeyError, AttributeError):
            value = self.func(instance)
            try:
                cache = instance._cache
            except AttributeError:
                cache = instance._cache = {}
            cache[self.__name__] = value
        return value

def getmeta(resource_or_instance):
    """
    Get meta object from a resource or resource instance.

    :param resource_or_instance: Resource or instance of a resource.
    :type resource_or_instance: odin.resources.ResourceType | odin.resources.ResourceBase
    :return: Meta options class
    :rtype: odin.resources.ResourceOptions

    """
    return getattr(resource_or_instance, '_meta')

def field_iter(resource, include_virtual=True):
    """
    Return an iterator that yields fields from a resource.

    :param resource: Resource to iterate over.
    :param include_virtual: Include virtual fields.
    :returns: an iterator that returns fields.

    """
    meta = getmeta(resource)
    if include_virtual:
        return iter(meta.all_fields)
    else:
        return iter(meta.fields)

def field_iter_items(resource, fields=None):
    """
    Return an iterator that yields fields and their values from a resource.

    :param resource: Resource to iterate over.
    :param fields: Fields to use; if :const:`None` defaults to all of the resources fields.
    :returns: an iterator that returns (field, value) tuples.

    """
    meta = getmeta(resource)
    if fields is None:
        fields = meta.all_fields
    for f in fields:
        yield f, f.prepare(f.value_from_object(resource))

def virtual_field_iter_items(resource):
    """
    Return an iterator that yields virtual fields and their values from a resource.

    :param resource: Resource to iterate over.
    :returns: an iterator that returns (field, value) tuples.

    """
    meta = getmeta(resource)
    return field_iter_items(resource, meta.virtual_fields)

def attribute_field_iter_items(resource):
    """
    Return an iterator that yields fields and their values from a resource that have the attribute flag set.

    :param resource: Resource to iterate over.
    :returns: an iterator that returns (field, value) tuples.

    :note::
        This iterator is designed for codecs that have a distinction between attribute and element fields (eg XML).

    """
    return field_iter_items(resource, resource._meta.attribute_fields)

def element_field_iter_items(resource):
    """
    Return an iterator that yields fields and their values from a resource that do not have the attribute flag set.

    :param resource: Resource to iterate over.
    :returns: an iterator that returns (field, value) tuples.

    :note::
        This iterator is designed for codecs that have a distinction between attribute and element fields (eg XML).

    """
    return field_iter_items(resource, resource._meta.element_fields)

def extract_fields_from_dict(d, resource):
    """
    Extract values from a dict that are defined on a resource.

    Fields that are not found will not be included in the output dict.

    :param d: the source dictionary.
    :param resource: the resource that provides the fields.
    :returns: a dictionary of the resource fields that where found in the dict.

    """
    return dict((f.name, d[f.name]) for f in field_iter(resource) if f.name in d)

def value_in_choices(value, choices):
    """
    Check if the value appears in the choices list (a iterable of tuples, the first value of which is the choice value)

    :param value:
    :param choices:
    :return: True if value is in the choices iterable.

    """
    for choice in choices:
        if value == choice[0]:
            return True
    return False

def force_tuple(value):
    """
    Forces a value to be a tuple.

    Either by converting into a tuple (if is a list) or changing value to be a tuple.

    :type value: T | tuple[T] | list[T]
    :rtype: tuple[T]

    """
    if value is None:
        return tuple()
    if isinstance(value, tuple):
        return value
    if isinstance(value, list):
        return tuple(value)
    return value,

def chunk(iterable, n):
    """
    Return list of n items from an iterable.

    :param iterable: Iterable of items
    :param n: Size of iterable chunks to return.
    :type n: int
    :return: List containing n items.
    :rtype: list[T]

    """
    items = []
    for item in iterable:
        items.append(item)
        if len(items) == n:
            yield items
            items = []

    if items:
        yield items
<|endoftext|>


<commit_before>from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()
import argparse
import os

import matplotlib.pyplot as plt
import pandas as pd

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--title', type=str, default='')
    parser.add_argument('--file', action='append', dest='files',
                        default=[], type=str,
                        help='specify paths of scores.txt')
    parser.add_argument('--label', action='append', dest='labels',
                        default=[], type=str,
                        help='specify labels for scores.txt files')
    args = parser.parse_args()

    assert len(args.files) > 0
    assert len(args.labels) == len(args.files)

    for fpath, label in zip(args.files, args.labels):
        if os.path.isdir(fpath):
            fpath = os.path.join(fpath, 'scores.txt')
        assert os.path.exists(fpath)
        scores = pd.read_csv(fpath, delimiter='\t')
        plt.plot(scores['steps'], scores['mean'], label=label)

    plt.xlabel('steps')
    plt.ylabel('score')
    plt.legend(loc='best')
    if args.title:
        plt.title(args.title)

    fig_fname = args.files[0] + args.title + '.png'
    plt.savefig(fig_fname)
    print('Saved a figure as {}'.format(fig_fname))

if __name__ == '__main__':
    main()
<commit_msg>Make it work without X-server<commit_after>from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
from future import standard_library
standard_library.install_aliases()
import argparse
import os

import matplotlib
matplotlib.use('Agg')  # Needed to run without X-server
import matplotlib.pyplot as plt
import pandas as pd

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--title', type=str, default='')
    parser.add_argument('--file', action='append', dest='files',
                        default=[], type=str,
                        help='specify paths of scores.txt')
    parser.add_argument('--label', action='append', dest='labels',
                        default=[], type=str,
                        help='specify labels for scores.txt files')
    args = parser.parse_args()

    assert len(args.files) > 0
    assert len(args.labels) == len(args.files)

    for fpath, label in zip(args.files, args.labels):
        if os.path.isdir(fpath):
            fpath = os.path.join(fpath, 'scores.txt')
        assert os.path.exists(fpath)
        scores = pd.read_csv(fpath, delimiter='\t')
        plt.plot(scores['steps'], scores['mean'], label=label)

    plt.xlabel('steps')
    plt.ylabel('score')
    plt.legend(loc='best')
    if args.title:
        plt.title(args.title)

    fig_fname = args.files[0] + args.title + '.png'
    plt.savefig(fig_fname)
    print('Saved a figure as {}'.format(fig_fname))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.

Written (W) 2013 Heiko Strathmann
Written (W) 2013 Dino Sejdinovic
"""

from main.experiments.ExperimentAggregator import ExperimentAggregator
from main.kernel.GaussianKernel import GaussianKernel
from main.tools.RCodaTools import RCodaTools
from matplotlib.pyplot import plot, fill_between, savefig, ylim, clf, title
from numpy.lib.npyio import savetxt
from numpy.linalg.linalg import norm
from numpy.ma.core import arange, zeros, mean, std, allclose, sqrt, asarray, \
    array
from numpy.ma.extras import median

class SingleChainExperimentAggregator(ExperimentAggregator):
    def __init__(self, folders, ref_quantiles=arange(0.1, 1, 0.1)):
        ExperimentAggregator.__init__(self, folders)
        self.ref_quantiles = ref_quantiles

    def __process_results__(self):
        lines = []
        if len(self.experiments) == 0:
            lines.append("no experiments to process")
            return

        # burnin is the same for all chains
        burnin = self.experiments[0].mcmc_chain.mcmc_params.burnin

        quantiles = zeros((len(self.experiments), len(self.ref_quantiles)))
        norm_of_means = zeros(len(self.experiments))
        acceptance_rates = zeros(len(self.experiments))
        ess_0 = zeros(len(self.experiments))
        ess_1 = zeros(len(self.experiments))
        ess_minima = zeros(len(self.experiments))
        ess_medians = zeros(len(self.experiments))
        ess_maxima = zeros(len(self.experiments))
        times = zeros(len(self.experiments))

        for i in range(len(self.experiments)):
            burned_in = self.experiments[i].mcmc_chain.samples[burnin:, :]

            # use precomputed quantiles if they match with the provided ones
            if hasattr(self.experiments[i], "ref_quantiles") and \
               hasattr(self.experiments[i], "quantiles") and \
               allclose(self.ref_quantiles, self.experiments[i].ref_quantiles):
                quantiles[i, :] = self.experiments[i].quantiles
            else:
                try:
                    quantiles[i, :] = self.experiments[i].mcmc_chain.mcmc_sampler.distribution.emp_quantiles(\
                                      burned_in, self.ref_quantiles)
                except NotImplementedError:
                    print "skipping quantile computations, distribution does", \
                          "not support it."

            dim = self.experiments[i].mcmc_chain.mcmc_sampler.distribution.dimension
            norm_of_means[i] = norm(mean(burned_in, 0))
            acceptance_rates[i] = mean(self.experiments[i].mcmc_chain.accepteds[burnin:])

            # dump burned in samples to disc
            # sample_filename=self.experiments[0].experiment_dir + self.experiments[0].name + "_burned_in.txt"
            # savetxt(sample_filename, burned_in)

            # store minimum ess for every experiment
            ess_per_covariate = asarray([RCodaTools.ess_coda(burned_in[:, cov_idx]) for cov_idx in range(dim)])
            ess_0=ess_per_covariate[0]
            ess_1=ess_per_covariate[1]
            ess_minima[i] = min(ess_per_covariate)
            ess_medians[i] = median(ess_per_covariate)
            ess_maxima[i] = max(ess_per_covariate)

            # save chain time needed
            ellapsed = self.experiments[i].mcmc_chain.mcmc_outputs[0].times
            times[i] = int(round(sum(ellapsed)))

        mean_quantiles = mean(quantiles, 0)
        std_quantiles = std(quantiles, 0)

        sqrt_num_trials=len(self.experiments)

        # print median kernel width sigma
        sigma=GaussianKernel.get_sigma_median_heuristic(burned_in.T)
        lines.append("median kernel sigma: "+str(sigma))

        lines.append("quantiles:")
        for i in range(len(self.ref_quantiles)):
            lines.append(str(mean_quantiles[i]) + " +- " + str(std_quantiles[i]/sqrt_num_trials))

        lines.append("norm of means:")
        lines.append(str(mean(norm_of_means)) + " +- " + str(std(norm_of_means)/sqrt_num_trials))

        lines.append("acceptance rate:")
        lines.append(str(mean(acceptance_rates)) + " +- " + str(std(acceptance_rates)/sqrt_num_trials))

        lines.append("ess dimension 0:")
        lines.append(str(mean(ess_0)) + " +- " + str(std(ess_0)/sqrt_num_trials))

        lines.append("ess dimension 1:")
        lines.append(str(mean(ess_1)) + " +- " + str(std(ess_1)/sqrt_num_trials))

        lines.append("minimum ess:")
        lines.append(str(mean(ess_minima)) + " +- " + str(std(ess_minima)/sqrt_num_trials))

        lines.append("median ess:")
        lines.append(str(mean(ess_medians)) + " +- " + str(std(ess_medians)/sqrt_num_trials))

        lines.append("maximum ess:")
        lines.append(str(mean(ess_maxima)) + " +- " + str(std(ess_maxima)/sqrt_num_trials))

        lines.append("times:")
        lines.append(str(mean(times)) + " +- " + str(std(times)/sqrt_num_trials))

        # mean as a function of iterations, normalised by time
        step = 1000
        iterations = arange(self.experiments[0].mcmc_chain.mcmc_params.num_iterations - burnin, step=step)

        running_means = zeros(len(iterations))
        running_errors = zeros(len(iterations))
        for i in arange(len(iterations)):
            # norm of mean of chain up
            norm_of_means_yet = zeros(len(self.experiments))
            for j in range(len(self.experiments)):
                samples_yet = self.experiments[j].mcmc_chain.samples[burnin:(burnin + iterations[i] + 1 + step), :]
                norm_of_means_yet[j] = norm(mean(samples_yet, 0))

            running_means[i] = mean(norm_of_means_yet)
            error_level = 1.96
            running_errors[i] = error_level * std(norm_of_means_yet) / sqrt(len(norm_of_means_yet))

        plot(iterations, running_means/mean(times))
        fill_between(iterations, (running_means - running_errors)/mean(times), \
                     (running_means + running_errors)/mean(times), hold=True, color="gray")
        savefig(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_mean.png")
        clf()

        # also store plot X and Y
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_mean_X.txt", \
                iterations)
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_mean_Y.txt", \
                running_means/mean(times))
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_mean_errors.txt", \
                running_errors/mean(times))

        # quantile convergence of a single one
        desired_quantile=0.1
        running_quantiles=zeros(len(iterations))
        running_quantile_errors=zeros(len(iterations))
        for i in arange(len(iterations)):
            quantiles_yet = zeros(len(self.experiments))
            for j in range(len(self.experiments)):
                samples_yet = self.experiments[j].mcmc_chain.samples[burnin:(burnin + iterations[i] + 1 + step), :]

                # just compute one quantile for now
                quantile=self.experiments[j].mcmc_chain.mcmc_sampler.distribution.emp_quantiles(samples_yet, \
                                                                                array([desired_quantile]))
            running_quantiles[i] = mean(quantiles_yet)
            error_level = 1.96
            running_quantile_errors[i] = error_level * std(quantiles_yet) / sqrt(len(quantiles_yet))

        plot(iterations, running_quantiles/mean(times))
        fill_between(iterations, (running_quantiles - running_quantile_errors)/mean(times), \
                     (running_quantiles + running_quantile_errors)/mean(times), hold=True, color="gray")

        plot([iterations.min(),iterations.max()], [desired_quantile/mean(times) for _ in range(2)])

        title(str(desired_quantile)+"-quantile convergence")
        savefig(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile.png")
        clf()

        # also store plot X and Y
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile_X.txt", \
                iterations)
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile_Y.txt", \
                running_quantiles/mean(times))
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile_errors.txt", \
                running_quantile_errors/mean(times))
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile_reference.txt", \
                desired_quantile/mean(times))

        # add latex table line
        latex_lines = []
        latex_lines.append("Sampler & Acceptance & ESS2 & Norm(mean) & ")
        for i in range(len(self.ref_quantiles)):
            latex_lines.append('%.1f' % self.ref_quantiles[i] + "-quantile")
            if i < len(self.ref_quantiles) - 1:
                latex_lines.append(" & ")
        latex_lines.append("\\\\")
        lines.append("".join(latex_lines))

        latex_lines = []
        latex_lines.append(self.experiments[0].mcmc_chain.mcmc_sampler.__class__.__name__)
        latex_lines.append('$%.3f' % mean(acceptance_rates) + " \pm " + '%.3f$' % (std(acceptance_rates)/sqrt_num_trials))
        latex_lines.append('$%.3f' % mean(norm_of_means) + " \pm " + '%.3f$' % (std(norm_of_means)/sqrt_num_trials))
        for i in range(len(self.ref_quantiles)):
            latex_lines.append('$%.3f' % mean_quantiles[i] + " \pm " + '%.3f$' % (std_quantiles[i]/sqrt_num_trials))

        lines.append(" & ".join(latex_lines) + "\\\\")

        return lines

    def __str__(self):
        s = self.__class__.__name__ + "=["
        s += "ref_quantiles=" + str(self.ref_quantiles)
        s += ", " + ExperimentAggregator.__str__(self)
        s += "]"
        return s
<commit_msg>workaround for numpy savetxt bug when being passed a scalar<commit_after>"""
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.

Written (W) 2013 Heiko Strathmann
Written (W) 2013 Dino Sejdinovic
"""

from main.experiments.ExperimentAggregator import ExperimentAggregator
from main.kernel.GaussianKernel import GaussianKernel
from main.tools.RCodaTools import RCodaTools
from matplotlib.pyplot import plot, fill_between, savefig, ylim, clf, title
from numpy.lib.npyio import savetxt
from numpy.linalg.linalg import norm
from numpy.ma.core import arange, zeros, mean, std, allclose, sqrt, asarray, \
    array
from numpy.ma.extras import median

class SingleChainExperimentAggregator(ExperimentAggregator):
    def __init__(self, folders, ref_quantiles=arange(0.1, 1, 0.1)):
        ExperimentAggregator.__init__(self, folders)
        self.ref_quantiles = ref_quantiles

    def __process_results__(self):
        lines = []
        if len(self.experiments) == 0:
            lines.append("no experiments to process")
            return

        # burnin is the same for all chains
        burnin = self.experiments[0].mcmc_chain.mcmc_params.burnin

        quantiles = zeros((len(self.experiments), len(self.ref_quantiles)))
        norm_of_means = zeros(len(self.experiments))
        acceptance_rates = zeros(len(self.experiments))
        ess_0 = zeros(len(self.experiments))
        ess_1 = zeros(len(self.experiments))
        ess_minima = zeros(len(self.experiments))
        ess_medians = zeros(len(self.experiments))
        ess_maxima = zeros(len(self.experiments))
        times = zeros(len(self.experiments))

        for i in range(len(self.experiments)):
            burned_in = self.experiments[i].mcmc_chain.samples[burnin:, :]

            # use precomputed quantiles if they match with the provided ones
            if hasattr(self.experiments[i], "ref_quantiles") and \
               hasattr(self.experiments[i], "quantiles") and \
               allclose(self.ref_quantiles, self.experiments[i].ref_quantiles):
                quantiles[i, :] = self.experiments[i].quantiles
            else:
                try:
                    quantiles[i, :] = self.experiments[i].mcmc_chain.mcmc_sampler.distribution.emp_quantiles(\
                                      burned_in, self.ref_quantiles)
                except NotImplementedError:
                    print "skipping quantile computations, distribution does", \
                          "not support it."

            dim = self.experiments[i].mcmc_chain.mcmc_sampler.distribution.dimension
            norm_of_means[i] = norm(mean(burned_in, 0))
            acceptance_rates[i] = mean(self.experiments[i].mcmc_chain.accepteds[burnin:])

            # dump burned in samples to disc
            # sample_filename=self.experiments[0].experiment_dir + self.experiments[0].name + "_burned_in.txt"
            # savetxt(sample_filename, burned_in)

            # store minimum ess for every experiment
            ess_per_covariate = asarray([RCodaTools.ess_coda(burned_in[:, cov_idx]) for cov_idx in range(dim)])
            ess_0=ess_per_covariate[0]
            ess_1=ess_per_covariate[1]
            ess_minima[i] = min(ess_per_covariate)
            ess_medians[i] = median(ess_per_covariate)
            ess_maxima[i] = max(ess_per_covariate)

            # save chain time needed
            ellapsed = self.experiments[i].mcmc_chain.mcmc_outputs[0].times
            times[i] = int(round(sum(ellapsed)))

        mean_quantiles = mean(quantiles, 0)
        std_quantiles = std(quantiles, 0)

        sqrt_num_trials=len(self.experiments)

        # print median kernel width sigma
        sigma=GaussianKernel.get_sigma_median_heuristic(burned_in.T)
        lines.append("median kernel sigma: "+str(sigma))

        lines.append("quantiles:")
        for i in range(len(self.ref_quantiles)):
            lines.append(str(mean_quantiles[i]) + " +- " + str(std_quantiles[i]/sqrt_num_trials))

        lines.append("norm of means:")
        lines.append(str(mean(norm_of_means)) + " +- " + str(std(norm_of_means)/sqrt_num_trials))

        lines.append("acceptance rate:")
        lines.append(str(mean(acceptance_rates)) + " +- " + str(std(acceptance_rates)/sqrt_num_trials))

        lines.append("ess dimension 0:")
        lines.append(str(mean(ess_0)) + " +- " + str(std(ess_0)/sqrt_num_trials))

        lines.append("ess dimension 1:")
        lines.append(str(mean(ess_1)) + " +- " + str(std(ess_1)/sqrt_num_trials))

        lines.append("minimum ess:")
        lines.append(str(mean(ess_minima)) + " +- " + str(std(ess_minima)/sqrt_num_trials))

        lines.append("median ess:")
        lines.append(str(mean(ess_medians)) + " +- " + str(std(ess_medians)/sqrt_num_trials))

        lines.append("maximum ess:")
        lines.append(str(mean(ess_maxima)) + " +- " + str(std(ess_maxima)/sqrt_num_trials))

        lines.append("times:")
        lines.append(str(mean(times)) + " +- " + str(std(times)/sqrt_num_trials))

        # mean as a function of iterations, normalised by time
        step = 1000
        iterations = arange(self.experiments[0].mcmc_chain.mcmc_params.num_iterations - burnin, step=step)

        running_means = zeros(len(iterations))
        running_errors = zeros(len(iterations))
        for i in arange(len(iterations)):
            # norm of mean of chain up
            norm_of_means_yet = zeros(len(self.experiments))
            for j in range(len(self.experiments)):
                samples_yet = self.experiments[j].mcmc_chain.samples[burnin:(burnin + iterations[i] + 1 + step), :]
                norm_of_means_yet[j] = norm(mean(samples_yet, 0))

            running_means[i] = mean(norm_of_means_yet)
            error_level = 1.96
            running_errors[i] = error_level * std(norm_of_means_yet) / sqrt(len(norm_of_means_yet))

        plot(iterations, running_means/mean(times))
        fill_between(iterations, (running_means - running_errors)/mean(times), \
                     (running_means + running_errors)/mean(times), hold=True, color="gray")
        savefig(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_mean.png")
        clf()

        # also store plot X and Y
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_mean_X.txt", \
                iterations)
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_mean_Y.txt", \
                running_means/mean(times))
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_mean_errors.txt", \
                running_errors/mean(times))

        # quantile convergence of a single one
        desired_quantile=0.1
        running_quantiles=zeros(len(iterations))
        running_quantile_errors=zeros(len(iterations))
        for i in arange(len(iterations)):
            quantiles_yet = zeros(len(self.experiments))
            for j in range(len(self.experiments)):
                samples_yet = self.experiments[j].mcmc_chain.samples[burnin:(burnin + iterations[i] + 1 + step), :]

                # just compute one quantile for now
                quantile=self.experiments[j].mcmc_chain.mcmc_sampler.distribution.emp_quantiles(samples_yet, \
                                                                                array([desired_quantile]))
            running_quantiles[i] = mean(quantiles_yet)
            error_level = 1.96
            running_quantile_errors[i] = error_level * std(quantiles_yet) / sqrt(len(quantiles_yet))

        plot(iterations, running_quantiles/mean(times))
        fill_between(iterations, (running_quantiles - running_quantile_errors)/mean(times), \
                     (running_quantiles + running_quantile_errors)/mean(times), hold=True, color="gray")

        plot([iterations.min(),iterations.max()], [desired_quantile/mean(times) for _ in range(2)])

        title(str(desired_quantile)+"-quantile convergence")
        savefig(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile.png")
        clf()

        # also store plot X and Y
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile_X.txt", \
                iterations)
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile_Y.txt", \
                running_quantiles/mean(times))
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile_errors.txt", \
                running_quantile_errors/mean(times))
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile_reference.txt", \
                [desired_quantile/mean(times)])

        # add latex table line
        latex_lines = []
        latex_lines.append("Sampler & Acceptance & ESS2 & Norm(mean) & ")
        for i in range(len(self.ref_quantiles)):
            latex_lines.append('%.1f' % self.ref_quantiles[i] + "-quantile")
            if i < len(self.ref_quantiles) - 1:
                latex_lines.append(" & ")
        latex_lines.append("\\\\")
        lines.append("".join(latex_lines))

        latex_lines = []
        latex_lines.append(self.experiments[0].mcmc_chain.mcmc_sampler.__class__.__name__)
        latex_lines.append('$%.3f' % mean(acceptance_rates) + " \pm " + '%.3f$' % (std(acceptance_rates)/sqrt_num_trials))
        latex_lines.append('$%.3f' % mean(norm_of_means) + " \pm " + '%.3f$' % (std(norm_of_means)/sqrt_num_trials))
        for i in range(len(self.ref_quantiles)):
            latex_lines.append('$%.3f' % mean_quantiles[i] + " \pm " + '%.3f$' % (std_quantiles[i]/sqrt_num_trials))

        lines.append(" & ".join(latex_lines) + "\\\\")

        return lines

    def __str__(self):
        s = self.__class__.__name__ + "=["
        s += "ref_quantiles=" + str(self.ref_quantiles)
        s += ", " + ExperimentAggregator.__str__(self)
        s += "]"
        return s
<|endoftext|>


<commit_before>#-*- coding: utf8
import odml.dtypes as dtypes
import odml.base as base
import odml.format as format

import string
from odml.tools.doc_inherit import inherit_docstring, allow_inherit_docstring

class Value(base._baseobj):
    pass

@allow_inherit_docstring
class BaseValue(base.baseobject, Value):
    """
    An odML value

    data
        mandatory (unless value is set). It's the content itself.
        (see the data and value attributes of this object)

    uncertainty (optional)
        an estimation of the value's uncertainty.

    unit (optional)
        the value's unit

    dtype (optional)
        the data type of the value

    reference (optional)
        an external reference number (e.g. entry in a database)

    definition (optional)
        additional comments on the value of the property

    value
        mandatory (unless data is set). It's the string representation of the value.

    TODO: comment
    """

    _format = format.Value

    def __init__(self, data=None, uncertainty=None, unit=None, dtype=None,
                 definition=None, reference=None, comment=None, value=None):
        if data is None and value is None:
            raise TypeError("either data or value has to be set")
        if data is not None and value is not None:
            raise TypeError("only one of data or value can be set")

        self._property = None
        self._unit = unit
        self._uncertainty = uncertainty
        self._dtype = dtype
        self._definition = definition
        self._reference = reference
        self._comment = comment

        if value is not None:
            # assign value directly (through property would raise a change-event)
            self._value = dtypes.get(value, self._dtype)
        elif data is not None:
            if dtype is None:
                self._dtype = dtypes.infer_dtype(data)
            self._value = data

    def __repr__(self):
        if self._dtype:
            return "<%s %s>" % (str(self._dtype), self.get_display())
        return "<%s>" % str(self._value)

    @property
    def parent(self):
        """the property containing this value"""
        return self._property

    @property
    def data(self):
        """
        used to access the raw data of the value
        (i.e. a datetime-object if dtype is "datetime")
        see also the value attribute
        """
        return self._value

    @data.setter
    def data(self, new_value):
        self._value = new_value

    @property
    def value(self):
        """
        used to access typed data of the value as a string.
        Use data to access the raw type, i.e.:

        >>> v = Value(1, type="float")
        >>> v.data
        1.0
        >>> v.data = 1.5
        >>> v.value
        "1.5"
        >>> v.value = 2
        >>> v.data
        2.0
        """
        return dtypes.set(self._value, self._dtype)

    @value.setter
    def value(self, new_string):
        self._value = dtypes.get(new_string, self._dtype)

    @property
    def dtype(self):
        """
        the data type of the value

        If the data type is changed, it is tried, to convert the value to the new type.

        If this doesn't work, the change is refused.
        This behaviour can be overridden by directly accessing the *_dtype* attribute
        and adjusting the *data* attribute manually.
        """
        return self._dtype

    @dtype.setter
    def dtype(self, new_type):
        # check if this is a valid type
        if not dtypes.valid_type(new_type):
            raise AttributeError("'%s' is not a valid type." % new_type)
        # we convert the value if possible
        old_type = self._dtype
        old_value = dtypes.set(self._value, self._dtype)
        try:
            new_value = dtypes.get(old_value, new_type)
        except:
            # cannot convert, try the other way around
            try:
                old_value = dtypes.set(self._value, new_type)
                new_value = dtypes.get(old_value, new_type)
            except:
                #doesn't work either, therefore refuse
                raise ValueError("cannot convert '%s' from '%s' to '%s'" % (self.value, old_type, new_type))
        self._value = new_value
        self._dtype = new_type

    @property
    def uncertainty(self):
        return self._uncertainty

    @uncertainty.setter
    def uncertainty(self, new_value):
        self._uncertainty = new_value

    @property
    def unit(self):
        return self._unit

    @unit.setter
    def unit(self, new_value):
        self._unit = new_value

    @property
    def reference(self):
        return self._reference

    @reference.setter
    def reference(self, new_value):
        self._reference = new_value

    @property
    def definition(self):
        return self._definition

    @definition.setter
    def definition(self, new_value):
        self._definition = new_value

    @property
    def comment(self):
        return self._comment

    @comment.setter
    def comment(self, new_value):
        self._comment = new_value

    def can_display(self, text=None, max_length=-1):
        """
        return whether the content of this can be safely displayed in the gui
        """
        if text is None:
            text = self._value
        if text is None:
            return True

        if max_length != -1 and len(text) > max_length:
            return False

        if "\n" in text or "\t" in text:
            return False

        return True

    def get_display(self, max_length=-1):
        """
        return a textual representation that can be used for display

        typically takes the first line (max *max_length* chars) and adds '…'
        """
        text = self.value
        if self.can_display(text, max_length):
            return text

        text = text.split("\n")[0]
        if max_length != -1:
            text = text[:max_length]
        if self.can_display(text, max_length):
            return text + u'…'

        return "(%d bytes)" % len(self._value)

#    def can_edit(self, max_length=-1)
#        if not self.can_display(self): return False
#
#        return True
#
    @inherit_docstring
    def reorder(self, new_index):
        return self._reorder(self.parent.values, new_index)

    def clone(self):
        obj = super(BaseValue, self).clone()
        obj._property = None
        return obj

    @inherit_docstring
    def get_terminology_equivalent(self):
        prop = self._property.get_terminology_equivalent()

        if prop is None:
            return None

        for val in prop:
            if val == self:  # TODO: shouldn't we take the index or st.?
                return val

        return None
<commit_msg>Fix string representation<commit_after>#-*- coding: utf8
import odml.dtypes as dtypes
import odml.base as base
import odml.format as format

import string
from odml.tools.doc_inherit import inherit_docstring, allow_inherit_docstring

class Value(base._baseobj):
    pass

@allow_inherit_docstring
class BaseValue(base.baseobject, Value):
    """
    An odML value

    data
        mandatory (unless value is set). It's the content itself.
        (see the data and value attributes of this object)

    uncertainty (optional)
        an estimation of the value's uncertainty.

    unit (optional)
        the value's unit

    dtype (optional)
        the data type of the value

    reference (optional)
        an external reference number (e.g. entry in a database)

    definition (optional)
        additional comments on the value of the property

    value
        mandatory (unless data is set). It's the string representation of the value.

    TODO: comment
    """

    _format = format.Value

    def __init__(self, data=None, uncertainty=None, unit=None, dtype=None,
                 definition=None, reference=None, comment=None, value=None):
        if data is None and value is None:
            raise TypeError("either data or value has to be set")
        if data is not None and value is not None:
            raise TypeError("only one of data or value can be set")

        self._property = None
        self._unit = unit
        self._uncertainty = uncertainty
        self._dtype = dtype
        self._definition = definition
        self._reference = reference
        self._comment = comment

        if value is not None:
            # assign value directly (through property would raise a change-event)
            self._value = dtypes.get(value, self._dtype)
        elif data is not None:
            if dtype is None:
                self._dtype = dtypes.infer_dtype(data)
            self._value = data

    def __repr__(self):
        if self._dtype:
            return "<%s %s>" % (str(self._dtype), self.get_display())
        return "<%s>" % str(self._value)

    @property
    def parent(self):
        """the property containing this value"""
        return self._property

    @property
    def data(self):
        """
        used to access the raw data of the value
        (i.e. a datetime-object if dtype is "datetime")
        see also the value attribute
        """
        return self._value

    @data.setter
    def data(self, new_value):
        self._value = new_value

    @property
    def value(self):
        """
        used to access typed data of the value as a string.
        Use data to access the raw type, i.e.:

        >>> v = Value(1, type="float")
        >>> v.data
        1.0
        >>> v.data = 1.5
        >>> v.value
        "1.5"
        >>> v.value = 2
        >>> v.data
        2.0
        """
        return dtypes.set(self._value, self._dtype)

    @value.setter
    def value(self, new_string):
        self._value = dtypes.get(new_string, self._dtype)

    @property
    def dtype(self):
        """
        the data type of the value

        If the data type is changed, it is tried, to convert the value to the new type.

        If this doesn't work, the change is refused.
        This behaviour can be overridden by directly accessing the *_dtype* attribute
        and adjusting the *data* attribute manually.
        """
        return self._dtype

    @dtype.setter
    def dtype(self, new_type):
        # check if this is a valid type
        if not dtypes.valid_type(new_type):
            raise AttributeError("'%s' is not a valid type." % new_type)
        # we convert the value if possible
        old_type = self._dtype
        old_value = dtypes.set(self._value, self._dtype)
        try:
            new_value = dtypes.get(old_value, new_type)
        except:
            # cannot convert, try the other way around
            try:
                old_value = dtypes.set(self._value, new_type)
                new_value = dtypes.get(old_value, new_type)
            except:
                #doesn't work either, therefore refuse
                raise ValueError("cannot convert '%s' from '%s' to '%s'" % (self.value, old_type, new_type))
        self._value = new_value
        self._dtype = new_type

    @property
    def uncertainty(self):
        return self._uncertainty

    @uncertainty.setter
    def uncertainty(self, new_value):
        self._uncertainty = new_value

    @property
    def unit(self):
        return self._unit

    @unit.setter
    def unit(self, new_value):
        self._unit = new_value

    @property
    def reference(self):
        return self._reference

    @reference.setter
    def reference(self, new_value):
        self._reference = new_value

    @property
    def definition(self):
        return self._definition

    @definition.setter
    def definition(self, new_value):
        self._definition = new_value

    @property
    def comment(self):
        return self._comment

    @comment.setter
    def comment(self, new_value):
        self._comment = new_value

    def can_display(self, text=None, max_length=-1):
        """
        return whether the content of this can be safely displayed in the gui
        """
        if text is None:
            text = self._value
        if text is None:
            return True

        if max_length != -1 and len(text) > max_length:
            return False

        if "\n" in text or "\t" in text:
            return False

        return True

    def get_display(self, max_length=-1):
        """
        return a textual representation that can be used for display

        typically takes the first line (max *max_length* chars) and adds '…'
        """
        text = self.value
        if self.can_display(text, max_length):
            return text

        text = text.split("\n")[0]
        if max_length != -1:
            text = text[:max_length]
        if self.can_display(text, max_length):
            return (text + u'…').encode('utf-8')

        return "(%d bytes)" % len(self._value)

#    def can_edit(self, max_length=-1)
#        if not self.can_display(self): return False
#
#        return True
#
    @inherit_docstring
    def reorder(self, new_index):
        return self._reorder(self.parent.values, new_index)

    def clone(self):
        obj = super(BaseValue, self).clone()
        obj._property = None
        return obj

    @inherit_docstring
    def get_terminology_equivalent(self):
        prop = self._property.get_terminology_equivalent()

        if prop is None:
            return None

        for val in prop:
            if val == self:  # TODO: shouldn't we take the index or st.?
                return val

        return None
<|endoftext|>


<commit_before>import math
import unittest

import numpy as np

from . import dynamic_time_warping

# unit test
class MetricsTest(unittest.TestCase):
    testSize = 100

    def sine(self):
        return np.array([math.sin(x) for x in range(self.testSize)])

    def test_dynamic_time_warping_sqr(self):
        dtwm = dynamic_time_warping.norm_sqr
        sine = np.array([self.sine().tolist(),
                         self.sine().tolist(),
                         self.sine().tolist()])
        self.assertLess(dtwm(sine, sine), 1)
        self.assertLess(dtwm(sine, sine+0.9),
                        dtwm(sine, sine+1))
        self.assertLess(dtwm(np.zeros((self.testSize, self.testSize)),
                             sine),
                        dtwm(np.zeros((self.testSize, self.testSize)),
                             sine*1.1))
        self.assertLess(dtwm(np.zeros((self.testSize, self.testSize)),
                             np.zeros((2*self.testSize, self.testSize))+1),
                        dtwm(np.zeros((self.testSize, self.testSize)),
                             np.zeros((2*self.testSize, self.testSize))+1.1))
        self.assertAlmostEqual(dtwm(sine, sine*1.1),
                               dtwm(sine*1.1, sine))
<commit_msg>added linear stretch tests<commit_after>import math
import unittest

import numpy as np

from . import dynamic_time_warping
from . import linear_stretch

# unit test
class MetricsTest(unittest.TestCase):
    testSize = 100

    def sine(self):
        return np.array([math.sin(x) for x in range(self.testSize)])

    def test_dynamic_time_warping_sqr(self):
        dtwm = dynamic_time_warping.norm_sqr
        sine = np.array([self.sine().tolist(),
                         self.sine().tolist(),
                         self.sine().tolist()])
        self.assertLess(dtwm(sine, sine), 1)
        self.assertLess(dtwm(sine, sine+0.9),
                        dtwm(sine, sine+1))
        self.assertLess(dtwm(np.zeros((self.testSize, self.testSize)),
                             sine),
                        dtwm(np.zeros((self.testSize, self.testSize)),
                             sine*1.1))
        self.assertLess(dtwm(np.zeros((self.testSize, self.testSize)),
                             np.zeros((2*self.testSize, self.testSize))+1),
                        dtwm(np.zeros((self.testSize, self.testSize)),
                             np.zeros((2*self.testSize, self.testSize))+1.1))
        self.assertAlmostEqual(dtwm(sine, sine*1.1),
                               dtwm(sine*1.1, sine))

    def test_stretch_signal(self):
        signal = [0, 2, 4]
        stretched = linear_stretch._stretch_signal(signal, 5)
        for i in range(5):
            self.assertEqual(stretched[i], i)

        signal = np.array([_ for _ in range(0, 20, 2)])
        assert len(signal) == 10
        stretched = linear_stretch._stretch_signal(signal, 19)
        for i in range(19):
            self.assertEqual(stretched[i], i)

    def test_linear_interpolation_stretching(self):
        signal1 = np.array([_ for _ in range(0, 20, 2)])
        signal2 = np.array([_ for _ in range(0, 19)])

        self.assertAlmostEqual(linear_stretch.get_metric(signal1, signal2), 0)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2016 Marc-Antoine Ruel. All Rights Reserved. Use of this
# source code is governed by a BSD-style license that can be found in the
# LICENSE file.

import argparse
import os
import shutil
import subprocess
import tarfile
import sys

def _semver(v):
  v = v.split('.')
  for i in range(len(v)):
    try:
      v[i] = int(v[i])
    except ValueError:
      pass
  return v

def _check_output(*args, **kwargs):
  if sys.version_info.major == 3:
    kwargs['universal_newlines'] = True
  return subprocess.check_output(*args, **kwargs)

def from_sources():
  """Installing from sources."""
  home = os.path.expanduser('~')
  goroot = os.environ.get('GOROOT')
  if not goroot:
    goroot = os.path.join(home, 'src', 'golang')
  print('Installing Go in %s' % goroot)
  if os.path.isdir(goroot):
    subprocess.check_call(['git', 'fetch', '--all'], cwd=goroot)
  else:
    subprocess.check_call(
        ['git', 'clone', 'https://go.googlesource.com/go', goroot])

  #TAG="$(git tag | grep "^go" | egrep -v "beta|rc" | tail -n 1)"
  tags = _check_output(['git', 'tag'], cwd=goroot).splitlines()
  tags = sorted(
      (t for t in tags
      if t.startswith('go') and 'beta' not in t and 'rc' not in t),
      key=_semver)
  tag = tags[-1]
  print('Using %s' % tag)
  subprocess.check_call(['git', 'checkout', tag], cwd=goroot)

  go14 = os.path.join(home, 'go1.4')
  if not os.path.isdir(go14):
    from_binary(go14)

  print('Building.')
  subprocess.check_call(['./make.bash'], cwd=os.path.join(goroot, 'src'))

def from_binary(goroot):
  """Installs from pre-built binaries."""
  print('Installing Go in %s' % goroot)
  if os.path.isdir(goroot):
    shutil.rmtree(goroot)
  # TODO(maruel): Magically figure out latest version as done in
  # https://github.com/periph/bootstrap/blob/master/setup.sh
  version = '1.12.7'
  uname = os.uname()[4]
  arch = 'amd64'
  if uname.startswith('arm'):
    # ~70MB, at 1Mbit it takes 12 minutes...
    arch = 'armv6l'

  os_name = sys.platform
  if os_name == 'linux2':
    os_name = 'linux'
  filename = 'go' + version + '.' + os_name + '-' + arch + '.tar.gz'
  url = 'https://storage.googleapis.com/golang/' + filename
  print('Fetching %s' % url)
  subprocess.check_call(['wget', url])
  if not os.path.isdir(goroot):
    os.mkdir(goroot)
  subprocess.check_call(['tar', '-C', goroot, '--strip-components=1', '-xzf', filename])
  os.remove(filename)

def setup_profile(goroot):
  """Sets up the global profilet to include system wide Go."""
  with open('/etc/profile.d/golang.sh', 'wb') as f:
    f.write('export PATH="$PATH:%s/bin"\n' % goroot)
  os.chmod('/etc/profile.d/golang.sh', 0o555)

def main():
  parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
  parser.add_argument('--system', action='store_true')
  parser.add_argument('--skip', action='store_true', help='Skip updating Golang')
  args = parser.parse_args()
  if not args.skip:
    if args.system:
      goroot = '/usr/local/go'
      from_binary(goroot)
      setup_profile(goroot)
    else:
      from_sources()
  if os.geteuid() != 0:
    # Start getting useful projects right away, if not running as root.
    subprocess.check_call(
        ['go', 'get', '-v',
      'golang.org/x/tools/cmd/godoc',
      'golang.org/x/tools/cmd/goimports',
      'golang.org/x/tools/cmd/stringer',
      'github.com/maruel/panicparse/cmd/pp',
      'github.com/rjeczalik/bin/cmd/gobin'])
    # Rebuild all binaries in $GOPATH/bin:
    #   github.com/FiloSottile/gorebuild
    # Find unused packages:
    #   github.com/nf/deadleaves
    # Count lines of code:
    #   github.com/bytbox/sloc
  else:
    print('Skipping tooling because running as root')
  return 0

if __name__ == '__main__':
  sys.exit(main())
<commit_msg>install_golang.py: install more stuff<commit_after>#!/usr/bin/env python
# Copyright 2016 Marc-Antoine Ruel. All Rights Reserved. Use of this
# source code is governed by a BSD-style license that can be found in the
# LICENSE file.

import argparse
import os
import shutil
import subprocess
import tarfile
import sys

def _semver(v):
  v = v.split('.')
  for i in range(len(v)):
    try:
      v[i] = int(v[i])
    except ValueError:
      pass
  return v

def _check_output(*args, **kwargs):
  if sys.version_info.major == 3:
    kwargs['universal_newlines'] = True
  return subprocess.check_output(*args, **kwargs)

def from_sources():
  """Installing from sources."""
  home = os.path.expanduser('~')
  goroot = os.environ.get('GOROOT')
  if not goroot:
    goroot = os.path.join(home, 'src', 'golang')
  print('Installing Go in %s' % goroot)
  if os.path.isdir(goroot):
    subprocess.check_call(['git', 'fetch', '--all'], cwd=goroot)
  else:
    subprocess.check_call(
        ['git', 'clone', 'https://go.googlesource.com/go', goroot])

  #TAG="$(git tag | grep "^go" | egrep -v "beta|rc" | tail -n 1)"
  tags = _check_output(['git', 'tag'], cwd=goroot).splitlines()
  tags = sorted(
      (t for t in tags
      if t.startswith('go') and 'beta' not in t and 'rc' not in t),
      key=_semver)
  tag = tags[-1]
  print('Using %s' % tag)
  subprocess.check_call(['git', 'checkout', tag], cwd=goroot)

  go14 = os.path.join(home, 'go1.4')
  if not os.path.isdir(go14):
    from_binary(go14)

  print('Building.')
  subprocess.check_call(['./make.bash'], cwd=os.path.join(goroot, 'src'))

def from_binary(goroot):
  """Installs from pre-built binaries."""
  print('Installing Go in %s' % goroot)
  if os.path.isdir(goroot):
    shutil.rmtree(goroot)
  # TODO(maruel): Magically figure out latest version as done in
  # https://github.com/periph/bootstrap/blob/master/setup.sh
  version = '1.12.7'
  uname = os.uname()[4]
  arch = 'amd64'
  if uname.startswith('arm'):
    # ~70MB, at 1Mbit it takes 12 minutes...
    arch = 'armv6l'

  os_name = sys.platform
  if os_name == 'linux2':
    os_name = 'linux'
  filename = 'go' + version + '.' + os_name + '-' + arch + '.tar.gz'
  url = 'https://storage.googleapis.com/golang/' + filename
  print('Fetching %s' % url)
  subprocess.check_call(['wget', url])
  if not os.path.isdir(goroot):
    os.mkdir(goroot)
  subprocess.check_call(['tar', '-C', goroot, '--strip-components=1', '-xzf', filename])
  os.remove(filename)

def setup_profile(goroot):
  """Sets up the global profilet to include system wide Go."""
  with open('/etc/profile.d/golang.sh', 'wb') as f:
    f.write('export PATH="$PATH:%s/bin"\n' % goroot)
  os.chmod('/etc/profile.d/golang.sh', 0o555)

def main():
  parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
  parser.add_argument('--system', action='store_true')
  parser.add_argument('--skip', action='store_true', help='Skip updating Golang')
  args = parser.parse_args()
  if not args.skip:
    if args.system:
      goroot = '/usr/local/go'
      from_binary(goroot)
      setup_profile(goroot)
    else:
      from_sources()
  if os.geteuid() != 0:
    # Start getting useful projects right away, if not running as root.
    subprocess.check_call(
        ['go', 'get', '-v',
      'golang.org/x/tools/cmd/godoc',
      'golang.org/x/tools/cmd/goimports',
      'golang.org/x/tools/cmd/stringer',
      'github.com/boyter/scc',
      'github.com/maruel/panicparse/cmd/pp',
      'github.com/monochromegane/the_platinum_searcher/cmd/pt',
      'github.com/rjeczalik/bin/cmd/gobin',
      'github.com/FiloSottile/gorebuild',
      ])
  else:
    print('Skipping tooling because running as root')
  return 0

if __name__ == '__main__':
  sys.exit(main())
<|endoftext|>


<commit_before>VERSION = '0.2.0'
<commit_msg>0.2.1<commit_after>VERSION = '0.2.1'
<|endoftext|>


<commit_before># Copyright 2016 Raytheon BBN Technologies
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0

import socket
import time
import re
import numpy as np
from .instrument import SCPIInstrument, StringCommand, FloatCommand, IntCommand, is_valid_ipv4
from auspex.log import logger

class Agilent34970A(SCPIInstrument):
    """Agilent 34970A MUX"""

# Allowed value arrays

    RES_VALUES      = [1E2, 1E3, 1E4, 1E5, 1E6, 1E7, 1E8]
    PLC_VALUES      = [0.02, 0.2, 1, 10, 20, 100, 200]
    ONOFF_VALUES    = ['ON', 'OFF']
    TRIGSOUR_VALUES = ['BUS','IMM','EXT','TIM']
    ADVSOUR_VALUES  = ['EXT','BUS','IMM']

# Commands needed to configure MUX for measurement with an external instrument

    dmm            = StringCommand(scpi_string="INST:DMM",value_map={'ON': '1', 'OFF': '0'})
    trigger_source = StringCommand(scpi_string="TRIG:SOUR",allowed_values=TRIGSOUR_VALUES)
    advance_source = StringCommand(scpi_string="ROUT:CHAN:ADV:SOUR",allowed_values=ADVSOUR_VALUES)

# Generic init and connect methods

    def __init__(self, resource_name=None, *args, **kwargs):
        super(Agilent34970A, self).__init__(resource_name, *args, **kwargs)
        self.name = "Agilent 34970A MUX"

    def connect(self, resource_name=None, interface_type=None):
        if resource_name is not None:
            self.resource_name = resource_name
        super(Agilent34970A, self).connect(resource_name=self.resource_name, interface_type=interface_type)
        self.interface._resource.read_termination = u"\n"

#Channel to String helper function converts int array to channel list string

    def ch_to_str(self, ch_list):
        return ("(@"+','.join(['{:d}']*len(ch_list))+")").format(*ch_list)

#Setup Scan List

    @property
    def scanlist(self):
        slist = re.findall('\d{3}(?=[,)])',self.interface.query("ROUT:SCAN?"))
        return [int(i) for i in slist]

    @scanlist.setter
    def scanlist(self, ch_list):
        self.interface.write("ROUT:SCAN "+self.ch_to_str(ch_list))

#Start Scan
    def scan(self):
        self.interface.write("INIT")

#Read Values
    def read(self):
        if self.dmm=="ON":
            return self.interface.query_ascii_values("FETC?", converter=u'e')
        else:
            raise Exception("Cannot issue command when DMM is disabled. Enable DMM")

# Commands that configure resistance measurement type, 2 or 4 wire

    def set_resistance_chan(self, ch_list, fw=False):
        if self.dmm=="ON":
            fw_char = "F" if fw else ""
            self.interface.write(("CONF:{}RES "+self.ch_to_str(ch_list)).format(fw_char))
        else:
            fw_char = "ON," if fw else "OFF,"
            self.interface.write(("ROUT:CHAN:FWIR {}"+self.ch_to_str(ch_list)).format(fw_char))

    def get_resistance_chan(self, ch_list):
        if self.dmm=="ON":
            query_str = "SENS:FUNC? "+self.ch_to_str(ch_list)
            output = self.interface.query_ascii_values(query_str, converter=u's')
        else:
            query_str = "ROUT:CHAN:FWIR? "+self.ch_to_str(ch_list)
            output = self.interface.query_ascii_values(query_str, converter=u'd')
        return {ch: val for ch, val in zip(ch_list, output) }

# Commands that configure resistance measurements with internal DMM

    def set_resistance_range(self, val, ch_list, fw=False):
        fw_char = "F" if fw else ""
        if val not in self.RES_VALUES:
            raise ValueError(("Resistance range must be "+'|'.join(['{:E}']*len(self.RES_VALUES))+" Ohms").format(*self.RES_VALUES))
        if self.dmm=="OFF":
            raise Exception("Cannot issue command when DMM is disabled. Enable DMM")
        else:
            self.interface.write(("SENS:{}RES:RANG {:E},"+self.ch_to_str(ch_list)).format(fw_char,val))

    def get_resistance_range(self, ch_list, fw=False):
        fw_char = "F" if fw else ""
        if self.dmm=="OFF":
            raise Exception("Cannot issue command when DMM is disabled. Enable DMM")
        else:
            query_str = ("SENS:{}RES:RANG? "+self.ch_to_str(ch_list)).format(fw_char)
            output = self.interface.query_ascii_values(query_str, converter=u'e')
            return {ch: val for ch, val in zip(ch_list, output) }

    def set_resistance_resolution(self, val, ch_list, fw=False):
        fw_char = "F" if fw else ""
        if val not in self.PLC_VALUES:
            raise ValueError(("PLC integration times must be "+'|'.join(['{:E}']*len(self.PLC_VALUES))+" cycles").format(*self.PLC_VALUES))
        if self.dmm=="OFF":
            raise Exception("Cannot issue command when DMM is disabled. Enable DMM")
        else:
            self.interface.write(("SENS:{}RES:NPLC {:E},"+self.ch_to_str(ch_list)).format(fw_char,val))

    def get_resistance_resolution(self, ch_list, fw=False):
        fw_char = "F" if fw else ""
        if self.dmm=="OFF":
            raise Exception("Cannot issue command when DMM is disabled. Enable DMM")
        else:
            query_str = ("SENS:{}RES:NPLC? "+self.ch_to_str(ch_list)).format(fw_char)
            output = self.interface.query_ascii_values(query_str, converter=u'e')
            return {ch: val for ch, val in zip(ch_list, output) }

    def set_resistance_zcomp(self, val, ch_list, fw=False):
        fw_char = "F" if fw else ""
        if val not in self.ONOFF_VALUES:
            raise ValueError("Zero compensation must be ON or OFF. Only valid for resistance range less than 100 kOhm")
        if self.dmm=="OFF":
            raise Exception("Cannot issue command when DMM is disabled. Enable DMM")
        else:
            self.interface.write(("SENS:{}RES:OCOM {:s},"+self.ch_to_str(ch_list)).format(fw_char,val))

    def get_resistance_zcomp(self, ch_list, fw=False):
        fw_char = "F" if fw else ""
        if self.dmm=="OFF":
            raise Exception("Cannot issue command when DMM is disabled. Enable DMM")
        else:
            query_str = ("SENS:{}RES:OCOM? "+self.ch_to_str(ch_list)).format(fw_char)
            output = self.interface.query_ascii_values(query_str, converter=u'd')
            return {ch: val for ch, val in zip(ch_list, output) }

class AgilentN5183A(SCPIInstrument):
    """AgilentN5183A microwave source"""
    instrument_type = "Microwave Source"

    frequency = FloatCommand(scpi_string=":freq")
    power     = FloatCommand(scpi_string=":power")
    phase     = FloatCommand(scpi_string=":phase")

    alc       = StringCommand(scpi_string=":power:alc", value_map={True: '1', False: '0'})
    mod       = StringCommand(scpi_string=":output:mod", value_map={True: '1', False: '0'})

    output    = StringCommand(scpi_string=":output", value_map={True: '1', False: '0'})

    def __init__(self, resource_name=None, *args, **kwargs):
        #If we only have an IP address then tack on the raw socket port to the VISA resource string
        super(AgilentN5183A, self).__init__(resource_name, *args, **kwargs)

    def connect(self, resource_name=None, interface_type=None):
        if resource_name is not None:
            self.resource_name = resource_name
        if is_valid_ipv4(self.resource_name):
            self.resource_name += "::5025::SOCKET"
        print(self.resource_name)
        super(AgilentN5183A, self).connect(resource_name=resource_name, interface_type=interface_type)
        self.interface._resource.read_termination = u"\n"
        self.interface._resource.write_termination = u"\n"
        self.interface._resource.timeout = 3000 #seem to have trouble timing out on first query sometimes

    def set_all(self, settings):
        settings['frequency'] = settings['frequency']*1e9
        super(AgilentN5183A, self).set_all(settings)

class AgilentE8363C(SCPIInstrument):
    """Agilent E8363C VNA"""
    instrument_type = "Vector Network Analyzer"

    power              = FloatCommand(scpi_string=":SOURce:POWer:LEVel:IMMediate:AMPLitude", value_range=(-27, 20))
    frequency_center   = FloatCommand(scpi_string=":SENSe:FREQuency:CENTer")
    frequency_span     = FloatCommand(scpi_string=":SENSe:FREQuency:SPAN")
    frequency_start    = FloatCommand(scpi_string=":SENSe:FREQuency:STARt")
    frequency_stop     = FloatCommand(scpi_string=":SENSe:FREQuency:STOP")
    sweep_num_points   = IntCommand(scpi_string=":SENSe:SWEep:POINts")
    averaging_factor   = IntCommand(scpi_string=":SENSe1:AVERage:COUNt")
    averaging_enable   = StringCommand(get_string=":SENSe1:AVERage:STATe?", set_string=":SENSe1:AVERage:STATe {:c}", value_map={False:"0", True:"1"})
    averaging_complete = StringCommand(get_string=":STATus:OPERation:AVERaging1:CONDition?", value_map={False:"+0", True:"+2"})

    def __init__(self, resource_name=None, *args, **kwargs):
        #If we only have an IP address then tack on the raw socket port to the VISA resource string
        super(AgilentE8363C, self).__init__(resource_name, *args, **kwargs)

    def connect(self, resource_name=None, interface_type="VISA"):
        if resource_name is not None:
            self.resource_name = resource_name
        if is_valid_ipv4(self.resource_name):
            self.resource_name += "::hpib7,16::INSTR"
        else:
            logger.error("The resource name for the Agilent E8363C: {} is " +
                "not a valid IPv4 address.".format(self.resource_name))
        super(AgilentE8363C, self).connect(resource_name=None,
            interface_type=interface_type)
        self.interface._resource.read_termination = u"\n"
        self.interface._resource.write_termination = u"\n"

    def averaging_restart(self):
        """ Restart trace averaging """
        self.interface.write(":SENSe1:AVERage:CLEar")

    def reaverage(self):
        """ Restart averaging and block until complete """
        self.averaging_restart()
        while not self.averaging_complete:
            #TODO with Python 3.5 turn into coroutine and use await asyncio.sleep()
            time.sleep(0.1)

    def get_trace(self, measurement=None):
        """ Return a tupple of the trace frequencies and corrected complex points """
        #If the measurement is not passed in just take the first one
        if measurement is None:
            traces = self.interface.query(":CALCulate:PARameter:CATalog?")
            #traces come e.g. as  u'"CH1_S11_1,S11,CH1_S21_2,S21"'
            #so split on comma and avoid first quote
            measurement = traces.split(",")[0][1:]
        #Select the measurment
        self.interface.write(":CALCulate:PARameter:SELect '{}'".format(measurement))

        #Take the data as interleaved complex values
        interleaved_vals = self.interface.values(":CALCulate:DATA? SDATA")
        vals = interleaved_vals[::2] + 1j*interleaved_vals[1::2]

        #Get the associated frequencies
        freqs = np.linspace(self.frequency_start, self.frequency_stop, self.sweep_num_points)

        return (freqs, vals)

class AgilentE9010A(SCPIInstrument):
    """Agilent E9010A SA"""
    instrument_type = "Spectrum Analyzer"

    frequency_center = FloatCommand(scpi_string=":FREQuency:CENTer")
    frequency_span   = FloatCommand(scpi_string=":FREQuency:SPAN")
    frequency_start  = FloatCommand(scpi_string=":FREQuency:STARt")
    frequency_stop   = FloatCommand(scpi_string=":FREQuency:STOP")

    num_sweep_points = FloatCommand(scpi_string=":SWEep:POINTs")
    resolution_bandwidth = FloatCommand(scpi_string=":BANDwidth")
    sweep_time = FloatCommand(scpi_string=":SWEep:TIME")

    def __init__(self, resource_name=None, *args, **kwargs):
        super(AgilentE9010A, self).__init__(resource_name, *args, **kwargs)

    def connect(self, resource_name=None, interface_type=None):
        if resource_name is not None:
            self.resource_name = resource_name
        #If we only have an IP address then tack on the raw socket port to the VISA resource string
        if is_valid_ipv4(resource_name):
            resource_name += "::5025::SOCKET"
        super(AgilentE9010A, self).connect(resource_name=resource_name, interface_type=interface_type)
        self.interface._resource.read_termination = u"\n"
        self.interface._resource.write_termination = u"\n"
        self.interface._resource.timeout = 3000 #seem to have trouble timing out on first query sometimes

    def get_axis(self):
        return np.linspace(self.frequency_start, self.frequency_stop, self.num_sweep_points)

    def get_trace(self, num=1):
        self.interface.write(':FORM:DATA REAL,32')
        return self.interface.query_binary_values(":TRACE:DATA? TRACE{:d}".format(num),
            datatype="f", is_big_endian=True)

    def restart_sweep(self):
        """ Aborts current sweep and restarts. """
        self.interface.write(":INITiate:RESTart")
<commit_msg>Agilent N9010A: phase noise application commands<commit_after># Copyright 2016 Raytheon BBN Technologies
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0

import socket
import time
import re
import numpy as np
from .instrument import SCPIInstrument, StringCommand, FloatCommand, IntCommand, is_valid_ipv4
from auspex.log import logger

class Agilent34970A(SCPIInstrument):
    """Agilent 34970A MUX"""

# Allowed value arrays

    RES_VALUES      = [1E2, 1E3, 1E4, 1E5, 1E6, 1E7, 1E8]
    PLC_VALUES      = [0.02, 0.2, 1, 10, 20, 100, 200]
    ONOFF_VALUES    = ['ON', 'OFF']
    TRIGSOUR_VALUES = ['BUS','IMM','EXT','TIM']
    ADVSOUR_VALUES  = ['EXT','BUS','IMM']

# Commands needed to configure MUX for measurement with an external instrument

    dmm            = StringCommand(scpi_string="INST:DMM",value_map={'ON': '1', 'OFF': '0'})
    trigger_source = StringCommand(scpi_string="TRIG:SOUR",allowed_values=TRIGSOUR_VALUES)
    advance_source = StringCommand(scpi_string="ROUT:CHAN:ADV:SOUR",allowed_values=ADVSOUR_VALUES)

# Generic init and connect methods

    def __init__(self, resource_name=None, *args, **kwargs):
        super(Agilent34970A, self).__init__(resource_name, *args, **kwargs)
        self.name = "Agilent 34970A MUX"

    def connect(self, resource_name=None, interface_type=None):
        if resource_name is not None:
            self.resource_name = resource_name
        super(Agilent34970A, self).connect(resource_name=self.resource_name, interface_type=interface_type)
        self.interface._resource.read_termination = u"\n"

#Channel to String helper function converts int array to channel list string

    def ch_to_str(self, ch_list):
        return ("(@"+','.join(['{:d}']*len(ch_list))+")").format(*ch_list)

#Setup Scan List

    @property
    def scanlist(self):
        slist = re.findall('\d{3}(?=[,)])',self.interface.query("ROUT:SCAN?"))
        return [int(i) for i in slist]

    @scanlist.setter
    def scanlist(self, ch_list):
        self.interface.write("ROUT:SCAN "+self.ch_to_str(ch_list))

#Start Scan
    def scan(self):
        self.interface.write("INIT")

#Read Values
    def read(self):
        if self.dmm=="ON":
            return self.interface.query_ascii_values("FETC?", converter=u'e')
        else:
            raise Exception("Cannot issue command when DMM is disabled. Enable DMM")

# Commands that configure resistance measurement type, 2 or 4 wire

    def set_resistance_chan(self, ch_list, fw=False):
        if self.dmm=="ON":
            fw_char = "F" if fw else ""
            self.interface.write(("CONF:{}RES "+self.ch_to_str(ch_list)).format(fw_char))
        else:
            fw_char = "ON," if fw else "OFF,"
            self.interface.write(("ROUT:CHAN:FWIR {}"+self.ch_to_str(ch_list)).format(fw_char))

    def get_resistance_chan(self, ch_list):
        if self.dmm=="ON":
            query_str = "SENS:FUNC? "+self.ch_to_str(ch_list)
            output = self.interface.query_ascii_values(query_str, converter=u's')
        else:
            query_str = "ROUT:CHAN:FWIR? "+self.ch_to_str(ch_list)
            output = self.interface.query_ascii_values(query_str, converter=u'd')
        return {ch: val for ch, val in zip(ch_list, output) }

# Commands that configure resistance measurements with internal DMM

    def set_resistance_range(self, val, ch_list, fw=False):
        fw_char = "F" if fw else ""
        if val not in self.RES_VALUES:
            raise ValueError(("Resistance range must be "+'|'.join(['{:E}']*len(self.RES_VALUES))+" Ohms").format(*self.RES_VALUES))
        if self.dmm=="OFF":
            raise Exception("Cannot issue command when DMM is disabled. Enable DMM")
        else:
            self.interface.write(("SENS:{}RES:RANG {:E},"+self.ch_to_str(ch_list)).format(fw_char,val))

    def get_resistance_range(self, ch_list, fw=False):
        fw_char = "F" if fw else ""
        if self.dmm=="OFF":
            raise Exception("Cannot issue command when DMM is disabled. Enable DMM")
        else:
            query_str = ("SENS:{}RES:RANG? "+self.ch_to_str(ch_list)).format(fw_char)
            output = self.interface.query_ascii_values(query_str, converter=u'e')
            return {ch: val for ch, val in zip(ch_list, output) }

    def set_resistance_resolution(self, val, ch_list, fw=False):
        fw_char = "F" if fw else ""
        if val not in self.PLC_VALUES:
            raise ValueError(("PLC integration times must be "+'|'.join(['{:E}']*len(self.PLC_VALUES))+" cycles").format(*self.PLC_VALUES))
        if self.dmm=="OFF":
            raise Exception("Cannot issue command when DMM is disabled. Enable DMM")
        else:
            self.interface.write(("SENS:{}RES:NPLC {:E},"+self.ch_to_str(ch_list)).format(fw_char,val))

    def get_resistance_resolution(self, ch_list, fw=False):
        fw_char = "F" if fw else ""
        if self.dmm=="OFF":
            raise Exception("Cannot issue command when DMM is disabled. Enable DMM")
        else:
            query_str = ("SENS:{}RES:NPLC? "+self.ch_to_str(ch_list)).format(fw_char)
            output = self.interface.query_ascii_values(query_str, converter=u'e')
            return {ch: val for ch, val in zip(ch_list, output) }

    def set_resistance_zcomp(self, val, ch_list, fw=False):
        fw_char = "F" if fw else ""
        if val not in self.ONOFF_VALUES:
            raise ValueError("Zero compensation must be ON or OFF. Only valid for resistance range less than 100 kOhm")
        if self.dmm=="OFF":
            raise Exception("Cannot issue command when DMM is disabled. Enable DMM")
        else:
            self.interface.write(("SENS:{}RES:OCOM {:s},"+self.ch_to_str(ch_list)).format(fw_char,val))

    def get_resistance_zcomp(self, ch_list, fw=False):
        fw_char = "F" if fw else ""
        if self.dmm=="OFF":
            raise Exception("Cannot issue command when DMM is disabled. Enable DMM")
        else:
            query_str = ("SENS:{}RES:OCOM? "+self.ch_to_str(ch_list)).format(fw_char)
            output = self.interface.query_ascii_values(query_str, converter=u'd')
            return {ch: val for ch, val in zip(ch_list, output) }

class AgilentN5183A(SCPIInstrument):
    """AgilentN5183A microwave source"""
    instrument_type = "Microwave Source"

    frequency = FloatCommand(scpi_string=":freq")
    power     = FloatCommand(scpi_string=":power")
    phase     = FloatCommand(scpi_string=":phase")

    alc       = StringCommand(scpi_string=":power:alc", value_map={True: '1', False: '0'})
    mod       = StringCommand(scpi_string=":output:mod", value_map={True: '1', False: '0'})

    output    = StringCommand(scpi_string=":output", value_map={True: '1', False: '0'})

    def __init__(self, resource_name=None, *args, **kwargs):
        #If we only have an IP address then tack on the raw socket port to the VISA resource string
        super(AgilentN5183A, self).__init__(resource_name, *args, **kwargs)

    def connect(self, resource_name=None, interface_type=None):
        if resource_name is not None:
            self.resource_name = resource_name
        if is_valid_ipv4(self.resource_name):
            self.resource_name += "::5025::SOCKET"
        print(self.resource_name)
        super(AgilentN5183A, self).connect(resource_name=resource_name, interface_type=interface_type)
        self.interface._resource.read_termination = u"\n"
        self.interface._resource.write_termination = u"\n"
        self.interface._resource.timeout = 3000 #seem to have trouble timing out on first query sometimes

    def set_all(self, settings):
        settings['frequency'] = settings['frequency']*1e9
        super(AgilentN5183A, self).set_all(settings)

class AgilentE8363C(SCPIInstrument):
    """Agilent E8363C VNA"""
    instrument_type = "Vector Network Analyzer"

    power              = FloatCommand(scpi_string=":SOURce:POWer:LEVel:IMMediate:AMPLitude", value_range=(-27, 20))
    frequency_center   = FloatCommand(scpi_string=":SENSe:FREQuency:CENTer")
    frequency_span     = FloatCommand(scpi_string=":SENSe:FREQuency:SPAN")
    frequency_start    = FloatCommand(scpi_string=":SENSe:FREQuency:STARt")
    frequency_stop     = FloatCommand(scpi_string=":SENSe:FREQuency:STOP")
    sweep_num_points   = IntCommand(scpi_string=":SENSe:SWEep:POINts")
    averaging_factor   = IntCommand(scpi_string=":SENSe1:AVERage:COUNt")
    averaging_enable   = StringCommand(get_string=":SENSe1:AVERage:STATe?", set_string=":SENSe1:AVERage:STATe {:c}", value_map={False:"0", True:"1"})
    averaging_complete = StringCommand(get_string=":STATus:OPERation:AVERaging1:CONDition?", value_map={False:"+0", True:"+2"})

    def __init__(self, resource_name=None, *args, **kwargs):
        #If we only have an IP address then tack on the raw socket port to the VISA resource string
        super(AgilentE8363C, self).__init__(resource_name, *args, **kwargs)

    def connect(self, resource_name=None, interface_type="VISA"):
        if resource_name is not None:
            self.resource_name = resource_name
        if is_valid_ipv4(self.resource_name):
            self.resource_name += "::hpib7,16::INSTR"
        else:
            logger.error("The resource name for the Agilent E8363C: {} is " +
                "not a valid IPv4 address.".format(self.resource_name))
        super(AgilentE8363C, self).connect(resource_name=None,
            interface_type=interface_type)
        self.interface._resource.read_termination = u"\n"
        self.interface._resource.write_termination = u"\n"

    def averaging_restart(self):
        """ Restart trace averaging """
        self.interface.write(":SENSe1:AVERage:CLEar")

    def reaverage(self):
        """ Restart averaging and block until complete """
        self.averaging_restart()
        while not self.averaging_complete:
            #TODO with Python 3.5 turn into coroutine and use await asyncio.sleep()
            time.sleep(0.1)

    def get_trace(self, measurement=None):
        """ Return a tupple of the trace frequencies and corrected complex points """
        #If the measurement is not passed in just take the first one
        if measurement is None:
            traces = self.interface.query(":CALCulate:PARameter:CATalog?")
            #traces come e.g. as  u'"CH1_S11_1,S11,CH1_S21_2,S21"'
            #so split on comma and avoid first quote
            measurement = traces.split(",")[0][1:]
        #Select the measurment
        self.interface.write(":CALCulate:PARameter:SELect '{}'".format(measurement))

        #Take the data as interleaved complex values
        interleaved_vals = self.interface.values(":CALCulate:DATA? SDATA")
        vals = interleaved_vals[::2] + 1j*interleaved_vals[1::2]

        #Get the associated frequencies
        freqs = np.linspace(self.frequency_start, self.frequency_stop, self.sweep_num_points)

        return (freqs, vals)

class AgilentE9010A(SCPIInstrument):
    """Agilent E9010A SA"""
    instrument_type = "Spectrum Analyzer"

    frequency_center = FloatCommand(scpi_string=":FREQuency:CENTer")
    frequency_span   = FloatCommand(scpi_string=":FREQuency:SPAN")
    frequency_start  = FloatCommand(scpi_string=":FREQuency:STARt")
    frequency_stop   = FloatCommand(scpi_string=":FREQuency:STOP")

    num_sweep_points = FloatCommand(scpi_string=":SWEep:POINTs")
    resolution_bandwidth = FloatCommand(scpi_string=":BANDwidth")
    sweep_time = FloatCommand(scpi_string=":SWEep:TIME")

    mode = StringCommand(scpi_string=":INSTrument", allowed_values=["SA", "BASIC", "PULSE", "PNOISE"])

    # phase noise application commands
    pn_offset_start = FloatCommand(scpi_string=":LPLot:FREQuency:OFFSet:STARt")
    pn_offset_stop  = FloatCommand(scpi_string=":LPLot:FREQuency:OFFSet:STOP")
    pn_carrier_freq = FloatCommand(scpi_string=":FREQuency:CARRier")

    def __init__(self, resource_name=None, *args, **kwargs):
        super(AgilentE9010A, self).__init__(resource_name, *args, **kwargs)

    def connect(self, resource_name=None, interface_type=None):
        if resource_name is not None:
            self.resource_name = resource_name
        #If we only have an IP address then tack on the raw socket port to the VISA resource string
        if is_valid_ipv4(resource_name):
            resource_name += "::5025::SOCKET"
        super(AgilentE9010A, self).connect(resource_name=resource_name, interface_type=interface_type)
        self.interface._resource.read_termination = u"\n"
        self.interface._resource.write_termination = u"\n"
        self.interface._resource.timeout = 3000 #seem to have trouble timing out on first query sometimes

    def get_axis(self):
        return np.linspace(self.frequency_start, self.frequency_stop, self.num_sweep_points)

    def get_trace(self, num=1):
        self.interface.write(':FORM:DATA REAL,32')
        return self.interface.query_binary_values(":TRACE:DATA? TRACE{:d}".format(num),
            datatype="f", is_big_endian=True)

    def get_pn_trace(self, num=3):
        # num = 3 is raw data
        # num = 4 is smoothed data
        # returns a tuple of (freqs, dBc/Hz)
        self.interface.write(":FORM:DATA ASCII")
        response = self.interface.query(":FETCH:LPLot{:d}?".format(num))
        xypts = np.array([float(x) for x in response.split(',')])
        return xypts[::2], xypts[1::2]

    def restart_sweep(self):
        """ Aborts current sweep and restarts. """
        self.interface.write(":INITiate:RESTart")
<|endoftext|>