<commit_before># coding: utf-8

from __future__ import absolute_import, division, print_function

"""
Examine the distribution of diffraction spot intensities.

This module defines a class IntensityDist, with several methods for exploring
the distribution of measured spot intensities in an X-ray diffraction
experiment.  The user may wish to use this information to inform decisions
regarding the error model employed in analysing the data.  Data are passed in
as an unmerged MTZ file (see http://www.ccp4.ac.uk/html/mtzformat.html) and the
resulting IntensityDist instance contains the pertinent columns of data, along
with normal order statistic medians of the z-scores of the intensities, for
constructing a normal probability plot (See
https://www.itl.nist.gov/div898/handbook/eda/section3/normprpl.htm).

If called as a script, read data from an unmerged MTZ file; generate a
histogram and a normal probability plot of the z-scores of the intensity data,
along with plots of z as a function of batch number, of multiplicity, of
detector position, of measured multiplicity, of absolute intensity and of
I/sigma.
"""

import logging
from dials.array_family import flex

log = logging.getLogger("dials.util.intensity_explorer")

class IntensityDist(object):
    def __init__(
        self,
        rtable,
        elist,
        calculate_variances=False,
        keep_singles=False,
        uncertainty="sigma",
        outfile=None,
    ):
        """
        Generate z-scores and a normal probability plot from a DIALS
        reflection_table and a dxtbx ExperimentList, containing the observations
        and the corresponding experiments, respectively.

        :param rtable: A reflection table object, containing at least the columns
          * ``miller_index``
          * ``intensity.sum.value``
          * ``intensity.sum.variance``
          * ``xyzobs.px.value``
        :type rtable: dials.array_family_flex_ext.reflection_table
        :param elist: A corresponding experiment list.
        :type elist: dxtbx_model_ext.ExperimentList
        :param calculate_variances: Choose whether to calculate weighted
        aggregate variances.  Doing so incurs a performance penalty.
        Defaullts to False.
        :type calculate_variances: bool
        :param keep_singles: Choose whether to keep multiplicity-1 reflections.
        Defaults to False.
        :type keep_singles: bool
        :param uncertainty: Measure of spread to use in normalising the
        z-scores, i.e. z = (I - <I>) / uncertainty.
        Possible values for uncertainty:
        * 'sigma':    Use measured sigma values;
        * 'stddev':   Use sample standard deviations calculated as
                      square-root of unbiased weighted sample variances
                      of symmetry-equivalent reflection intensities;
        Defaults to 'sigma'.
        :type uncertainty: str
        :param outfile: Filename root for output PNG plots.
        Defaults to None.
        :type: outfile: str
        """

        from dxtbx.model import ExperimentList
        from cctbx import miller

        if not isinstance(rtable, flex.reflection_table) or not isinstance(
            elist, ExperimentList
        ):
            raise TypeError(
                "Must be called with a reflection table and an experiment list."
            )

        rtable = rtable.copy()
        # Discard unindexed reflections (only necessary because of
        # https://github.com/dials/dials/issues/615 —
        # TODO remove the line below when issue #615 is fixed).
        rtable.del_selected(rtable["id"] == -1)
        rtable["miller_index.asu"] = rtable["miller_index"]

        # Divide reflections by the space group to which they have been indexed.
        self.rtables = {
            expt.crystal.get_space_group().make_tidy(): flex.reflection_table()
            for expt in elist
        }
        for expt, sel in rtable.iterate_experiments_and_indices(elist):
            sg = expt.crystal.get_space_group().make_tidy()
            self.rtables[sg].extend(rtable.select(sel))
        # Map Miller indices to asymmetric unit.
        for space_group, rtable in self.rtables.items():
            # TODO Handle anomalous flag sensibly.  Currently assumes not anomalous.
            miller.map_to_asu(space_group.type(), False, rtable["miller_index.asu"])

        # Calculate normal probability plot data.
        self._multiplicity_mean_error_stddev(
            calculate_variances=calculate_variances, keep_singles=keep_singles
        )
        self._make_z(uncertainty)
        self._probplot_data()

        self.rtable = flex.reflection_table()
        for rtable in self.rtables.values():
            self.rtable.extend(rtable)

        if not outfile:
            outfile = ""
        self.outfile = outfile

    def _multiplicity_mean_error_stddev(
        self, calculate_variances=False, keep_singles=False
    ):
        """"
        Calculate aggregate properties of grouped symmetry-equivalent reflections.

        Populate the reflection table of observations with the following
        properties:
          * ``multiplicity`` — Multiplicity of observations of a given reflection
          in the asymmetric unit;
          :type: `dials.array_family_flex_ext.int` array
          * ``intensity.mean.value`` — Mean of symmetry-equivalent reflections,
          weighted by measurement error;
          :type: `dials.array_family_flex_ext.double` array
          * ``intensity.mean.std_error`` — Standard error on the weighted mean;
          :type: `dials.array_family_flex_ext.double` array
          * (optional) ``intensity.mean.variance`` — variance of
          symmetry-equivalent reflections, weighted by measurement error;
          :type: `dials.array_family_flex_ext.double` array

        :param calculate_variances: Elect whether to calculate the weighted
        variances.  Defaults to False, to spare an expensive computation.
        :type calculate_variances: bool
        :param keep_singles: Choose whether to keep single-multiplicity
        reflections.
        :type keep_singles: bool
        """

        for key, rtable in self.rtables.items():
            # Sort the reflection table for speedier iteration.
            rtable.sort("miller_index.asu")
            # Record the positions of any multiplicity-1 reflections.
            if not keep_singles:
                singles = flex.size_t()
            # Record the multiplicities.
            multiplicity = flex.int()
            # For weighted averaging.
            weights = 1 / rtable["intensity.sum.variance"]
            sum_weights = flex.double()
            if calculate_variances:
                sum_square_weights = flex.double()
            # Calculate the weighted mean intensities.
            i_means = flex.double()
            # Calculate the standard deviations from unbiased weighted variances.
            variances = flex.double()

            # Iterate over the reflections, grouping by equivalent Miller index,
            # to calculate multiplicities, weighted mean intensities, etc..
            # Some time can be saved by only calculating variances if necessary.
            # Initial values:
            prev_index = None
            count = 1
            # One big loop through the entire reflection table:
            for j in range(rtable.size()):
                index = rtable["miller_index.asu"][j]
                weight = weights[j]
                # Aggregate within a symmetry-equivalent group of reflections:
                if index == prev_index:
                    count += 1
                    i_sum += weight * rtable["intensity.sum.value"][j]
                    sum_weight += weight
                    if calculate_variances:
                        sum_square_weight += weight * weight
                # Record the aggregated values for the group:
                elif prev_index:
                    if count == 1 and not keep_singles:
                        singles.append(j - 1)
                    multiplicity.extend(flex.int(count, count))
                    i_means.extend(flex.double(count, i_sum / sum_weight))
                    sum_weights.extend(flex.double(count, sum_weight))
                    if calculate_variances:
                        sum_square_weights.extend(flex.double(count, sum_square_weight))
                    # And reinitialise:
                    prev_index = index
                    count = 1
                    i_sum = weight * rtable["intensity.sum.value"][j]
                    sum_weight = weight
                    if calculate_variances:
                        sum_square_weight = weight * weight
                # Handle the first row:
                else:
                    prev_index = rtable["miller_index.asu"][j]
                    i_sum = weight * rtable["intensity.sum.value"][j]
                    sum_weight = weight
                    if calculate_variances:
                        sum_square_weight = weight * weight
            # Record the aggregated values for the last group:
            if count == 1 and not keep_singles:
                singles.append(rtable.size() - 1)
            multiplicity.extend(flex.int(count, count))
            i_means.extend(flex.double(count, i_sum / sum_weight))
            sum_weights.extend(flex.double(count, sum_weight))
            if calculate_variances:
                sum_square_weights.extend(flex.double(count, sum_square_weight))

            # Discard singletons:
            if not keep_singles:
                singles_del = flex.bool(rtable.size(), True)
                singles_del.set_selected(singles, False)
                multiplicity, weights, sum_weights, i_means = [
                    a.select(singles_del)
                    for a in (multiplicity, weights, sum_weights, i_means)
                ]
                rtable.del_selected(singles)
                if calculate_variances:
                    sum_square_weights = sum_square_weights.select(singles_del)

            # Record the multiplicities in the reflection table.
            rtable["multiplicity"] = multiplicity
            # Record the weighted mean intensities in the reflection table.
            rtable["intensity.mean.value"] = i_means
            # Record the standard errors on the means in the reflection table.
            rtable["intensity.mean.std_error"] = flex.sqrt(1 / sum_weights)

            if calculate_variances:
                # Initialise values:
                prev_index = None
                for j in range(rtable.size()):
                    index = rtable["miller_index.asu"][j]
                    weight = weights[j]
                    residual = rtable["intensity.sum.value"][j] - i_means[j]
                    # Aggregate within a symmetry-equivalent group of reflections:
                    if index == prev_index:
                        count += 1
                        weighted_sum_square_residual += weight * residual * residual
                    # Record the aggregated value for the group:
                    elif prev_index:
                        # The weighted variance is undefined for multiplicity=1,
                        # use the measured variance instead in this case.
                        if count == 1:
                            variances.append(rtable["intensity.sum.variance"][j - 1])
                        else:
                            sum_weight = sum_weights[j - 1]
                            var_weight = 1 / (
                                sum_weight - sum_square_weights[j - 1] / sum_weight
                            )
                            variances.extend(
                                flex.double(
                                    count, weighted_sum_square_residual * var_weight
                                )
                            )
                        # Reinitialise:
                        prev_index = index
                        count = 1
                        weighted_sum_square_residual = weight * residual * residual
                    # Handle the first row:
                    else:
                        prev_index = rtable["miller_index.asu"][j]
                        count = 1
                        weighted_sum_square_residual = weight * residual * residual
                # Record the aggregated values for the last group:
                # The weighted variance is undefined for multiplicity=1,
                # use the measured variance instead in this case.
                if count == 1:
                    variances.append(rtable["intensity.sum.variance"][-1])
                else:
                    sum_weight = sum_weights[-1]
                    var_weight = 1 / (sum_weight - sum_square_weights[-1] / sum_weight)
                    variances.extend(
                        flex.double(count, weighted_sum_square_residual * var_weight)
                    )
                # Record the variances in the reflection table.
                rtable["intensity.mean.variance"] = variances

            self.rtables[key] = rtable

    def _make_z(self, uncertainty="sigma"):
        """"
        Generate reflection z-scores.

        Calculate z-scores from reflection intensities, weighted mean
        intensities and a chosen measure of uncertainty in the intensity
        measurement.

        :param uncertainty: Chosen measure of uncertainty.  Options are
          * ``stddev`` — standard deviation, as calculated from the unbiased
          weighted variance aggregated amongst all symmetry-equivalent reflections;
          * ``sigma`` — measurement error for individual reflections.
        :type uncertainty: str
        """

        uncertainty_name = {
            "sigma": "intensity.sum.variance",
            "stddev": "intensity.mean.variance",
        }[uncertainty]

        for key, rtable in self.rtables.items():
            try:
                uncertainty_value = flex.sqrt(rtable[uncertainty_name])
            except KeyError:
                uncertainty_value = flex.sqrt(rtable["intensity.sum.variance"])
                log.warn(
                    u"""Weighted variances haven't been calculated,
      be sure to specify calculate_variances=True to use them.
      Defaulting to measured σ values as a measure of uncertainty instead."""
                )

            z = (
                rtable["intensity.sum.value"] - rtable["intensity.mean.value"]
            ) / uncertainty_value
            rtable["intensity.z_score"] = z

            self.rtables[key] = rtable

    def _probplot_data(self):
        """Generate the data for a normal probability plot of z-scores."""

        import scipy.stats

        for key, rtable in self.rtables.items():
            order = flex.sort_permutation(rtable["intensity.z_score"])
            osm = flex.double(rtable.size(), 0)
            probplot = scipy.stats.probplot(rtable["intensity.z_score"], fit=False)
            osm.set_selected(order, flex.double(probplot[0]))
            rtable["intensity.order_statistic_medians"] = osm

            self.rtables[key] = rtable

def data_from_unmerged_mtz(filename):
    """
    Produce a minimal reflection table from an MTZ file.

    The returned reflection table will not contain all the standard
    columns, only those that are necessary for the IntensityDist class.

    :param filename: Name of an unmerged MTZ input file.
    :type filename: str
    :return: A reflection table object, containing only the columns
      * ``miller_index``
      * ``intensity.sum.value``
      * ``intensity.sum.variance``
      * ``xyzobs.px.value``
      * ``id``
    :rtype: dials.array_family_flex_ext.reflection_table
    """

    from iotbx import mtz
    from dxtbx.model import Crystal, Experiment, ExperimentList

    m = mtz.object(filename).crystals()  # Parse MTZ, with lots of useful methods.
    # Get some data and turn it into a reflection table and experiment list.
    # First, the reflection table
    col_dict = {}
    for crystal in m:
        for dataset in crystal.datasets():
            cols = dataset.columns()  # Gets column objects.
            col_dict = {c.label(): c for c in cols}  # A dict of all the columns.
            if col_dict:
                break
        if col_dict:
            break
    if not col_dict:
        raise RuntimeError("Unable to read data from mtz file %s" % filename)
    h, k, l = (
        col_dict[label].extract_values().as_double().iround()
        for label in ("H", "K", "L")
    )
    intensity, sigI, x, y = (
        col_dict[label].extract_values().as_double()
        for label in ("I", "SIGI", "XDET", "YDET")
    )
    # Honestly flex?!  Oh well, for now, we have to go round the houses:
    frame = col_dict["BATCH"].extract_values().as_double().iround().as_double()

    rtable = flex.reflection_table()
    rtable["miller_index"] = flex.miller_index(h, k, l)
    rtable["intensity.sum.value"] = intensity
    rtable["intensity.sum.variance"] = flex.pow2(sigI)
    rtable["xyzobs.px.value"] = flex.vec3_double(x, y, frame)
    rtable["id"] = flex.int(rtable.size(), 0)

    # Now generate a corresponding experiment list.
    indices = flex.vec3_double([(1, 0, 0), (0, 1, 0), (0, 0, 1)])
    # Each Crystal object needs to be constructed from xyz unit cell
    # parameters and a space group.
    abc = [m[0].unit_cell().orthogonalize(vec) for vec in indices]
    space_group = m[0].crystal_symmetry().space_group()
    crystal_params = abc + [space_group]

    elist = ExperimentList([Experiment(crystal=Crystal(*crystal_params))])

    return rtable, elist

def data_from_pickle_and_json():
    from dials.util.options import (
        OptionParser,
        flatten_reflections,
        flatten_experiments,
    )

    help_message = """

  Generates a dials.array_family.flex.reflection_table and a
  dxtbx.model.experiment_list.ExperimentList from a observations.refl and a
  models.expt.

  Examples:

    dials.util.intensity_explorer indexed.refl indexed.expt

    dials.util.intensity_explorer integrated.refl integrated.expt
    """

    # Create the parser
    usage = (
        "usage: dials.util.intensity_explorer [options] "
        "observations.refl models.expt"
    )
    parser = OptionParser(
        usage=usage,
        read_reflections=True,
        read_experiments=True,
        check_format=True,
        epilog=help_message,
    )

    # Parse the command line arguments
    params, options = parser.parse_args(show_diff_phil=True)

    # Show the help
    if len(params.input.reflections) != 1 or len(params.input.experiments) != 1:
        parser.print_help()
        exit(0)

    rtable = flatten_reflections(params.input.reflections)
    elist = flatten_experiments(params.input.experiments)

    rtable.del_selected(rtable["intensity.sum.variance"] <= 0)

    return rtable, elist
<commit_msg>Clean up util/intensity_explorer.py<commit_after># coding: utf-8

from __future__ import absolute_import, division, print_function

"""
Examine the distribution of diffraction spot intensities.

This module defines a class IntensityDist, with several methods for exploring
the distribution of measured spot intensities in an X-ray diffraction
experiment.  The user may wish to use this information to inform decisions
regarding the error model employed in analysing the data.  Data are passed in
as an unmerged MTZ file (see http://www.ccp4.ac.uk/html/mtzformat.html) and the
resulting IntensityDist instance contains the pertinent columns of data, along
with normal order statistic medians of the z-scores of the intensities, for
constructing a normal probability plot (See
https://www.itl.nist.gov/div898/handbook/eda/section3/normprpl.htm).
"""

import logging

import scipy.stats
from cctbx import miller
from dials.array_family import flex
from dxtbx.model import ExperimentList

log = logging.getLogger("dials.util.intensity_explorer")

class IntensityDist(object):
    def __init__(
        self,
        rtable,
        elist,
        calculate_variances=False,
        keep_singles=False,
        uncertainty="sigma",
        outfile=None,
    ):
        """
        Generate z-scores and a normal probability plot from a DIALS
        reflection_table and a dxtbx ExperimentList, containing the observations
        and the corresponding experiments, respectively.

        :param rtable: A reflection table object, containing at least the columns
          * ``miller_index``
          * ``intensity.sum.value``
          * ``intensity.sum.variance``
          * ``xyzobs.px.value``
        :type rtable: dials.array_family_flex_ext.reflection_table
        :param elist: A corresponding experiment list.
        :type elist: dxtbx_model_ext.ExperimentList
        :param calculate_variances: Choose whether to calculate weighted
        aggregate variances.  Doing so incurs a performance penalty.
        Defaullts to False.
        :type calculate_variances: bool
        :param keep_singles: Choose whether to keep multiplicity-1 reflections.
        Defaults to False.
        :type keep_singles: bool
        :param uncertainty: Measure of spread to use in normalising the
        z-scores, i.e. z = (I - <I>) / uncertainty.
        Possible values for uncertainty:
        * 'sigma':    Use measured sigma values;
        * 'stddev':   Use sample standard deviations calculated as
                      square-root of unbiased weighted sample variances
                      of symmetry-equivalent reflection intensities;
        Defaults to 'sigma'.
        :type uncertainty: str
        :param outfile: Filename root for output PNG plots.
        Defaults to None.
        :type: outfile: str
        """

        if not isinstance(rtable, flex.reflection_table) or not isinstance(
            elist, ExperimentList
        ):
            raise TypeError(
                "Must be called with a reflection table and an experiment list."
            )

        rtable = rtable.copy()
        # Discard unindexed reflections (only necessary because of
        # https://github.com/dials/dials/issues/615 —
        # TODO remove the line below when issue #615 is fixed).
        rtable.del_selected(rtable["id"] == -1)
        rtable["miller_index.asu"] = rtable["miller_index"]

        # Divide reflections by the space group to which they have been indexed.
        self.rtables = {
            expt.crystal.get_space_group().make_tidy(): flex.reflection_table()
            for expt in elist
        }
        for expt, sel in rtable.iterate_experiments_and_indices(elist):
            sg = expt.crystal.get_space_group().make_tidy()
            self.rtables[sg].extend(rtable.select(sel))
        # Map Miller indices to asymmetric unit.
        for space_group, rtable in self.rtables.items():
            # TODO Handle anomalous flag sensibly.  Currently assumes not anomalous.
            miller.map_to_asu(space_group.type(), False, rtable["miller_index.asu"])

        # Calculate normal probability plot data.
        self._multiplicity_mean_error_stddev(
            calculate_variances=calculate_variances, keep_singles=keep_singles
        )
        self._make_z(uncertainty)
        self._probplot_data()

        self.rtable = flex.reflection_table()
        for rtable in self.rtables.values():
            self.rtable.extend(rtable)

        if not outfile:
            outfile = ""
        self.outfile = outfile

    def _multiplicity_mean_error_stddev(
        self, calculate_variances=False, keep_singles=False
    ):
        """"
        Calculate aggregate properties of grouped symmetry-equivalent reflections.

        Populate the reflection table of observations with the following
        properties:
          * ``multiplicity`` — Multiplicity of observations of a given reflection
          in the asymmetric unit;
          :type: `dials.array_family_flex_ext.int` array
          * ``intensity.mean.value`` — Mean of symmetry-equivalent reflections,
          weighted by measurement error;
          :type: `dials.array_family_flex_ext.double` array
          * ``intensity.mean.std_error`` — Standard error on the weighted mean;
          :type: `dials.array_family_flex_ext.double` array
          * (optional) ``intensity.mean.variance`` — variance of
          symmetry-equivalent reflections, weighted by measurement error;
          :type: `dials.array_family_flex_ext.double` array

        :param calculate_variances: Elect whether to calculate the weighted
        variances.  Defaults to False, to spare an expensive computation.
        :type calculate_variances: bool
        :param keep_singles: Choose whether to keep single-multiplicity
        reflections.
        :type keep_singles: bool
        """

        for key, rtable in self.rtables.items():
            # Sort the reflection table for speedier iteration.
            rtable.sort("miller_index.asu")
            # Record the positions of any multiplicity-1 reflections.
            if not keep_singles:
                singles = flex.size_t()
            # Record the multiplicities.
            multiplicity = flex.int()
            # For weighted averaging.
            weights = 1 / rtable["intensity.sum.variance"]
            sum_weights = flex.double()
            if calculate_variances:
                sum_square_weights = flex.double()
            # Calculate the weighted mean intensities.
            i_means = flex.double()
            # Calculate the standard deviations from unbiased weighted variances.
            variances = flex.double()

            # Iterate over the reflections, grouping by equivalent Miller index,
            # to calculate multiplicities, weighted mean intensities, etc..
            # Some time can be saved by only calculating variances if necessary.
            # Initial values:
            prev_index = None
            count = 1
            # The following will be set during loop iteration
            i_sum, sum_weight, sum_square_weight = None, None, None
            # One big loop through the entire reflection table:
            for j in range(rtable.size()):
                index = rtable["miller_index.asu"][j]
                weight = weights[j]
                # Aggregate within a symmetry-equivalent group of reflections:
                if index == prev_index:
                    count += 1
                    i_sum += weight * rtable["intensity.sum.value"][j]
                    sum_weight += weight
                    if calculate_variances:
                        sum_square_weight += weight * weight
                # Record the aggregated values for the group:
                elif prev_index:
                    if count == 1 and not keep_singles:
                        singles.append(j - 1)
                    multiplicity.extend(flex.int(count, count))
                    i_means.extend(flex.double(count, i_sum / sum_weight))
                    sum_weights.extend(flex.double(count, sum_weight))
                    if calculate_variances:
                        sum_square_weights.extend(flex.double(count, sum_square_weight))
                    # And reinitialise:
                    prev_index = index
                    count = 1
                    i_sum = weight * rtable["intensity.sum.value"][j]
                    sum_weight = weight
                    if calculate_variances:
                        sum_square_weight = weight * weight
                # Handle the first row:
                else:
                    prev_index = rtable["miller_index.asu"][j]
                    i_sum = weight * rtable["intensity.sum.value"][j]
                    sum_weight = weight
                    if calculate_variances:
                        sum_square_weight = weight * weight
            # Record the aggregated values for the last group:
            if count == 1 and not keep_singles:
                singles.append(rtable.size() - 1)
            multiplicity.extend(flex.int(count, count))
            i_means.extend(flex.double(count, i_sum / sum_weight))
            sum_weights.extend(flex.double(count, sum_weight))
            if calculate_variances:
                sum_square_weights.extend(flex.double(count, sum_square_weight))

            # Discard singletons:
            if not keep_singles:
                singles_del = flex.bool(rtable.size(), True)
                singles_del.set_selected(singles, False)
                multiplicity, weights, sum_weights, i_means = [
                    a.select(singles_del)
                    for a in (multiplicity, weights, sum_weights, i_means)
                ]
                rtable.del_selected(singles)
                if calculate_variances:
                    sum_square_weights = sum_square_weights.select(singles_del)

            # Record the multiplicities in the reflection table.
            rtable["multiplicity"] = multiplicity
            # Record the weighted mean intensities in the reflection table.
            rtable["intensity.mean.value"] = i_means
            # Record the standard errors on the means in the reflection table.
            rtable["intensity.mean.std_error"] = flex.sqrt(1 / sum_weights)

            if calculate_variances:
                # Initialise values:
                prev_index = None
                weighted_sum_square_residual = None
                for j in range(rtable.size()):
                    index = rtable["miller_index.asu"][j]
                    weight = weights[j]
                    residual = rtable["intensity.sum.value"][j] - i_means[j]
                    # Aggregate within a symmetry-equivalent group of reflections:
                    if index == prev_index:
                        count += 1
                        weighted_sum_square_residual += weight * residual * residual
                    # Record the aggregated value for the group:
                    elif prev_index:
                        # The weighted variance is undefined for multiplicity=1,
                        # use the measured variance instead in this case.
                        if count == 1:
                            variances.append(rtable["intensity.sum.variance"][j - 1])
                        else:
                            sum_weight = sum_weights[j - 1]
                            var_weight = 1 / (
                                sum_weight - sum_square_weights[j - 1] / sum_weight
                            )
                            variances.extend(
                                flex.double(
                                    count, weighted_sum_square_residual * var_weight
                                )
                            )
                        # Reinitialise:
                        prev_index = index
                        count = 1
                        weighted_sum_square_residual = weight * residual * residual
                    # Handle the first row:
                    else:
                        prev_index = rtable["miller_index.asu"][j]
                        count = 1
                        weighted_sum_square_residual = weight * residual * residual
                # Record the aggregated values for the last group:
                # The weighted variance is undefined for multiplicity=1,
                # use the measured variance instead in this case.
                if count == 1:
                    variances.append(rtable["intensity.sum.variance"][-1])
                else:
                    sum_weight = sum_weights[-1]
                    var_weight = 1 / (sum_weight - sum_square_weights[-1] / sum_weight)
                    variances.extend(
                        flex.double(count, weighted_sum_square_residual * var_weight)
                    )
                # Record the variances in the reflection table.
                rtable["intensity.mean.variance"] = variances

            self.rtables[key] = rtable

    def _make_z(self, uncertainty="sigma"):
        """"
        Generate reflection z-scores.

        Calculate z-scores from reflection intensities, weighted mean
        intensities and a chosen measure of uncertainty in the intensity
        measurement.

        :param uncertainty: Chosen measure of uncertainty.  Options are
          * ``stddev`` — standard deviation, as calculated from the unbiased
          weighted variance aggregated amongst all symmetry-equivalent reflections;
          * ``sigma`` — measurement error for individual reflections.
        :type uncertainty: str
        """

        uncertainty_name = {
            "sigma": "intensity.sum.variance",
            "stddev": "intensity.mean.variance",
        }[uncertainty]

        for key, rtable in self.rtables.items():
            try:
                uncertainty_value = flex.sqrt(rtable[uncertainty_name])
            except KeyError:
                uncertainty_value = flex.sqrt(rtable["intensity.sum.variance"])
                log.warn(
                    u"""Weighted variances haven't been calculated,
      be sure to specify calculate_variances=True to use them.
      Defaulting to measured σ values as a measure of uncertainty instead."""
                )

            z = (
                rtable["intensity.sum.value"] - rtable["intensity.mean.value"]
            ) / uncertainty_value
            rtable["intensity.z_score"] = z

            self.rtables[key] = rtable

    def _probplot_data(self):
        """Generate the data for a normal probability plot of z-scores."""

        for key, rtable in self.rtables.items():
            order = flex.sort_permutation(rtable["intensity.z_score"])
            osm = flex.double(rtable.size(), 0)
            probplot = scipy.stats.probplot(rtable["intensity.z_score"], fit=False)
            osm.set_selected(order, flex.double(probplot[0]))
            rtable["intensity.order_statistic_medians"] = osm

            self.rtables[key] = rtable
<|endoftext|>


<commit_before># Copyright lowRISC contributors.
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
# SPDX-License-Identifier: Apache-2.0
"""
Generate C header from validated register JSON tree
"""

import io
import logging as log
import sys
import textwrap
import warnings

def genout(outfile, msg):
    outfile.write(msg)

def as_define(s):
    s = s.upper()
    r = ''
    for i in range(0, len(s)):
        r += s[i] if s[i].isalnum() else '_'
    return r

def first_line(s):
    """Returns the first line of a multi-line string"""
    return s.splitlines()[0]

def format_comment(s):
    """Formats a string to comment wrapped to an 80 character line width

    Returns wrapped string including newline and // comment characters.
    """
    return '\n'.join(
        textwrap.wrap(
            s, width=77, initial_indent='// ', subsequent_indent='// ')) + '\n'

def gen_define(name, args, body, existing_defines, indent='  '):
    r"""Produces a #define string, will split into two lines if a single line
    has a width greater than 80 characters. Result includes newline.

    Arguments:
    name - Name of the #define
    args - List of arguments for the define, provide an empty list if there are
        none
    body - Body of the #define
    existing_defines - set of already generated define names.
        Error if `name` is in `existing_defines`.
    indent - Gives string to prepend on any new lines produced by
    wrapping (default '  ')

    Example result:
    name = 'A_MACRO'
    args = ['arg1', 'arg2'],
    body = 'arg1 + arg2 + 10'

    #define A_MACRO(arg1, arg2) arg1 + arg2 + 10

    When the macro is wrapped the break happens after the argument list (or
    macro name if there is no argument list

    #define A_MACRO(arg1, arg2) \
      arg1 + arg2 + 10

    """

    if name in existing_defines:
        log.error("Duplicate #define for " + name)
        sys.exit(1)

    if len(args) != 0:
        define_declare = '#define ' + name + '(' + ', '.join(args) + ')'
    else:
        define_declare = '#define ' + name

    oneline_define = define_declare + ' ' + body

    existing_defines.add(name)

    if len(oneline_define) <= 80:
        return oneline_define + '\n'

    return define_declare + ' \\\n' + indent + body + '\n'

def gen_cdefine_register(outstr, reg, comp, width, rnames, existing_defines):
    rname = reg['name']
    offset = reg['genoffset']

    genout(outstr, format_comment(first_line(reg['desc'])))
    defname = as_define(comp + '_' + rname)
    genout(
        outstr,
        gen_define(defname + '_REG_OFFSET', [], hex(offset), existing_defines))

    for field in reg['fields']:
        fieldlsb = field['bitinfo'][2]
        fname = field['name']
        dname = defname + '_' + as_define(fname)

        if field['bitinfo'][1] == 1:
            # single bit
            genout(outstr,
                   gen_define(dname + '_BIT', [], str(fieldlsb), existing_defines))
        else:
            # multiple bits (unless it is the whole register)
            if field['bitinfo'][1] != width:
                mask = field['bitinfo'][0] >> fieldlsb
                genout(
                    outstr,
                    gen_define(dname + '_MASK', [], hex(mask),
                               existing_defines))
                genout(
                    outstr,
                    gen_define(dname + '_OFFSET', [], str(fieldlsb),
                               existing_defines))
            if 'enum' in field:
                for enum in field['enum']:
                    ename = as_define(enum['name'])
                    value = hex(int(enum['value'], 0))
                    genout(
                        outstr,
                        gen_define(
                            defname + '_' + as_define(field['name']) +
                            '_VALUE_' + ename, [], value, existing_defines))
    genout(outstr, '\n')
    return

def gen_cdefine_window(outstr, win, comp, regwidth, rnames, existing_defines):
    wname = win['name']
    offset = win['genoffset']

    genout(outstr, format_comment('Memory area: ' + first_line(win['desc'])))
    defname = as_define(comp + '_' + wname)
    genout(
        outstr,
        gen_define(defname + '_REG_OFFSET', [], hex(offset), existing_defines))
    items = int(win['items'])
    genout(
        outstr,
        gen_define(defname + '_SIZE_WORDS', [], str(items), existing_defines))
    items = items * (regwidth // 8)
    genout(
        outstr,
        gen_define(defname + '_SIZE_BYTES', [], str(items), existing_defines))

    wid = win['genvalidbits']
    if (wid != regwidth):
        mask = (1 << wid) - 1
        genout(outstr,
               gen_define(defname + '_MASK ', [], hex(mask), existing_defines))

def gen_cdefines_module_param(outstr, param, module_name, existing_defines):
    param_type = param['type']

    # Do not generate C defines for parameters that are not localparams defined
    # in the corresponding SystemVerilog package.
    if param["local"].lower() == "false":
        return

    # Presently there is only one type (int), however if the new types are
    # added, they potentially need to be handled differently.
    known_types = ["int"]
    if param_type not in known_types:
        warnings.warn(
            "Cannot generate a module define of type {}".format(param_type))
        return

    genout(outstr, format_comment(first_line(param['desc'])))
    define_name = as_define(module_name + '_PARAM_' + param['name'])
    if param_type == "int":
        define = gen_define(define_name, [], param['default'],
                            existing_defines)

    genout(outstr, define)
    genout(outstr, '\n')

def gen_cdefines_module_params(outstr, module_data, module_name,
                               register_width, existing_defines):
    module_params = set()

    if 'param_list' in module_data:
        module_params = module_data['param_list']

    for param in module_params:
        gen_cdefines_module_param(outstr, param, module_name, existing_defines)

    genout(outstr, format_comment(first_line("Register width")))
    define_name = as_define(module_name + '_PARAM_REG_WIDTH')
    define = gen_define(define_name, [], str(register_width), existing_defines)
    genout(outstr, define)
    genout(outstr, '\n')

def gen_multireg_field_defines(outstr, regname, field, subreg_num, regwidth,
                               existing_defines):
    field_width = field['bitinfo'][1]
    fields_per_reg = regwidth // field_width

    define_name = regname + '_' + as_define(field['name'] + "_FIELD_WIDTH")
    define = gen_define(define_name, [], str(field_width), existing_defines)
    genout(outstr, define)

    define_name = regname + '_' + as_define(field['name'] + "_FIELDS_PER_REG")
    define = gen_define(define_name, [], str(fields_per_reg), existing_defines)
    genout(outstr, define)

    define_name = regname + "_MULTIREG_COUNT"
    define = gen_define(define_name, [], str(subreg_num), existing_defines)
    genout(outstr, define)

    genout(outstr, '\n')

def gen_cdefine_multireg(outstr, register, component, regwidth, rnames,
                         existing_defines):
    multireg = register['multireg']
    subregs = multireg['genregs']

    comment = multireg['desc'] + " (common parameters)"
    genout(outstr, format_comment(first_line(comment)))
    if len(multireg['fields']) == 1:
        regname = as_define(component + '_' + multireg['name'])
        gen_multireg_field_defines(outstr, regname, multireg['fields'][0],
                                   len(subregs), regwidth, existing_defines)
    else:
        log.warn("Non-homogeneous multireg " + multireg['name'] +
                 " skip multireg specific data generation.")

    for subreg in subregs:
        gen_cdefine_register(outstr, subreg, component, regwidth, rnames,
                             existing_defines)

def gen_cdefines_interrupt_field(outstr, interrupt, component, regwidth,
                                 existing_defines):
    fieldlsb = interrupt['bitinfo'][2]
    iname = interrupt['name']
    defname = as_define(component + '_INTR_COMMON_' + iname)

    if interrupt['bitinfo'][1] == 1:
        # single bit
        genout(outstr, gen_define(defname + '_BIT', [], str(fieldlsb),
                                  existing_defines))
    else:
        # multiple bits (unless it is the whole register)
        if interrupt['bitinfo'][1] != regwidth:
            mask = interrupt['bitinfo'][0] >> fieldlsb
            genout(
                outstr,
                gen_define(defname + '_MASK', [], hex(mask), existing_defines))
            genout(
                outstr,
                gen_define(defname + '_OFFSET', [], str(fieldlsb),
                           existing_defines))

def gen_cdefines_interrupts(outstr, regs, component, regwidth,
                            existing_defines):
    # no_auto_intr_regs controls whether interrupt registers are automatically
    # generated from the interrupt_list. This key could be 'true' or 'false',
    # but might also be True or False (the python booleans).
    no_auto_i = False
    if 'no_auto_intr_regs' in regs:
        no_auto_intr_regs_val = regs['no_auto_intr_regs']
        if isinstance(no_auto_intr_regs_val, bool):
            no_auto_i = no_auto_intr_regs_val
        elif no_auto_intr_regs_val.lower() in ["true", "false"]:
            no_auto_i = no_auto_intr_regs_val == "true"
        else:
            pass

    # If no_auto_intr_regs is true, then we do not generate common defines,
    # because the bit offsets for a particular interrupt may differ between
    # the interrupt enable/state/test registers.
    if no_auto_i:
        return

    interrupts = regs.get('interrupt_list', [])
    genout(outstr, format_comment(first_line("Common Interrupt Offsets")))
    for intr in interrupts:
        gen_cdefines_interrupt_field(outstr, intr, component, regwidth,
                                     existing_defines)
    genout(outstr, '\n')

# Must have called validate, so should have no errors
def gen_cdefines(regs, outfile, src_lic, src_copy):
    component = regs['name']
    registers = regs['registers']
    rnames = regs['genrnames']
    outstr = io.StringIO()

    # This tracks the defines that have been generated so far, so we
    # can error if we attempt to duplicate a definition
    existing_defines = set()

    if 'regwidth' in regs:
        regwidth = int(regs['regwidth'], 0)
    else:
        regwidth = 32

    gen_cdefines_module_params(outstr, regs, component, regwidth,
                               existing_defines)

    gen_cdefines_interrupts(outstr, regs, component, regwidth,
                            existing_defines)

    for x in registers:
        if 'reserved' in x:
            continue

        if 'skipto' in x:
            continue

        if 'sameaddr' in x:
            for sareg in x['sameaddr']:
                gen_cdefine_register(outstr, sareg, component, regwidth,
                                     rnames, existing_defines)
            continue

        if 'window' in x:
            gen_cdefine_window(outstr, x['window'], component, regwidth,
                               rnames, existing_defines)
            continue

        if 'multireg' in x:
            gen_cdefine_multireg(outstr, x, component, regwidth, rnames,
                                 existing_defines)
            continue

        gen_cdefine_register(outstr, x, component, regwidth, rnames,
                             existing_defines)

    generated = outstr.getvalue()
    outstr.close()

    genout(outfile, '// Generated register defines for ' + component + '\n\n')
    if src_copy != '':
        genout(outfile, '// Copyright information found in source file:\n')
        genout(outfile, '// ' + src_copy + '\n\n')
    if src_lic is not None:
        genout(outfile, '// Licensing information found in source file:\n')
        for line in src_lic.splitlines():
            genout(outfile, '// ' + line + '\n')
        genout(outfile, '\n')

    # Header Include Guard
    genout(outfile, '#ifndef _' + as_define(component) + '_REG_DEFS_\n')
    genout(outfile, '#define _' + as_define(component) + '_REG_DEFS_\n\n')

    # Header Extern Guard (so header can be used from C and C++)
    genout(outfile, '#ifdef __cplusplus\n')
    genout(outfile, 'extern "C" {\n')
    genout(outfile, '#endif\n')

    genout(outfile, generated)

    # Header Extern Guard
    genout(outfile, '#ifdef __cplusplus\n')
    genout(outfile, '}  // extern "C"\n')
    genout(outfile, '#endif\n')

    # Header Include Guard
    genout(outfile, '#endif  // _' + as_define(component) + '_REG_DEFS_\n')

    genout(outfile, '// End generated register defines for ' + component)

    return

def test_gen_define():
    basic_oneline = '#define MACRO_NAME body\n'
    assert gen_define('MACRO_NAME', [], 'body', set()) == basic_oneline

    basic_oneline_with_args = '#define MACRO_NAME(arg1, arg2) arg1 + arg2\n'
    assert (gen_define('MACRO_NAME', ['arg1', 'arg2'], 'arg1 + arg2',
                       set()) == basic_oneline_with_args)

    long_macro_name = 'A_VERY_VERY_VERY_VERY_VERY_VERY_VERY_VERY_VERY_VERY_VERY_LONG_MACRO_NAME'

    multiline = ('#define ' + long_macro_name + ' \\\n' +
                 '  a_fairly_long_body + something_else + 10\n')

    assert (gen_define(long_macro_name, [],
                       'a_fairly_long_body + something_else + 10',
                       set()) == multiline)

    multiline_with_args = ('#define ' + long_macro_name +
                           '(arg1, arg2, arg3) \\\n' +
                           '  a_fairly_long_body + arg1 + arg2 + arg3\n')

    assert (gen_define(long_macro_name, ['arg1', 'arg2', 'arg3'],
                       'a_fairly_long_body + arg1 + arg2 + arg3',
                       set()) == multiline_with_args)

    multiline_with_args_big_indent = (
        '#define ' + long_macro_name + '(arg1, arg2, arg3) \\\n' +
        '    a_fairly_long_body + arg1 + arg2 + arg3\n')

    assert (gen_define(long_macro_name, ['arg1', 'arg2', 'arg3'],
                       'a_fairly_long_body + arg1 + arg2 + arg3',
                       set(),
                       indent='    ') == multiline_with_args_big_indent)
<commit_msg>[reggen] Generate bitfield.h constants for use in C code<commit_after># Copyright lowRISC contributors.
# Licensed under the Apache License, Version 2.0, see LICENSE for details.
# SPDX-License-Identifier: Apache-2.0
"""
Generate C header from validated register JSON tree
"""

import io
import logging as log
import sys
import textwrap
import warnings

def genout(outfile, msg):
    outfile.write(msg)

def as_define(s):
    s = s.upper()
    r = ''
    for i in range(0, len(s)):
        r += s[i] if s[i].isalnum() else '_'
    return r

def first_line(s):
    """Returns the first line of a multi-line string"""
    return s.splitlines()[0]

def format_comment(s):
    """Formats a string to comment wrapped to an 80 character line width

    Returns wrapped string including newline and // comment characters.
    """
    return '\n'.join(
        textwrap.wrap(
            s, width=77, initial_indent='// ', subsequent_indent='// ')) + '\n'

def gen_define(name, args, body, existing_defines, indent='  '):
    r"""Produces a #define string, will split into two lines if a single line
    has a width greater than 80 characters. Result includes newline.

    Arguments:
    name - Name of the #define
    args - List of arguments for the define, provide an empty list if there are
        none
    body - Body of the #define
    existing_defines - set of already generated define names.
        Error if `name` is in `existing_defines`.
    indent - Gives string to prepend on any new lines produced by
    wrapping (default '  ')

    Example result:
    name = 'A_MACRO'
    args = ['arg1', 'arg2'],
    body = 'arg1 + arg2 + 10'

    #define A_MACRO(arg1, arg2) arg1 + arg2 + 10

    When the macro is wrapped the break happens after the argument list (or
    macro name if there is no argument list

    #define A_MACRO(arg1, arg2) \
      arg1 + arg2 + 10

    """

    if name in existing_defines:
        log.error("Duplicate #define for " + name)
        sys.exit(1)

    if len(args) != 0:
        define_declare = '#define ' + name + '(' + ', '.join(args) + ')'
    else:
        define_declare = '#define ' + name

    oneline_define = define_declare + ' ' + body

    existing_defines.add(name)

    if len(oneline_define) <= 80:
        return oneline_define + '\n'

    return define_declare + ' \\\n' + indent + body + '\n'

def gen_cdefine_register(outstr, reg, comp, width, rnames, existing_defines):
    rname = reg['name']
    offset = reg['genoffset']

    genout(outstr, format_comment(first_line(reg['desc'])))
    defname = as_define(comp + '_' + rname)
    genout(
        outstr,
        gen_define(defname + '_REG_OFFSET', [], hex(offset), existing_defines))

    for field in reg['fields']:
        fieldlsb = field['bitinfo'][2]
        fname = field['name']
        dname = defname + '_' + as_define(fname)

        if field['bitinfo'][1] == 1:
            # single bit
            genout(outstr,
                   gen_define(dname + '_BIT', [], str(fieldlsb), existing_defines))
        else:
            # multiple bits (unless it is the whole register)
            if field['bitinfo'][1] != width:
                mask = field['bitinfo'][0] >> fieldlsb
                genout(
                    outstr,
                    gen_define(dname + '_MASK', [], hex(mask),
                               existing_defines))
                genout(
                    outstr,
                    gen_define(dname + '_OFFSET', [], str(fieldlsb),
                               existing_defines))
                genout(
                    outstr,
                    gen_define(
                        dname + '_FIELD', [],
                        '((bitfield_field32_t) {{ .mask = {dname}_MASK, .index = {dname}_OFFSET }})'
                        .format(dname=dname), existing_defines))
            if 'enum' in field:
                for enum in field['enum']:
                    ename = as_define(enum['name'])
                    value = hex(int(enum['value'], 0))
                    genout(
                        outstr,
                        gen_define(
                            defname + '_' + as_define(field['name']) +
                            '_VALUE_' + ename, [], value, existing_defines))
    genout(outstr, '\n')
    return

def gen_cdefine_window(outstr, win, comp, regwidth, rnames, existing_defines):
    wname = win['name']
    offset = win['genoffset']

    genout(outstr, format_comment('Memory area: ' + first_line(win['desc'])))
    defname = as_define(comp + '_' + wname)
    genout(
        outstr,
        gen_define(defname + '_REG_OFFSET', [], hex(offset), existing_defines))
    items = int(win['items'])
    genout(
        outstr,
        gen_define(defname + '_SIZE_WORDS', [], str(items), existing_defines))
    items = items * (regwidth // 8)
    genout(
        outstr,
        gen_define(defname + '_SIZE_BYTES', [], str(items), existing_defines))

    wid = win['genvalidbits']
    if (wid != regwidth):
        mask = (1 << wid) - 1
        genout(outstr,
               gen_define(defname + '_MASK ', [], hex(mask), existing_defines))

def gen_cdefines_module_param(outstr, param, module_name, existing_defines):
    param_type = param['type']

    # Do not generate C defines for parameters that are not localparams defined
    # in the corresponding SystemVerilog package.
    if param["local"].lower() == "false":
        return

    # Presently there is only one type (int), however if the new types are
    # added, they potentially need to be handled differently.
    known_types = ["int"]
    if param_type not in known_types:
        warnings.warn(
            "Cannot generate a module define of type {}".format(param_type))
        return

    genout(outstr, format_comment(first_line(param['desc'])))
    define_name = as_define(module_name + '_PARAM_' + param['name'])
    if param_type == "int":
        define = gen_define(define_name, [], param['default'],
                            existing_defines)

    genout(outstr, define)
    genout(outstr, '\n')

def gen_cdefines_module_params(outstr, module_data, module_name,
                               register_width, existing_defines):
    module_params = set()

    if 'param_list' in module_data:
        module_params = module_data['param_list']

    for param in module_params:
        gen_cdefines_module_param(outstr, param, module_name, existing_defines)

    genout(outstr, format_comment(first_line("Register width")))
    define_name = as_define(module_name + '_PARAM_REG_WIDTH')
    define = gen_define(define_name, [], str(register_width), existing_defines)
    genout(outstr, define)
    genout(outstr, '\n')

def gen_multireg_field_defines(outstr, regname, field, subreg_num, regwidth,
                               existing_defines):
    field_width = field['bitinfo'][1]
    fields_per_reg = regwidth // field_width

    define_name = regname + '_' + as_define(field['name'] + "_FIELD_WIDTH")
    define = gen_define(define_name, [], str(field_width), existing_defines)
    genout(outstr, define)

    define_name = regname + '_' + as_define(field['name'] + "_FIELDS_PER_REG")
    define = gen_define(define_name, [], str(fields_per_reg), existing_defines)
    genout(outstr, define)

    define_name = regname + "_MULTIREG_COUNT"
    define = gen_define(define_name, [], str(subreg_num), existing_defines)
    genout(outstr, define)

    genout(outstr, '\n')

def gen_cdefine_multireg(outstr, register, component, regwidth, rnames,
                         existing_defines):
    multireg = register['multireg']
    subregs = multireg['genregs']

    comment = multireg['desc'] + " (common parameters)"
    genout(outstr, format_comment(first_line(comment)))
    if len(multireg['fields']) == 1:
        regname = as_define(component + '_' + multireg['name'])
        gen_multireg_field_defines(outstr, regname, multireg['fields'][0],
                                   len(subregs), regwidth, existing_defines)
    else:
        log.warn("Non-homogeneous multireg " + multireg['name'] +
                 " skip multireg specific data generation.")

    for subreg in subregs:
        gen_cdefine_register(outstr, subreg, component, regwidth, rnames,
                             existing_defines)

def gen_cdefines_interrupt_field(outstr, interrupt, component, regwidth,
                                 existing_defines):
    fieldlsb = interrupt['bitinfo'][2]
    iname = interrupt['name']
    defname = as_define(component + '_INTR_COMMON_' + iname)

    if interrupt['bitinfo'][1] == 1:
        # single bit
        genout(outstr, gen_define(defname + '_BIT', [], str(fieldlsb),
                                  existing_defines))
    else:
        # multiple bits (unless it is the whole register)
        if interrupt['bitinfo'][1] != regwidth:
            mask = interrupt['bitinfo'][0] >> fieldlsb
            genout(
                outstr,
                gen_define(defname + '_MASK', [], hex(mask), existing_defines))
            genout(
                outstr,
                gen_define(defname + '_OFFSET', [], str(fieldlsb),
                           existing_defines))
            genout(
                outstr,
                gen_define(
                    defname + '_FIELD', [],
                    '((bitfield_field32_t) {{ .mask = {dname}_MASK, .index = {dname}_OFFSET }})'
                    .format(dname=defname), existing_defines))

def gen_cdefines_interrupts(outstr, regs, component, regwidth,
                            existing_defines):
    # no_auto_intr_regs controls whether interrupt registers are automatically
    # generated from the interrupt_list. This key could be 'true' or 'false',
    # but might also be True or False (the python booleans).
    no_auto_i = False
    if 'no_auto_intr_regs' in regs:
        no_auto_intr_regs_val = regs['no_auto_intr_regs']
        if isinstance(no_auto_intr_regs_val, bool):
            no_auto_i = no_auto_intr_regs_val
        elif no_auto_intr_regs_val.lower() in ["true", "false"]:
            no_auto_i = no_auto_intr_regs_val == "true"
        else:
            pass

    # If no_auto_intr_regs is true, then we do not generate common defines,
    # because the bit offsets for a particular interrupt may differ between
    # the interrupt enable/state/test registers.
    if no_auto_i:
        return

    interrupts = regs.get('interrupt_list', [])
    genout(outstr, format_comment(first_line("Common Interrupt Offsets")))
    for intr in interrupts:
        gen_cdefines_interrupt_field(outstr, intr, component, regwidth,
                                     existing_defines)
    genout(outstr, '\n')

# Must have called validate, so should have no errors
def gen_cdefines(regs, outfile, src_lic, src_copy):
    component = regs['name']
    registers = regs['registers']
    rnames = regs['genrnames']
    outstr = io.StringIO()

    # This tracks the defines that have been generated so far, so we
    # can error if we attempt to duplicate a definition
    existing_defines = set()

    if 'regwidth' in regs:
        regwidth = int(regs['regwidth'], 0)
    else:
        regwidth = 32

    gen_cdefines_module_params(outstr, regs, component, regwidth,
                               existing_defines)

    gen_cdefines_interrupts(outstr, regs, component, regwidth,
                            existing_defines)

    for x in registers:
        if 'reserved' in x:
            continue

        if 'skipto' in x:
            continue

        if 'sameaddr' in x:
            for sareg in x['sameaddr']:
                gen_cdefine_register(outstr, sareg, component, regwidth,
                                     rnames, existing_defines)
            continue

        if 'window' in x:
            gen_cdefine_window(outstr, x['window'], component, regwidth,
                               rnames, existing_defines)
            continue

        if 'multireg' in x:
            gen_cdefine_multireg(outstr, x, component, regwidth, rnames,
                                 existing_defines)
            continue

        gen_cdefine_register(outstr, x, component, regwidth, rnames,
                             existing_defines)

    generated = outstr.getvalue()
    outstr.close()

    genout(outfile, '// Generated register defines for ' + component + '\n\n')
    if src_copy != '':
        genout(outfile, '// Copyright information found in source file:\n')
        genout(outfile, '// ' + src_copy + '\n\n')
    if src_lic is not None:
        genout(outfile, '// Licensing information found in source file:\n')
        for line in src_lic.splitlines():
            genout(outfile, '// ' + line + '\n')
        genout(outfile, '\n')

    # Header Include Guard
    genout(outfile, '#ifndef _' + as_define(component) + '_REG_DEFS_\n')
    genout(outfile, '#define _' + as_define(component) + '_REG_DEFS_\n\n')

    # Header Extern Guard (so header can be used from C and C++)
    genout(outfile, '#ifdef __cplusplus\n')
    genout(outfile, 'extern "C" {\n')
    genout(outfile, '#endif\n')

    genout(outfile, generated)

    # Header Extern Guard
    genout(outfile, '#ifdef __cplusplus\n')
    genout(outfile, '}  // extern "C"\n')
    genout(outfile, '#endif\n')

    # Header Include Guard
    genout(outfile, '#endif  // _' + as_define(component) + '_REG_DEFS_\n')

    genout(outfile, '// End generated register defines for ' + component)

    return

def test_gen_define():
    basic_oneline = '#define MACRO_NAME body\n'
    assert gen_define('MACRO_NAME', [], 'body', set()) == basic_oneline

    basic_oneline_with_args = '#define MACRO_NAME(arg1, arg2) arg1 + arg2\n'
    assert (gen_define('MACRO_NAME', ['arg1', 'arg2'], 'arg1 + arg2',
                       set()) == basic_oneline_with_args)

    long_macro_name = 'A_VERY_VERY_VERY_VERY_VERY_VERY_VERY_VERY_VERY_VERY_VERY_LONG_MACRO_NAME'

    multiline = ('#define ' + long_macro_name + ' \\\n' +
                 '  a_fairly_long_body + something_else + 10\n')

    assert (gen_define(long_macro_name, [],
                       'a_fairly_long_body + something_else + 10',
                       set()) == multiline)

    multiline_with_args = ('#define ' + long_macro_name +
                           '(arg1, arg2, arg3) \\\n' +
                           '  a_fairly_long_body + arg1 + arg2 + arg3\n')

    assert (gen_define(long_macro_name, ['arg1', 'arg2', 'arg3'],
                       'a_fairly_long_body + arg1 + arg2 + arg3',
                       set()) == multiline_with_args)

    multiline_with_args_big_indent = (
        '#define ' + long_macro_name + '(arg1, arg2, arg3) \\\n' +
        '    a_fairly_long_body + arg1 + arg2 + arg3\n')

    assert (gen_define(long_macro_name, ['arg1', 'arg2', 'arg3'],
                       'a_fairly_long_body + arg1 + arg2 + arg3',
                       set(),
                       indent='    ') == multiline_with_args_big_indent)
<|endoftext|>


<commit_before>import atexit
import importlib

from sqlalchemy.orm.session import sessionmaker
from blueberrypy.config import BlueberryPyConfiguration
from sqlalchemy.engine import engine_from_config

def get_user_namespace(config, include_pkg=False):
    assert isinstance(config, BlueberryPyConfiguration), type(config)

    ns = {'__name__': 'blueberrypy-shell'}

    pkg_name = get_package_name(config)

    if include_pkg:
        app_module = importlib.import_module(pkg_name)
        if hasattr(app_module, '__all__'):
            for name in app_module.__all__:
                ns[name] = getattr(app_module, name)
        else:
            for name, obj in vars(app_module).viewitems():
                if not name.startswith("_"):
                    ns[name] = obj

    ns['cherrypy'] = importlib.import_module("cherrypy")
    ns['blueberrypy'] = importlib.import_module("blueberrypy")

    if config.use_sqlalchemy:
        engine = _make_sa_engine(config)
        if isinstance(engine, dict):
            Session = sessionmaker(twophase=True)
            Session.configure(binds=engine)
        else:
            Session = sessionmaker()
            Session.configure(bind=engine)
        model = importlib.import_module(pkg_name + ".model")
        model.metadata.bind = engine
        ns['session'] = session = Session()
        session.bind.echo = True
        atexit.register(session.close)

    return ns

def get_package_name(config):
    return (
        config.project_metadata["package"]
        if config.project_metadata else None
    )

def _make_sa_engine(config):
    sa_prefix = 'sqlalchemy_engine'
    sa_config = config.sqlalchemy_config

    if sa_prefix in sa_config:
        section = sa_config[sa_prefix]
        return engine_from_config(section, '')
    else:
        engine_bindings = {}
        for section_name, section in sa_config.viewitems():
            if section_name.startswith(sa_prefix):
                model_definition = section_name[len(sa_prefix) + 1:]
                pkg_name, obj_name = model_definition.rsplit('.', 1)
                package = __import__(pkg_name, globals(), locals(), [obj_name])
                model = getattr(package, obj_name)
                engine_bindings[model] = engine_from_config(section)
        return engine_bindings

<commit_msg>Move SQLAlchemy imports to the places of use<commit_after>import atexit
import importlib

from blueberrypy.config import BlueberryPyConfiguration

def get_user_namespace(config, include_pkg=False):
    assert isinstance(config, BlueberryPyConfiguration), type(config)

    ns = {'__name__': 'blueberrypy-shell'}

    pkg_name = get_package_name(config)

    if include_pkg:
        app_module = importlib.import_module(pkg_name)
        if hasattr(app_module, '__all__'):
            for name in app_module.__all__:
                ns[name] = getattr(app_module, name)
        else:
            for name, obj in vars(app_module).viewitems():
                if not name.startswith("_"):
                    ns[name] = obj

    ns['cherrypy'] = importlib.import_module("cherrypy")
    ns['blueberrypy'] = importlib.import_module("blueberrypy")

    if config.use_sqlalchemy:
        from sqlalchemy.orm.session import sessionmaker
        engine = _make_sa_engine(config)
        if isinstance(engine, dict):
            Session = sessionmaker(twophase=True)
            Session.configure(binds=engine)
        else:
            Session = sessionmaker()
            Session.configure(bind=engine)
        model = importlib.import_module(pkg_name + ".model")
        model.metadata.bind = engine
        ns['session'] = session = Session()
        session.bind.echo = True
        atexit.register(session.close)

    return ns

def get_package_name(config):
    return (
        config.project_metadata["package"]
        if config.project_metadata else None
    )

def _make_sa_engine(config):
    from sqlalchemy.engine import engine_from_config
    sa_prefix = 'sqlalchemy_engine'
    sa_config = config.sqlalchemy_config

    if sa_prefix in sa_config:
        section = sa_config[sa_prefix]
        return engine_from_config(section, '')
    else:
        engine_bindings = {}
        for section_name, section in sa_config.viewitems():
            if section_name.startswith(sa_prefix):
                model_definition = section_name[len(sa_prefix) + 1:]
                pkg_name, obj_name = model_definition.rsplit('.', 1)
                package = __import__(pkg_name, globals(), locals(), [obj_name])
                model = getattr(package, obj_name)
                engine_bindings[model] = engine_from_config(section)
        return engine_bindings

<|endoftext|>


<commit_before># coding=utf-8
"""
© 2013 LinkedIn Corp. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
import datetime
import logging
import os
import re
import sys
import threading

from naarad.metrics.metric import Metric
import naarad.utils
from naarad.naarad_constants import important_sub_metrics_import

logger = logging.getLogger('naarad.metrics.GCMetric')

class GCMetric(Metric):
  """ Class for GC logs, deriving from class Metric """
  clock_format = '%Y-%m-%d %H:%M:%S'
  rate_types = ()
  val_types = ('alloc', 'promo', 'used0', 'used1', 'used', 'commit0', 'commit1', 'commit', 'gen0', 'gen0t', 'gen0usr', 'gen0sys', 'gen0real',
      'cmsIM', 'cmsRM', 'cmsRS', 'GCPause', 'cmsCM', 'cmsCP', 'cmsCS', 'cmsCR', 'safept', 'apptime', 'used0AfterGC', 'used1AfterGC', 'usedAfterGC',
      'gen1t', 'g1-pause-young', 'g1-pause-mixed', 'g1-pause-remark', 'g1-pause-cleanup', 'g1-pause-remark.ref-proc', 'g1-pause-young.parallel',
      'g1-pause-young.parallel.gcworkers', 'g1-pause-young.parallel.ext-root-scanning.avg', 'g1-pause-young.parallel.ext-root-scanning.max',
      'g1-pause-young.parallel.update-rs.avg', 'g1-pause-young.parallel.update-rs.max', 'g1-pause-young.parallel.update-rs.processed-buffers.avg',
      'g1-pause-young.parallel.update-rs.processed-buffers.max', 'g1-pause-young.parallel.scan-rs.avg', 'g1-pause-young.parallel.scan-rs.max',
      'g1-pause-young.parallel.object-copy-rs.avg', 'g1-pause-young.parallel.object-copy-rs.max', 'g1-pause-young.parallel.termination.avg',
      'g1-pause-young.parallel.termination.max', 'g1-pause-young.parallel.gc-worker-other.avg', 'g1-pause-young.parallel.gc-worker-other.max',
      'g1-pause-young.parallel.gc-worker-total.avg', 'g1-pause-young.parallel.gc-worker-total.max', 'g1-pause-young.parallel.gc-worker-end.avg',
      'g1-pause-young.parallel.gc-worker-end.max', 'g1-pause-young.code-root-fixup', 'g1-pause-young.clear-ct', 'g1-pause-young.other',
      'g1-pause-young.other.choose-cset', 'g1-pause-young.other.ref-proc', 'g1-pause-young.other.reg-enq', 'g1-pause-young.other.free-cset',
      'g1-pause-mixed.parallel', 'g1-pause-mixed.parallel.gcworkers', 'g1-pause-mixed.parallel.ext-root-scanning.avg',
      'g1-pause-mixed.parallel.ext-root-scanning.max', 'g1-pause-mixed.parallel.update-rs.avg', 'g1-pause-mixed.parallel.update-rs.max',
      'g1-pause-mixed.parallel.update-rs.processed-buffers.avg', 'g1-pause-mixed.parallel.update-rs.processed-buffers.max',
      'g1-pause-mixed.parallel.scan-rs.avg', 'g1-pause-mixed.parallel.scan-rs.max', 'g1-pause-mixed.parallel.object-copy-rs.avg',
      'g1-pause-mixed.parallel.object-copy-rs.max', 'g1-pause-mixed.parallel.termination.avg', 'g1-pause-mixed.parallel.termination.max',
      'g1-pause-mixed.parallel.gc-worker-other.avg', 'g1-pause-mixed.parallel.gc-worker-other.max', 'g1-pause-mixed.parallel.gc-worker-total.avg',
      'g1-pause-mixed.parallel.gc-worker-total.max', 'g1-pause-mixed.parallel.gc-worker-end.avg', 'g1-pause-mixed.parallel.gc-worker-end.max',
      'g1-pause-mixed.code-root-fixup', 'g1-pause-mixed.clear-ct', 'g1-pause-mixed.other', 'g1-pause-mixed.other.choose-cset',
      'g1-pause-mixed.other.ref-proc', 'g1-pause-mixed.other.reg-enq', 'g1-pause-mixed.other.free-cset', 'g1-pause-young.parallel.gc-worker-start.avg',
      'g1-pause-young.parallel.gc-worker-start.max', 'g1-pause-mixed.parallel.gc-worker-start.avg', 'g1-pause-mixed.parallel.gc-worker-start.max',
      'g1-eden-occupancy-before-gc', 'g1-eden-capacity-before-gc', 'g1-eden-occupancy-after-gc', 'g1-eden-capacity-after-gc', 'g1-survivor-before-gc',
      'g1-survivor-after-gc', 'g1-heap-occupancy-before-gc', 'g1-heap-capacity-before-gc', 'g1-heap-occupancy-after-gc', 'g1-heap-capacity-after-gc',
      'g1-young-cpu.sys', 'g1-young-cpu.usr', 'g1-young-cpu.real', 'g1-mixed-cpu.usr', 'g1-mixed-cpu.sys', 'g1-mixed-cpu.real')

  def __init__ (self, metric_type, infile_list, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings,
                important_sub_metrics, **other_options):
    Metric.__init__(self, metric_type, infile_list, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings,
                    important_sub_metrics)
    if not self.important_sub_metrics:
      self.important_sub_metrics = important_sub_metrics_import['GC']
    self.sub_metrics = self.val_types
    self.beginning_ts = None
    self.beginning_date = None
    for (key, val) in other_options.iteritems():
      if key == 'gc-options' or key == 'sub_metrics':
        self.sub_metrics = val.split()
      else:
        setattr(self, key, val)
    self.sub_metric_description = {
      'gen0' : 'young gen collection time, excluding gc_prologue & gc_epilogue',
      'gen0t' : 'young gen collection time, including gc_prologue & gc_epilogue',
      'gen0usr' : 'young gen collection time in cpu user secs',
      'gen0sys' : 'young gen collection time in cpu sys secs',
      'gen0real' : 'young gen collection time in elapsed secs',
      'gen1i' : 'train generation incremental collection',
      'gen1t' : 'old generation collection/full GC',
      'cmsIM' : 'CMS initial mark pause',
      'cmsRM' : 'CMS remark pause',
      'cmsRS' : 'CMS resize pause',
      'GCPause' : 'all stop-the-world GC pauses',
      'cmsCM' : 'CMS concurrent mark phase',
      'cmsCP' : 'CMS concurrent preclean phase',
      'cmsCS' : 'CMS concurrent sweep phase',
      'cmsCR' : 'CMS concurrent reset phase',
      'alloc' : 'object allocation in MB (approximate***)',
      'promo' : 'object promotion in MB (approximate***)',
      'used0' : 'young gen used memory size (before gc)',
      'used1' : 'old gen used memory size (before gc)',
      'used' : 'heap space used memory size (before gc) (excludes perm gen)',
      'commit0' : 'young gen committed memory size (after gc)',
      'commit1' : 'old gen committed memory size (after gc)',
      'commit' : 'heap committed memory size (after gc) (excludes perm gen)',
      'apptime' : 'amount of time application threads were running',
      'safept' : 'amount of time the VM spent at safepoints (app threads stopped)',
      'used0AfterGC' : 'young gen used memory size (after gc)',
      'used1AfterGC' : 'old gen used memory size (after gc)',
      'usedAfterGC' : 'heap space used memory size (after gc)'
      }

  def parse(self):
    prefix = os.path.join(self.resource_directory, self.label)
    awk_cmd = os.path.join(self.bin_path, 'PrintGCStats')
    gc_metrics = set(self.val_types) & set(self.sub_metrics)
    if self.ts_start:
      awk_cmd += ' -v ts_start="' + naarad.utils.get_standardized_timestamp(self.ts_start, None) + '"'
    if self.ts_end:
      awk_cmd += ' -v ts_end="' + naarad.utils.get_standardized_timestamp(self.ts_end, None) + '"'
    cmd = awk_cmd + ' -v plot=' + ','.join(gc_metrics) + ' -v splitfiles=1 -v datestamps=1 -v plotcolumns=2 -v splitfileprefix=' + prefix + ' ' + ' '.join(self.infile_list)
    logger.info("Parsing GC metric with cmd: %s", cmd)
    os.system(cmd)
    for gc_sub_metric in gc_metrics:
      outcsv = self.get_csv(gc_sub_metric)
      if naarad.utils.is_valid_file(outcsv):
        self.csv_files.append(outcsv)
    return True
<commit_msg>Add G1 submetric descriptions<commit_after># coding=utf-8
"""
© 2013 LinkedIn Corp. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
"""
import datetime
import logging
import os
import re
import sys
import threading

from naarad.metrics.metric import Metric
import naarad.utils
from naarad.naarad_constants import important_sub_metrics_import

logger = logging.getLogger('naarad.metrics.GCMetric')

class GCMetric(Metric):
  """ Class for GC logs, deriving from class Metric """
  clock_format = '%Y-%m-%d %H:%M:%S'
  rate_types = ()
  val_types = ('alloc', 'promo', 'used0', 'used1', 'used', 'commit0', 'commit1', 'commit', 'gen0', 'gen0t', 'gen0usr', 'gen0sys', 'gen0real',
      'cmsIM', 'cmsRM', 'cmsRS', 'GCPause', 'cmsCM', 'cmsCP', 'cmsCS', 'cmsCR', 'safept', 'apptime', 'used0AfterGC', 'used1AfterGC', 'usedAfterGC',
      'gen1t', 'g1-pause-young', 'g1-pause-mixed', 'g1-pause-remark', 'g1-pause-cleanup', 'g1-pause-remark.ref-proc', 'g1-pause-young.parallel',
      'g1-pause-young.parallel.gcworkers', 'g1-pause-young.parallel.ext-root-scanning.avg', 'g1-pause-young.parallel.ext-root-scanning.max',
      'g1-pause-young.parallel.update-rs.avg', 'g1-pause-young.parallel.update-rs.max', 'g1-pause-young.parallel.update-rs.processed-buffers.avg',
      'g1-pause-young.parallel.update-rs.processed-buffers.max', 'g1-pause-young.parallel.scan-rs.avg', 'g1-pause-young.parallel.scan-rs.max',
      'g1-pause-young.parallel.object-copy-rs.avg', 'g1-pause-young.parallel.object-copy-rs.max', 'g1-pause-young.parallel.termination.avg',
      'g1-pause-young.parallel.termination.max', 'g1-pause-young.parallel.gc-worker-other.avg', 'g1-pause-young.parallel.gc-worker-other.max',
      'g1-pause-young.parallel.gc-worker-total.avg', 'g1-pause-young.parallel.gc-worker-total.max', 'g1-pause-young.parallel.gc-worker-end.avg',
      'g1-pause-young.parallel.gc-worker-end.max', 'g1-pause-young.code-root-fixup', 'g1-pause-young.clear-ct', 'g1-pause-young.other',
      'g1-pause-young.other.choose-cset', 'g1-pause-young.other.ref-proc', 'g1-pause-young.other.reg-enq', 'g1-pause-young.other.free-cset',
      'g1-pause-mixed.parallel', 'g1-pause-mixed.parallel.gcworkers', 'g1-pause-mixed.parallel.ext-root-scanning.avg',
      'g1-pause-mixed.parallel.ext-root-scanning.max', 'g1-pause-mixed.parallel.update-rs.avg', 'g1-pause-mixed.parallel.update-rs.max',
      'g1-pause-mixed.parallel.update-rs.processed-buffers.avg', 'g1-pause-mixed.parallel.update-rs.processed-buffers.max',
      'g1-pause-mixed.parallel.scan-rs.avg', 'g1-pause-mixed.parallel.scan-rs.max', 'g1-pause-mixed.parallel.object-copy-rs.avg',
      'g1-pause-mixed.parallel.object-copy-rs.max', 'g1-pause-mixed.parallel.termination.avg', 'g1-pause-mixed.parallel.termination.max',
      'g1-pause-mixed.parallel.gc-worker-other.avg', 'g1-pause-mixed.parallel.gc-worker-other.max', 'g1-pause-mixed.parallel.gc-worker-total.avg',
      'g1-pause-mixed.parallel.gc-worker-total.max', 'g1-pause-mixed.parallel.gc-worker-end.avg', 'g1-pause-mixed.parallel.gc-worker-end.max',
      'g1-pause-mixed.code-root-fixup', 'g1-pause-mixed.clear-ct', 'g1-pause-mixed.other', 'g1-pause-mixed.other.choose-cset',
      'g1-pause-mixed.other.ref-proc', 'g1-pause-mixed.other.reg-enq', 'g1-pause-mixed.other.free-cset', 'g1-pause-young.parallel.gc-worker-start.avg',
      'g1-pause-young.parallel.gc-worker-start.max', 'g1-pause-mixed.parallel.gc-worker-start.avg', 'g1-pause-mixed.parallel.gc-worker-start.max',
      'g1-eden-occupancy-before-gc', 'g1-eden-capacity-before-gc', 'g1-eden-occupancy-after-gc', 'g1-eden-capacity-after-gc', 'g1-survivor-before-gc',
      'g1-survivor-after-gc', 'g1-heap-occupancy-before-gc', 'g1-heap-capacity-before-gc', 'g1-heap-occupancy-after-gc', 'g1-heap-capacity-after-gc',
      'g1-young-cpu.sys', 'g1-young-cpu.usr', 'g1-young-cpu.real', 'g1-mixed-cpu.usr', 'g1-mixed-cpu.sys', 'g1-mixed-cpu.real')

  def __init__ (self, metric_type, infile_list, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings,
                important_sub_metrics, **other_options):
    Metric.__init__(self, metric_type, infile_list, hostname, outdir, resource_path, label, ts_start, ts_end, rule_strings,
                    important_sub_metrics)
    if not self.important_sub_metrics:
      self.important_sub_metrics = important_sub_metrics_import['GC']
    self.sub_metrics = self.val_types
    self.beginning_ts = None
    self.beginning_date = None
    for (key, val) in other_options.iteritems():
      if key == 'gc-options' or key == 'sub_metrics':
        self.sub_metrics = val.split()
      else:
        setattr(self, key, val)
    self.sub_metric_description = {
      'gen0' : 'young gen collection time, excluding gc_prologue & gc_epilogue',
      'gen0t' : 'young gen collection time, including gc_prologue & gc_epilogue',
      'gen0usr' : 'young gen collection time in cpu user secs',
      'gen0sys' : 'young gen collection time in cpu sys secs',
      'gen0real' : 'young gen collection time in elapsed secs',
      'gen1i' : 'train generation incremental collection',
      'gen1t' : 'old generation collection/full GC',
      'cmsIM' : 'CMS initial mark pause',
      'cmsRM' : 'CMS remark pause',
      'cmsRS' : 'CMS resize pause',
      'GCPause' : 'all stop-the-world GC pauses',
      'cmsCM' : 'CMS concurrent mark phase',
      'cmsCP' : 'CMS concurrent preclean phase',
      'cmsCS' : 'CMS concurrent sweep phase',
      'cmsCR' : 'CMS concurrent reset phase',
      'alloc' : 'object allocation in MB (approximate***)',
      'promo' : 'object promotion in MB (approximate***)',
      'used0' : 'young gen used memory size (before gc)',
      'used1' : 'old gen used memory size (before gc)',
      'used' : 'heap space used memory size (before gc) (excludes perm gen)',
      'commit0' : 'young gen committed memory size (after gc)',
      'commit1' : 'old gen committed memory size (after gc)',
      'commit' : 'heap committed memory size (after gc) (excludes perm gen)',
      'apptime' : 'amount of time application threads were running',
      'safept' : 'amount of time the VM spent at safepoints (app threads stopped)',
      'used0AfterGC' : 'young gen used memory size (after gc)',
      'used1AfterGC' : 'old gen used memory size (after gc)',
      'usedAfterGC' : 'heap space used memory size (after gc)',
      'g1-pause-young' : 'G1 Young GC Pause (seconds)',
      'g1-pause-mixed' : 'G1 Mixed GC Pause (seconds)',
      'g1-pause-remark' : 'G1 Remark Pause (seconds)',
      'g1-pause-cleanup' : 'G1 Cleanup Pause (seconds)',
      'g1-pause-remark.ref-proc' : 'G1 Remark: Reference Processing (seconds)',
      'g1-pause-young.parallel' : 'G1 Young GC Pause: Parallel Operations (ms)',
      'g1-pause-young.parallel.gcworkers' : 'G1 Young GC Pause: Number of Parallel GC Workers',
      'g1-pause-young.parallel.gc-worker-start.avg' : 'G1 Young GC Pause : Parallel : Avg Time spent in GC worker start (ms)',
      'g1-pause-young.parallel.gc-worker-start.max' : 'G1 Young GC Pause : Parallel : Max Time spent in GC worker start (ms)',
      'g1-pause-young.parallel.ext-root-scanning.avg' : 'G1 Young GC Pause: Avg Time spent in ext-root-scanning',
      'g1-pause-young.parallel.ext-root-scanning.max' : 'G1 Young GC Pause: Max Time spent in ext-root-scanning',
      'g1-pause-young.parallel.update-rs.avg' : 'G1 Young GC Pause: Parallel : Avg Time spent in updating Rsets',
      'g1-pause-young.parallel.update-rs.max' : 'G1 Young GC Pause: Parallel : Max Time spent in updating Rsets',
      'g1-pause-young.parallel.update-rs.processed-buffers.avg' : 'G1 Young GC Pause : Parallel : Update Rset : Avg number of processed buffers',
      'g1-pause-young.parallel.update-rs.processed-buffers.max' : 'G1 Young GC Pause : Parallel : Update Rset : Max number of processed buffers',
      'g1-pause-young.parallel.scan-rs.avg' : 'G1 Young GC Pause: Parallel : Avg Time spent in scanning Rsets',
      'g1-pause-young.parallel.scan-rs.max' : 'G1 Young GC Pause: Parallel : Max Time spent in scannning Rsets',
      'g1-pause-young.parallel.object-copy-rs.avg' : 'G1 Young GC Pause : Parallel : Avg Time spent in Object Copy',
      'g1-pause-young.parallel.object-copy-rs.max' : 'G1 Young GC Pause : Parallel : Max Time spent in Object Copy',
      'g1-pause-young.parallel.termination.avg' : 'G1 Young GC Pause : Parallel : Avg Time spent in termination',
      'g1-pause-young.parallel.termination.max' : 'G1 Young GC Pause : Parallel : Max Time spent in termination',
      'g1-pause-young.parallel.gc-worker-other.avg' : 'G1 Young GC Pause : Parallel : Avg Time spent in other',
      'g1-pause-young.parallel.gc-worker-other.max' : 'G1 Young GC Pause : Parallel : Max Time spent in other',
      'g1-pause-young.parallel.gc-worker-total.avg' : 'G1 Young GC Pause : Parallel : Avg Total time for GC worker',
      'g1-pause-young.parallel.gc-worker-total.max' : 'G1 Young GC Pause : Parallel : Max Total time for GC worker',
      'g1-pause-young.parallel.gc-worker-end.avg' : 'G1 Young GC Pause : Parallel : Avg Time for GC worker end',
      'g1-pause-young.parallel.gc-worker-end.max' : 'G1 Young GC Pause : Parallel : Max Time for GC worker end',
      'g1-pause-young.code-root-fixup' : 'G1 Young GC Pause : Time spent in code root fixup (ms)',
      'g1-pause-young.clear-ct' : 'G1 Young GC Pause: Time spent in clear ct (ms)',
      'g1-pause-young.other' : 'G1 Young GC Pause: Time spent in other (ms)',
      'g1-pause-young.other.choose-cset' : 'G1 Young GC Pause : Other : Time spent in choosing CSet (ms)',
      'g1-pause-young.other.ref-proc' : 'G1 Young GC Pause : Other : Time spent in reference processing (ms)',
      'g1-pause-young.other.reg-enq' : 'G1 Young GC Pause : Other : Time spent in reg-enq(ms)',
      'g1-pause-young.other.free-cset' : 'G1 Young GC Pause : Other : Time spent in processing free Cset(ms)',
      'g1-pause-mixed.parallel' : 'G1 Mixed GC Pause: Parallel Operations (ms)',
      'g1-pause-mixed.parallel.gcworkers' : 'G1 Mixed GC Pause: Number of Parallel GC Workers',
      'g1-pause-mixed.parallel.gc-worker-start.avg' : 'G1 Mixed GC Pause : Parallel : Avg Time spent in GC worker start (ms)',
      'g1-pause-mixed.parallel.gc-worker-start.max' : 'G1 Mixed GC Pause : Parallel : Max Time spent in GC worker start (ms)',
      'g1-pause-mixed.parallel.ext-root-scanning.avg' : 'G1 Mixed GC Pause: Avg Time spent in ext-root-scanning',
      'g1-pause-mixed.parallel.ext-root-scanning.max' : 'G1 Mixed GC Pause: Max Time spent in ext-root-scanning',
      'g1-pause-mixed.parallel.update-rs.avg' : 'G1 Mixed GC Pause: Parallel : Avg Time spent in updating Rsets',
      'g1-pause-mixed.parallel.update-rs.max' : 'G1 Mixed GC Pause: Parallel : Max Time spent in updating Rsets',
      'g1-pause-mixed.parallel.update-rs.processed-buffers.avg' : 'G1 Mixed GC Pause : Parallel : Update Rset : Avg number of processed buffers',
      'g1-pause-mixed.parallel.update-rs.processed-buffers.max' : 'G1 Mixed GC Pause : Parallel : Update Rset : Max number of processed buffers',
      'g1-pause-mixed.parallel.scan-rs.avg' : 'G1 Mixed GC Pause: Parallel : Avg Time spent in scanning Rsets',
      'g1-pause-mixed.parallel.scan-rs.max' : 'G1 Mixed GC Pause: Parallel : Max Time spent in scannning Rsets',
      'g1-pause-mixed.parallel.object-copy-rs.avg' : 'G1 Mixed GC Pause : Parallel : Avg Time spent in Object Copy',
      'g1-pause-mixed.parallel.object-copy-rs.max' : 'G1 Mixed GC Pause : Parallel : Max Time spent in Object Copy',
      'g1-pause-mixed.parallel.termination.avg' : 'G1 Mixed GC Pause : Parallel : Avg Time spent in termination',
      'g1-pause-mixed.parallel.termination.max' : 'G1 Mixed GC Pause : Parallel : Max Time spent in termination',
      'g1-pause-mixed.parallel.gc-worker-other.avg' : 'G1 Mixed GC Pause : Parallel : Avg Time spent in other',
      'g1-pause-mixed.parallel.gc-worker-other.max' : 'G1 Mixed GC Pause : Parallel : Max Time spent in other',
      'g1-pause-mixed.parallel.gc-worker-total.avg' : 'G1 Mixed GC Pause : Parallel : Avg Total time for GC worker',
      'g1-pause-mixed.parallel.gc-worker-total.max' : 'G1 Mixed GC Pause : Parallel : Max Total time for GC worker',
      'g1-pause-mixed.parallel.gc-worker-end.avg' : 'G1 Mixed GC Pause : Parallel : Avg Time for GC worker end',
      'g1-pause-mixed.parallel.gc-worker-end.max' : 'G1 Mixed GC Pause : Parallel : Max Time for GC worker end',
      'g1-pause-mixed.code-root-fixup' : 'G1 Mixed GC Pause : Time spent in code root fixup (ms)',
      'g1-pause-mixed.clear-ct' : 'G1 Mixed GC Pause: Time spent in clear ct (ms)',
      'g1-pause-mixed.other' : 'G1 Mixed GC Pause: Time spent in other (ms)',
      'g1-pause-mixed.other.choose-cset' : 'G1 Mixed GC Pause : Other : Time spent in choosing CSet (ms)',
      'g1-pause-mixed.other.ref-proc' : 'G1 Mixed GC Pause : Other : Time spent in reference processing (ms)',
      'g1-pause-mixed.other.reg-enq' : 'G1 Mixed GC Pause : Other : Time spent in reg-enq(ms)',
      'g1-pause-mixed.other.free-cset' : 'G1 Mixed GC Pause : Other : Time spent in processing free Cset(ms)',
      'g1-eden-occupancy-before-gc' : 'G1 Eden Occupancy (MB) (Before GC)',
      'g1-eden-capacity-before-gc' : 'G1 Eden Capacity (MB) (Before GC)',
      'g1-eden-occupancy-after-gc' : 'G1 Eden Occupancy (MB) (After GC)',
      'g1-eden-capacity-after-gc' : 'G1 Efen Capacity (MB) (After GC)',
      'g1-survivor-before-gc' : 'G1 Survivor Size (MB) (Before GC)',
      'g1-survivor-after-gc' : 'G1 Survivor Size (MB) (After GC)',
      'g1-heap-occupancy-before-gc' : 'G1 Heap Occupancy (MB) (Before GC)',
      'g1-heap-capacity-before-gc' : 'G1 Heap Capacity (MB) (Before GC)',
      'g1-heap-occupancy-after-gc' : 'G1 Heap Occupancy (MB) (After GC)',
      'g1-heap-capacity-after-gc' : 'G1 Heap Capacity (MB) (After GC)',
      'g1-young-cpu.sys' : 'G1 Young GC : sys cpu time (seconds)',
      'g1-young-cpu.usr' : 'G1 Young GC : usr cpu time (seconds)',
      'g1-young-cpu.real' : 'G1 Young GC : elapsed time (seconds)',
      'g1-mixed-cpu.usr' : 'G1 Mixed GC : usr cpu time (seconds)',
      'g1-mixed-cpu.sys' : 'G1 Mixed GC : sys cpu time (seconds)',
      'g1-mixed-cpu.real' : 'G1 Mixed GC : elapsed time (seconds)'
      }

  def parse(self):
    prefix = os.path.join(self.resource_directory, self.label)
    awk_cmd = os.path.join(self.bin_path, 'PrintGCStats')
    gc_metrics = set(self.val_types) & set(self.sub_metrics)
    if self.ts_start:
      awk_cmd += ' -v ts_start="' + naarad.utils.get_standardized_timestamp(self.ts_start, None) + '"'
    if self.ts_end:
      awk_cmd += ' -v ts_end="' + naarad.utils.get_standardized_timestamp(self.ts_end, None) + '"'
    cmd = awk_cmd + ' -v plot=' + ','.join(gc_metrics) + ' -v splitfiles=1 -v datestamps=1 -v plotcolumns=2 -v splitfileprefix=' + prefix + ' ' + ' '.join(self.infile_list)
    logger.info("Parsing GC metric with cmd: %s", cmd)
    os.system(cmd)
    for gc_sub_metric in gc_metrics:
      outcsv = self.get_csv(gc_sub_metric)
      if naarad.utils.is_valid_file(outcsv):
        self.csv_files.append(outcsv)
    return True
<|endoftext|>


<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# coding: utf-8
# pylint: disable=
"""Dataset sampler."""
__all__ = ['Sampler', 'SequentialSampler', 'RandomSampler', 'BatchSampler']

import random

class Sampler(object):
    """Base class for samplers.

    All samplers should subclass `Sampler` and define `__iter__` and `__len__`
    methods.
    """
    def __iter__(self):
        raise NotImplementedError

    def __len__(self):
        raise NotImplementedError

class SequentialSampler(Sampler):
    """Samples elements from [0, length) sequentially.

    Parameters
    ----------
    length : int
        Length of the sequence.
    """
    def __init__(self, length):
        self._length = length

    def __iter__(self):
        return iter(range(self._length))

    def __len__(self):
        return self._length

class RandomSampler(Sampler):
    """Samples elements from [0, length) randomly without replacement.

    Parameters
    ----------
    length : int
        Length of the sequence.
    """
    def __init__(self, length):
        self._length = length

    def __iter__(self):
        indices = list(range(self._length))
        random.shuffle(indices)
        return iter(indices)

    def __len__(self):
        return self._length

class BatchSampler(Sampler):
    """Wraps over another `Sampler` and return mini-batches of samples.

    Parameters
    ----------
    sampler : Sampler
        The source Sampler.
    batch_size : int
        Size of mini-batch.
    last_batch : {'keep', 'discard', 'rollover'}
        Specifies how the last batch is handled if batch_size does not evenly
        divide sequence length.

        If 'keep', the last batch will be returned directly, but will contain
        less element than `batch_size` requires.

        If 'discard', the last batch will be discarded.

        If 'rollover', the remaining elements will be rolled over to the next
        iteration.

    Examples
    --------
    >>> sampler = gluon.data.SequentialSampler(10)
    >>> batch_sampler = gluon.data.BatchSampler(sampler, 3, 'keep')
    >>> list(batch_sampler)
    [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
    """
    def __init__(self, sampler, batch_size, last_batch='keep'):
        self._sampler = sampler
        self._batch_size = batch_size
        self._last_batch = last_batch
        self._prev = []

    def __iter__(self):
        batch, self._prev = self._prev, []
        for i in self._sampler:
            batch.append(i)
            if len(batch) == self._batch_size:
                yield batch
                batch = []
        if batch:
            if self._last_batch == 'keep':
                yield batch
            elif self._last_batch == 'discard':
                return
            elif self._last_batch == 'rollover':
                self._prev = batch
            else:
                raise ValueError(
                    "last_batch must be one of 'keep', 'discard', or 'rollover', " \
                    "but got %s"%self._last_batch)

    def __len__(self):
        if self._last_batch == 'keep':
            return (len(self._sampler) + self._batch_size - 1) // self._batch_size
        if self._last_batch == 'discard':
            return len(self._sampler) // self._batch_size
        if self._last_batch == 'rollover':
            return (len(self._prev) + len(self._sampler)) // self._batch_size
        raise ValueError(
            "last_batch must be one of 'keep', 'discard', or 'rollover', " \
            "but got %s"%self._last_batch)
<commit_msg>Use numpy.arange in RandomSampler (#10768)<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# coding: utf-8
# pylint: disable=
"""Dataset sampler."""
__all__ = ['Sampler', 'SequentialSampler', 'RandomSampler', 'BatchSampler']

import numpy as np

class Sampler(object):
    """Base class for samplers.

    All samplers should subclass `Sampler` and define `__iter__` and `__len__`
    methods.
    """
    def __iter__(self):
        raise NotImplementedError

    def __len__(self):
        raise NotImplementedError

class SequentialSampler(Sampler):
    """Samples elements from [0, length) sequentially.

    Parameters
    ----------
    length : int
        Length of the sequence.
    """
    def __init__(self, length):
        self._length = length

    def __iter__(self):
        return iter(range(self._length))

    def __len__(self):
        return self._length

class RandomSampler(Sampler):
    """Samples elements from [0, length) randomly without replacement.

    Parameters
    ----------
    length : int
        Length of the sequence.
    """
    def __init__(self, length):
        self._length = length

    def __iter__(self):
        indices = np.arange(self._length)
        np.random.shuffle(indices)
        return iter(indices)

    def __len__(self):
        return self._length

class BatchSampler(Sampler):
    """Wraps over another `Sampler` and return mini-batches of samples.

    Parameters
    ----------
    sampler : Sampler
        The source Sampler.
    batch_size : int
        Size of mini-batch.
    last_batch : {'keep', 'discard', 'rollover'}
        Specifies how the last batch is handled if batch_size does not evenly
        divide sequence length.

        If 'keep', the last batch will be returned directly, but will contain
        less element than `batch_size` requires.

        If 'discard', the last batch will be discarded.

        If 'rollover', the remaining elements will be rolled over to the next
        iteration.

    Examples
    --------
    >>> sampler = gluon.data.SequentialSampler(10)
    >>> batch_sampler = gluon.data.BatchSampler(sampler, 3, 'keep')
    >>> list(batch_sampler)
    [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
    """
    def __init__(self, sampler, batch_size, last_batch='keep'):
        self._sampler = sampler
        self._batch_size = batch_size
        self._last_batch = last_batch
        self._prev = []

    def __iter__(self):
        batch, self._prev = self._prev, []
        for i in self._sampler:
            batch.append(i)
            if len(batch) == self._batch_size:
                yield batch
                batch = []
        if batch:
            if self._last_batch == 'keep':
                yield batch
            elif self._last_batch == 'discard':
                return
            elif self._last_batch == 'rollover':
                self._prev = batch
            else:
                raise ValueError(
                    "last_batch must be one of 'keep', 'discard', or 'rollover', " \
                    "but got %s"%self._last_batch)

    def __len__(self):
        if self._last_batch == 'keep':
            return (len(self._sampler) + self._batch_size - 1) // self._batch_size
        if self._last_batch == 'discard':
            return len(self._sampler) // self._batch_size
        if self._last_batch == 'rollover':
            return (len(self._prev) + len(self._sampler)) // self._batch_size
        raise ValueError(
            "last_batch must be one of 'keep', 'discard', or 'rollover', " \
            "but got %s"%self._last_batch)
<|endoftext|>