<commit_before>"""
This module provides functionality for manipulating proteomics data sets.

Functionality includes merging data sets and interfacing with attributes in a
structured format.
"""

# Built-ins
from collections import OrderedDict
import copy
import logging

# Core data analysis libraries
import pandas as pd
import numpy as np
from scipy.stats import ttest_ind

from . import loading, utils

LOGGER = logging.getLogger("pyproteome.DataSet")

class DataSet:
    """
    Class that encompasses a proteomics data set.

    Includes peptide list, scan list, channels, groups, quantitative
    phenotypes, etc.

    Attributes
    ----------
    psms : pandas.DataFrame
        Contains at least "Proteins", "Sequence", and "Modifications" columns.
    channels : dict of str, str
        Maps label channel to sample name.
    groups : dict of str, list of str
        Maps groups to list of sample names. The primary group is considered
        as the first in this sequence.
    phenotypes : dict of str, (dict of str, float)
        Primary key is the phenotype and its value is a dictionary mapping
        sample names to that phenotype's value.
    name : str
    enrichments : list of str
    tissues : list of str
    sets : int
        Number of sets merged into this data set.
    source : str
    scan_list : dict of str, list of int
    validated : bool
    """
    def __init__(
        self, channels,
        psms=None,
        mascot_name=None,
        camv_name=None,
        groups=None, phenotypes=None,
        name="", enrichments=None, tissues=None,
        dropna=True,
        camv_slices=None,
    ):
        """
        Initializes a data set.

        Parameters
        ----------
        channels : dict of str, str
        psms : pandas.DataFrame, optional
        mascot_name : str, optional
        camv_name : str, optional
        groups : dict of str, list of str, optional
        phenotypes : dict of str, (dict of str, float), optional
        name : str, optional
        enrichments : list of str, optional
        tissues : list of str, optional
        dropna : bool, optional
        camv_slices : int, optional
        """
        assert psms is not None or \
            mascot_name is not None or \
            camv_name is not None

        self.source = "unknown"
        self.scan_lists = None
        self.validated = False

        if mascot_name:
            psms, self.scan_lists, self.validated = loading.load_mascot_psms(
                mascot_name,
                camv_slices=camv_slices,
            )
            self.source = "MASCOT"
        elif camv_name:
            psms = loading.load_validated_psms(
                camv_name,
            )
            self.source = "CAMV"
            self.validated = True

        self.psms = psms
        self.channels = channels
        self.groups = groups
        self.phenotypes = phenotypes
        self.name = name
        self.enrichments = enrichments
        self.tissues = tissues

        self.normalized = False
        self.sets = 1

        if dropna:
            LOGGER.info("Dropping channels with NaN values.")
            self.dropna(inplace=True)

    def copy(self):
        new = copy.copy(self)
        new.psms = new.psms.copy()
        return new

    @property
    def enrichment(self):
        return "+".join(self.enrichments)

    @property
    def tissue(self):
        return "+".join(self.tissues)

    def __str__(self):
        return (
            "<pyproteome.DataSet object" +
            (
                ": " + self.name
                if self.name else
                " at " + hex(id(self))
            ) +
            ">"
        )

    def __getitem__(self, key):
        if isinstance(key, slice):
            new = self.copy()
            new.psms = new.psms[key]
            return new

        if isinstance(key, str) or isinstance(key, pd.Series):
            return self.psms[key]

        raise TypeError

    def _merge_psms(self):
        channels = list(self.channels.keys())

        agg_dict = dict((channel, sum) for channel in channels)
        self.psms = self.psms.groupby(
            [
                "Proteins",
                "Sequence",
                "Modifications",
            ],
            sort=False,
            as_index=False,
        ).agg(agg_dict)

    def __add__(self, other):
        """
        Concatenate two data sets.

        Combines two data sets, adding together the channel values for any
        common data.

        Parameters
        ----------
        other : pyproteome.DataSet

        Returns
        -------
        pyproteome.DataSet
        """
        return merge_data([self, other])

    def normalize(self, levels, inplace=False):
        """
        Normalizes a channel to given levels.

        Divides all channel values by a given level.

        Parameters
        ----------
        levels : dict of str, float
            Mapping of channel names to normalized levels.
        inplace : bool, optional
            Modify this data set in place.

        Returns
        -------
        pyproteome.DataSet
        """
        new = self

        # Don't normalize a data set twice!
        assert not new.normalized

        if not inplace:
            new = new.copy()

        new_channels = utils.norm(self.channels)

        for key, norm_key in zip(self.channels, new_channels):
            new.psms[norm_key] = new.psms[key] / levels[key]

        new.normalized = True
        new.channels = new_channels
        new.groups = OrderedDict(
            (key, utils.norm(val))
            for key, val in self.groups.items()
        )

        new._merge_psms()
        new._update_snr_change()

        return new

    def dropna(self, inplace=False):
        """
        Drop any channels with NaN values.

        Parameters
        ----------
        inplace : bool, optional

        Returns
        -------
        pyproteome.DataSet
        """
        new = self

        if not inplace:
            new = new.copy()

        new.psms = new.psms.dropna(
            axis=0,
            how="any",
            subset=list(new.channels.keys()),
        )

        return new

    def filter(
        self,
        ion_score_cutoff=None, confidence_cutoff=None,
        snr_cutoff=None,
        asym_snr_cutoff=None,
        p_cutoff=None,
        fold_cutoff=None,
        asym_fold_cutoff=None,
        inplace=False,
    ):
        """
        Filters a data set.

        Parameters
        ----------
        ion_score_cutoff : int, optional
        confidence_cutoff : {"High", "Medium", "Low"}, optional
        snr_cutoff : float, optional
        asym_snr_cutoff : float, optional
        p_cutoff : float, optional
        fold_cutoff : float, optional
        asym_fold_cutoff : float, optional
        inplace : bool, optional

        Returns
        -------
        pyproteome.DataSet
        """
        if ion_score_cutoff or confidence_cutoff:
            assert self.source in ["MASCOT"]

        new = self

        if not inplace:
            new = new.copy()

        confidence_levels = []

        # MASCOT confidence levels
        if confidence_cutoff:
            confidence_levels += [confidence_cutoff]
        if confidence_cutoff in ["Medium", "Low"]:
            confidence_levels += ["High"]
        if confidence_levels in ["Low"]:
            confidence_levels += ["Medium"]

        if confidence_levels:
            new.psms = new.psms[
                new.psms["Confidence Level"].isin(confidence_levels)
            ]

        # MASCOT ion scores
        if ion_score_cutoff:
            new.psms = new.psms[
                new.psms["IonScore"] >= ion_score_cutoff
            ]

        if snr_cutoff:
            new.psms = new.psms[
                abs(new.psms["SNR"]) >= snr_cutoff
            ]

        if asym_snr_cutoff:
            if asym_snr_cutoff > 0:
                new.psms = new.psms[
                    new.psms["SNR"] >= asym_snr_cutoff
                ]
            else:
                new.psms = new.psms[
                    new.psms["SNR"] <= asym_snr_cutoff
                ]

        if p_cutoff:
            new.psms = new.psms[
                new.psms["p-value"] <= p_cutoff
            ]

        if asym_fold_cutoff:
            if asym_fold_cutoff > 1:
                new.psms = new.psms[
                    new.psms["Fold Change"] >= asym_fold_cutoff
                ]
            else:
                new.psms = new.psms[
                    new.psms["Fold Change"] <= asym_fold_cutoff
                ]

        if fold_cutoff:
            if fold_cutoff < 1:
                fold_cutoff = 1 / fold_cutoff

            new.psms = new.psms[
                np.maximum.reduce(
                    [
                        new.psms["Fold Change"],
                        1 / new.psms["Fold Change"],
                    ]
                ) >= fold_cutoff
            ]

        return new

    def _update_snr_change(self):
        """
        Update a table's SNR, Fold-Change, and p-value columns.

        Values are calculated based on changes between group_a and group_b.

        Parameters
        ----------
        psms : pandas.DataFrame
        group_a : list of str
        group_b : list of str
        normalize : bool, optional

        Returns
        -------
        pandas.DataFrame
        """
        groups = list(self.groups.values())

        if len(groups) == 2:
            self.psms["SNR"] = pd.Series(
                [
                    _snr(row[groups[0]], row[groups[1]])
                    for index, row in self.psms.iterrows()
                ]
            )

            self.psms["Fold Change"] = pd.Series(
                self.psms[groups[0]].mean(axis=1) /
                self.psms[groups[1]].mean(axis=1)
            )
            self.psms["p-value"] = pd.Series(
                ttest_ind(
                    self.psms[groups[0]],
                    self.psms[groups[1]],
                    axis=1,
                )[1]
            )

def merge_data(data_sets, name=None):
    """
    Merge a list of data sets together.

    Parameters
    ----------
    data_sets : list of pyproteome.DataSet
    name : str, optional

    Returns
    -------
    pyproteome.DataSet
    """
    assert len(data_sets) > 0

    if any(not isinstance(data, DataSet) for data in data_sets):
        raise TypeError(
            "Incompatible types: {}".format(
                [type(data) for data in data_sets]
            )
        )

    if any(data.groups != data_sets[0].groups for data in data_sets):
        raise Exception("Incongruent groups between data sets.")

    new = data_sets[0].copy()
    new.psms = pd.concat([data.psms for data in data_sets])
    new._merge_psms()
    new.sets = sum(data.sets for data in data_sets)
    new.enrichments = sorted(
        set(
            enrichment
            for data in data_sets
            for enrichment in data.enrichments
        )
    )
    new.tissues = sorted(
        set(
            tissue
            for data in data_sets
            for tissue in data.tissues
        )
    )

    if new.groups:
        new._update_snr_change()

    if name:
        new.name = name

    return new

def _log_cum_fold_change(vals):
    """
    Calculate the cumulative fold change (in base-2) of values.

    Fold-change is normalized to the first element in the array.

    Parameters
    ----------
    vals : numpy.array

    Returns
    -------
    float
    """
    return sum(
        abs(np.log2(i / (vals[0])))
        for i in vals[1:]
        if i != 0
    )

def _snr(data1, data2):
    """
    Calculate the signal-to-noise ratio between two groups of data.

    Parameters
    ----------
    data1 : numpy.array of float
    data2 : numpy.array of float

    Returns
    -------
    float
    """
    return (data1.mean() - data2.mean()) \
        / (data1.std(ddof=1) + data2.std(ddof=1))
<commit_msg>Added merge_subsets and merge_duplicates toggles<commit_after>"""
This module provides functionality for manipulating proteomics data sets.

Functionality includes merging data sets and interfacing with attributes in a
structured format.
"""

# Built-ins
from collections import OrderedDict
import copy
import logging

# Core data analysis libraries
import pandas as pd
import numpy as np
from scipy.stats import ttest_ind

from . import loading, utils

LOGGER = logging.getLogger("pyproteome.DataSet")

class DataSet:
    """
    Class that encompasses a proteomics data set.

    Includes peptide list, scan list, channels, groups, quantitative
    phenotypes, etc.

    Attributes
    ----------
    psms : pandas.DataFrame
        Contains at least "Proteins", "Sequence", and "Modifications" columns.
    channels : dict of str, str
        Maps label channel to sample name.
    groups : dict of str, list of str
        Maps groups to list of sample names. The primary group is considered
        as the first in this sequence.
    phenotypes : dict of str, (dict of str, float)
        Primary key is the phenotype and its value is a dictionary mapping
        sample names to that phenotype's value.
    name : str
    enrichments : list of str
    tissues : list of str
    sets : int
        Number of sets merged into this data set.
    source : str
    scan_list : dict of str, list of int
    validated : bool
    """
    def __init__(
        self, channels,
        psms=None,
        mascot_name=None,
        camv_name=None,
        groups=None, phenotypes=None,
        name="", enrichments=None, tissues=None,
        dropna=True,
        camv_slices=None,
        merge_duplicates=True,
        merge_subsets=True,
    ):
        """
        Initializes a data set.

        Parameters
        ----------
        channels : dict of str, str
        psms : pandas.DataFrame, optional
        mascot_name : str, optional
        camv_name : str, optional
        groups : dict of str, list of str, optional
        phenotypes : dict of str, (dict of str, float), optional
        name : str, optional
        enrichments : list of str, optional
        tissues : list of str, optional
        dropna : bool, optional
        camv_slices : int, optional
        merge_duplicates : bool, optional
        merge_subsets : bool, optional
        """
        assert psms is not None or \
            mascot_name is not None or \
            camv_name is not None

        self.source = "unknown"
        self.scan_lists = None
        self.validated = False

        if mascot_name:
            psms, self.scan_lists, self.validated = loading.load_mascot_psms(
                mascot_name,
                camv_slices=camv_slices,
            )
            self.source = "MASCOT"
        elif camv_name:
            psms = loading.load_validated_psms(
                camv_name,
            )
            self.source = "CAMV"
            self.validated = True

        self.psms = psms
        self.channels = channels
        self.groups = groups
        self.phenotypes = phenotypes
        self.name = name
        self.enrichments = enrichments
        self.tissues = tissues

        self.normalized = False
        self.sets = 1

        if dropna:
            LOGGER.info("Dropping channels with NaN values.")
            self.dropna(inplace=True)

        if merge_duplicates:
            LOGGER.info("Merging duplicate peptide hits together.")
            self._merge_duplicates()

        if merge_subsets:
            LOGGER.info("Merging peptide hits that are subsets together.")
            self._merge_subsequences()

    def copy(self):
        new = copy.copy(self)
        new.psms = new.psms.copy()
        return new

    @property
    def enrichment(self):
        return "+".join(self.enrichments)

    @property
    def tissue(self):
        return "+".join(self.tissues)

    def __str__(self):
        return (
            "<pyproteome.DataSet object" +
            (
                ": " + self.name
                if self.name else
                " at " + hex(id(self))
            ) +
            ">"
        )

    def __getitem__(self, key):
        if isinstance(key, slice):
            new = self.copy()
            new.psms = new.psms[key]
            return new

        if isinstance(key, str) or isinstance(key, pd.Series):
            return self.psms[key]

        raise TypeError

    def _merge_duplicates(self):
        channels = list(self.channels.keys())
        agg_dict = dict((channel, sum) for channel in channels)

        self.psms = self.psms.groupby(
            by=[
                "Proteins",
                "Sequence",
                "Modifications",
            ],
            sort=False,
            as_index=False,
        ).agg(agg_dict)

    def _merge_subsequences(self):
        """
        Merges petides that are a subsequence of another peptide.

        Only merges peptides that contain the same set of modifications and
        that map to the same protein(s).
        """
        channels = list(self.channels.keys())
        agg_dict = dict((channel, sum) for channel in channels)

        psms = self.psms

        # Find all proteins that have more than one peptide mapping to them
        for index, row in psms[
            psms.duplicated(subset="Proteins", keep=False)
        ].iterrows():
            seq = row["Sequence"]

            # Then iterate over each peptide and find other non-identical
            # peptides that map to the same protein
            for o_index, o_row in psms[
                np.logical_and(
                    psms["Proteins"] == row["Proteins"],
                    psms["Sequence"] != seq,
                )
            ].iterrows():
                # If that other peptide is a subset of this peptide, rename it
                if o_row["Sequence"] in seq:
                    psms.set_value(o_index, "Sequence", seq)
                    psms.set_value(o_index, "Modifications", seq.modifications)

        # And finally group together peptides that were renamed
        self.psms = psms.groupby(
            by=[
                "Proteins",
                "Sequence",
                "Modifications",
            ],
            sort=False,
            as_index=False,
        ).agg(agg_dict)

    def __add__(self, other):
        """
        Concatenate two data sets.

        Combines two data sets, adding together the channel values for any
        common data.

        Parameters
        ----------
        other : pyproteome.DataSet

        Returns
        -------
        pyproteome.DataSet
        """
        return merge_data([self, other])

    def normalize(self, levels, inplace=False):
        """
        Normalizes a channel to given levels.

        Divides all channel values by a given level.

        Parameters
        ----------
        levels : dict of str, float
            Mapping of channel names to normalized levels.
        inplace : bool, optional
            Modify this data set in place.

        Returns
        -------
        pyproteome.DataSet
        """
        new = self

        # Don't normalize a data set twice!
        assert not new.normalized

        if not inplace:
            new = new.copy()

        new_channels = utils.norm(self.channels)

        for key, norm_key in zip(self.channels, new_channels):
            new.psms[norm_key] = new.psms[key] / levels[key]

        new.normalized = True
        new.channels = new_channels
        new.groups = OrderedDict(
            (key, utils.norm(val))
            for key, val in self.groups.items()
        )

        new._update_snr_change()

        return new

    def dropna(self, inplace=False):
        """
        Drop any channels with NaN values.

        Parameters
        ----------
        inplace : bool, optional

        Returns
        -------
        pyproteome.DataSet
        """
        new = self

        if not inplace:
            new = new.copy()

        new.psms = new.psms.dropna(
            axis=0,
            how="any",
            subset=list(new.channels.keys()),
        )

        return new

    def filter(
        self,
        ion_score_cutoff=None, confidence_cutoff=None,
        snr_cutoff=None,
        asym_snr_cutoff=None,
        p_cutoff=None,
        fold_cutoff=None,
        asym_fold_cutoff=None,
        inplace=False,
    ):
        """
        Filters a data set.

        Parameters
        ----------
        ion_score_cutoff : int, optional
        confidence_cutoff : {"High", "Medium", "Low"}, optional
        snr_cutoff : float, optional
        asym_snr_cutoff : float, optional
        p_cutoff : float, optional
        fold_cutoff : float, optional
        asym_fold_cutoff : float, optional
        inplace : bool, optional

        Returns
        -------
        pyproteome.DataSet
        """
        if ion_score_cutoff or confidence_cutoff:
            assert self.source in ["MASCOT"]

        new = self

        if not inplace:
            new = new.copy()

        confidence_levels = []

        # MASCOT confidence levels
        if confidence_cutoff:
            confidence_levels += [confidence_cutoff]
        if confidence_cutoff in ["Medium", "Low"]:
            confidence_levels += ["High"]
        if confidence_levels in ["Low"]:
            confidence_levels += ["Medium"]

        if confidence_levels:
            new.psms = new.psms[
                new.psms["Confidence Level"].isin(confidence_levels)
            ]

        # MASCOT ion scores
        if ion_score_cutoff:
            new.psms = new.psms[
                new.psms["IonScore"] >= ion_score_cutoff
            ]

        if snr_cutoff:
            new.psms = new.psms[
                abs(new.psms["SNR"]) >= snr_cutoff
            ]

        if asym_snr_cutoff:
            if asym_snr_cutoff > 0:
                new.psms = new.psms[
                    new.psms["SNR"] >= asym_snr_cutoff
                ]
            else:
                new.psms = new.psms[
                    new.psms["SNR"] <= asym_snr_cutoff
                ]

        if p_cutoff:
            new.psms = new.psms[
                new.psms["p-value"] <= p_cutoff
            ]

        if asym_fold_cutoff:
            if asym_fold_cutoff > 1:
                new.psms = new.psms[
                    new.psms["Fold Change"] >= asym_fold_cutoff
                ]
            else:
                new.psms = new.psms[
                    new.psms["Fold Change"] <= asym_fold_cutoff
                ]

        if fold_cutoff:
            if fold_cutoff < 1:
                fold_cutoff = 1 / fold_cutoff

            new.psms = new.psms[
                np.maximum.reduce(
                    [
                        new.psms["Fold Change"],
                        1 / new.psms["Fold Change"],
                    ]
                ) >= fold_cutoff
            ]

        return new

    def _update_snr_change(self):
        """
        Update a table's SNR, Fold-Change, and p-value columns.

        Values are calculated based on changes between group_a and group_b.

        Parameters
        ----------
        psms : pandas.DataFrame
        group_a : list of str
        group_b : list of str
        normalize : bool, optional

        Returns
        -------
        pandas.DataFrame
        """
        groups = list(self.groups.values())

        if len(groups) == 2:
            self.psms["SNR"] = pd.Series(
                [
                    _snr(row[groups[0]], row[groups[1]])
                    for index, row in self.psms.iterrows()
                ]
            )

            self.psms["Fold Change"] = pd.Series(
                self.psms[groups[0]].mean(axis=1) /
                self.psms[groups[1]].mean(axis=1)
            )
            self.psms["p-value"] = pd.Series(
                ttest_ind(
                    self.psms[groups[0]],
                    self.psms[groups[1]],
                    axis=1,
                )[1]
            )

def merge_data(
    data_sets, name=None,
    merge_duplicates=True, merge_subsets=True,
):
    """
    Merge a list of data sets together.

    Parameters
    ----------
    data_sets : list of pyproteome.DataSet
    name : str, optional
    merge_duplicates : bool, optional
    merge_subsets : bool, optional

    Returns
    -------
    pyproteome.DataSet
    """
    assert len(data_sets) > 0

    if any(not isinstance(data, DataSet) for data in data_sets):
        raise TypeError(
            "Incompatible types: {}".format(
                [type(data) for data in data_sets]
            )
        )

    if any(data.groups != data_sets[0].groups for data in data_sets):
        raise Exception("Incongruent groups between data sets.")

    new = data_sets[0].copy()
    new.psms = pd.concat([data.psms for data in data_sets])

    if merge_duplicates:
        new._merge_duplicates()

    if merge_subsets:
        new._merge_subsequences()

    new.sets = sum(data.sets for data in data_sets)
    new.enrichments = sorted(
        set(
            enrichment
            for data in data_sets
            for enrichment in data.enrichments
        )
    )
    new.tissues = sorted(
        set(
            tissue
            for data in data_sets
            for tissue in data.tissues
        )
    )

    if new.groups:
        new._update_snr_change()

    if name:
        new.name = name

    return new

def _log_cum_fold_change(vals):
    """
    Calculate the cumulative fold change (in base-2) of values.

    Fold-change is normalized to the first element in the array.

    Parameters
    ----------
    vals : numpy.array

    Returns
    -------
    float
    """
    return sum(
        abs(np.log2(i / (vals[0])))
        for i in vals[1:]
        if i != 0
    )

def _snr(data1, data2):
    """
    Calculate the signal-to-noise ratio between two groups of data.

    Parameters
    ----------
    data1 : numpy.array of float
    data2 : numpy.array of float

    Returns
    -------
    float
    """
    return (data1.mean() - data2.mean()) \
        / (data1.std(ddof=1) + data2.std(ddof=1))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
The core package contains the core components for writing a pyqode based
application. It is the "de facto" requirement for any pyqode extension.

It contains the base classes for both the backend and the frontend and provides
a series of modes and panels that might be useful for any kind of code editor
widget, i.e. pyqode.core is a generic code editor widget.

pyQode is a client-server API. The client side API is called frontend while the
server side API is called backend.

Frontend is implemented in python3 but the backend must supports both python2
and python3 in order to be able to run with any python interpreter
(this is needed to run tools such as pep8 and jedi on the backend and still
support python2 syntax).

pyqode is made up of the following top level packages/modules:

    - frontend: any class/function used to implement the client
                side GUI application
    - backend: any class/function used to implement the server side application

"""

__version__ = '2.0.0-rc1'
<commit_msg>Finish 2.0<commit_after># -*- coding: utf-8 -*-
"""
The core package contains the core components for writing a pyqode based
application. It is the "de facto" requirement for any pyqode extension.

It contains the base classes for both the backend and the frontend and provides
a series of modes and panels that might be useful for any kind of code editor
widget, i.e. pyqode.core is a generic code editor widget.

pyQode is a client-server API. The client side API is called frontend while the
server side API is called backend.

Frontend is implemented in python3 but the backend must supports both python2
and python3 in order to be able to run with any python interpreter
(this is needed to run tools such as pep8 and jedi on the backend and still
support python2 syntax).

pyqode is made up of the following top level packages/modules:

    - frontend: any class/function used to implement the client
                side GUI application
    - backend: any class/function used to implement the server side application

"""

__version__ = '2.0.0'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import functools
import warnings
from collections import namedtuple

from py_zipkin import Encoding
from py_zipkin import Kind
from py_zipkin.exception import ZipkinError
from py_zipkin.storage import get_default_tracer
from py_zipkin.transport import BaseTransportHandler

from pyramid_zipkin.request_helper import create_zipkin_attr
from pyramid_zipkin.request_helper import get_binary_annotations
from pyramid_zipkin.request_helper import should_not_sample_path
from pyramid_zipkin.request_helper import should_not_sample_route

def _getattr_path(obj, path):
    """
    getattr for a dot separated path

    If an AttributeError is raised, it will return None.
    """
    if not path:
        return None

    for attr in path.split('.'):
        obj = getattr(obj, attr, None)
    return obj

_ZipkinSettings = namedtuple('ZipkinSettings', [
    'zipkin_attrs',
    'transport_handler',
    'service_name',
    'span_name',
    'add_logging_annotation',
    'report_root_timestamp',
    'host',
    'port',
    'context_stack',
    'firehose_handler',
    'post_handler_hook',
    'max_span_batch_size',
    'use_pattern_as_span_name',
    'encoding',
])

def _get_settings_from_request(request):
    """Extracts Zipkin attributes and configuration from request attributes.
    See the `zipkin_span` context in py-zipkin for more detaied information on
    all the settings.

    Here are the supported Pyramid registry settings:

    zipkin.create_zipkin_attr: allows the service to override the creation of
        Zipkin attributes. For example, if you want to deterministically
        calculate trace ID from some service-specific attributes.
    zipkin.transport_handler: how py-zipkin will log the spans it generates.
    zipkin.stream_name: an additional parameter to be used as the first arg
        to the transport_handler function. A good example is a Kafka topic.
    zipkin.add_logging_annotation: if true, the outermost span in this service
        will have an annotation set when py-zipkin begins its logging.
    zipkin.report_root_timestamp: if true, the outermost span in this service
        will set its timestamp and duration attributes. Use this only if this
        service is not going to have a corresponding client span. See
        https://github.com/Yelp/pyramid_zipkin/issues/68
    zipkin.firehose_handler: [EXPERIMENTAL] this enables "firehose tracing",
        which will log 100% of the spans to this handler, regardless of
        sampling decision. This is experimental and may change or be removed
        at any time without warning.
    zipkin.use_pattern_as_span_name: if true, we'll use the pyramid route pattern
        as span name. If false (default) we'll keep using the raw url path.
    """
    settings = request.registry.settings

    # Creates zipkin_attrs and attaches a zipkin_trace_id attr to the request
    if 'zipkin.create_zipkin_attr' in settings:
        zipkin_attrs = settings['zipkin.create_zipkin_attr'](request)
    else:
        zipkin_attrs = create_zipkin_attr(request)

    if 'zipkin.transport_handler' in settings:
        transport_handler = settings['zipkin.transport_handler']
        if not isinstance(transport_handler, BaseTransportHandler):
            warnings.warn(
                'Using a function as transport_handler is deprecated. '
                'Please extend py_zipkin.transport.BaseTransportHandler',
                DeprecationWarning,
            )
            stream_name = settings.get('zipkin.stream_name', 'zipkin')
            transport_handler = functools.partial(transport_handler, stream_name)
    else:
        raise ZipkinError(
            "`zipkin.transport_handler` is a required config property, which"
            " is missing. Have a look at py_zipkin's docs for how to implement"
            " it: https://github.com/Yelp/py_zipkin#transport"
        )

    context_stack = _getattr_path(request, settings.get('zipkin.request_context'))

    service_name = settings.get('service_name', 'unknown')
    span_name = '{0} {1}'.format(request.method, request.path)
    add_logging_annotation = settings.get(
        'zipkin.add_logging_annotation',
        False,
    )

    # If the incoming request doesn't have Zipkin headers, this request is
    # assumed to be the root span of a trace. There's also a configuration
    # override to allow services to write their own logic for reporting
    # timestamp/duration.
    if 'zipkin.report_root_timestamp' in settings:
        report_root_timestamp = settings['zipkin.report_root_timestamp']
    else:
        report_root_timestamp = 'X-B3-TraceId' not in request.headers
    zipkin_host = settings.get('zipkin.host')
    zipkin_port = settings.get('zipkin.port', request.server_port)
    firehose_handler = settings.get('zipkin.firehose_handler')
    post_handler_hook = settings.get('zipkin.post_handler_hook')
    max_span_batch_size = settings.get('zipkin.max_span_batch_size')
    use_pattern_as_span_name = bool(
        settings.get('zipkin.use_pattern_as_span_name', False),
    )
    encoding = settings.get('zipkin.encoding', Encoding.V1_THRIFT)
    return _ZipkinSettings(
        zipkin_attrs,
        transport_handler,
        service_name,
        span_name,
        add_logging_annotation,
        report_root_timestamp,
        zipkin_host,
        zipkin_port,
        context_stack,
        firehose_handler,
        post_handler_hook,
        max_span_batch_size,
        use_pattern_as_span_name,
        encoding=encoding,
    )

def zipkin_tween(handler, registry):
    """
    Factory for pyramid tween to handle zipkin server logging. Note that even
    if the request isn't sampled, Zipkin attributes are generated and pushed
    into threadlocal storage, so `create_http_headers_for_new_span` and
    `zipkin_span` will have access to the proper Zipkin state.

    Consumes custom create_zipkin_attr function if one is set in the pyramid
    registry.

    :param handler: pyramid request handler
    :param registry: pyramid app registry

    :returns: pyramid tween
    """
    def tween(request):
        zipkin_settings = _get_settings_from_request(request)
        tracer = get_default_tracer()

        tween_kwargs = dict(
            service_name=zipkin_settings.service_name,
            span_name=zipkin_settings.span_name,
            zipkin_attrs=zipkin_settings.zipkin_attrs,
            transport_handler=zipkin_settings.transport_handler,
            host=zipkin_settings.host,
            port=zipkin_settings.port,
            add_logging_annotation=zipkin_settings.add_logging_annotation,
            report_root_timestamp=zipkin_settings.report_root_timestamp,
            context_stack=zipkin_settings.context_stack,
            max_span_batch_size=zipkin_settings.max_span_batch_size,
            encoding=zipkin_settings.encoding,
            kind=Kind.SERVER,
        )

        # Only set the firehose_handler if it's defined and only if the current
        # request is not blacklisted. This prevents py_zipkin from emitting
        # firehose spans for blacklisted paths like /status
        if zipkin_settings.firehose_handler is not None and \
                not should_not_sample_path(request) and \
                not should_not_sample_route(request):
            tween_kwargs['firehose_handler'] = zipkin_settings.firehose_handler

        with tracer.zipkin_span(**tween_kwargs) as zipkin_context:
            response = handler(request)
            if zipkin_settings.use_pattern_as_span_name and request.matched_route:
                zipkin_context.override_span_name('{} {}'.format(
                    request.method,
                    request.matched_route.pattern,
                ))
            zipkin_context.update_binary_annotations(
                get_binary_annotations(request, response),
            )

            if zipkin_settings.post_handler_hook:
                zipkin_settings.post_handler_hook(request, response, zipkin_context)

            return response

    return tween
<commit_msg>fix flake8 issue<commit_after># -*- coding: utf-8 -*-
import functools
import warnings
from collections import namedtuple

from py_zipkin import Encoding
from py_zipkin import Kind
from py_zipkin.exception import ZipkinError
from py_zipkin.storage import get_default_tracer
from py_zipkin.transport import BaseTransportHandler

from pyramid_zipkin.request_helper import create_zipkin_attr
from pyramid_zipkin.request_helper import get_binary_annotations
from pyramid_zipkin.request_helper import should_not_sample_path
from pyramid_zipkin.request_helper import should_not_sample_route

def _getattr_path(obj, path):
    """
    getattr for a dot separated path

    If an AttributeError is raised, it will return None.
    """
    if not path:
        return None

    for attr in path.split('.'):
        obj = getattr(obj, attr, None)
    return obj

_ZipkinSettings = namedtuple('ZipkinSettings', [
    'zipkin_attrs',
    'transport_handler',
    'service_name',
    'span_name',
    'add_logging_annotation',
    'report_root_timestamp',
    'host',
    'port',
    'context_stack',
    'firehose_handler',
    'post_handler_hook',
    'max_span_batch_size',
    'use_pattern_as_span_name',
    'encoding',
])

def _get_settings_from_request(request):
    """Extracts Zipkin attributes and configuration from request attributes.
    See the `zipkin_span` context in py-zipkin for more detaied information on
    all the settings.

    Here are the supported Pyramid registry settings:

    zipkin.create_zipkin_attr: allows the service to override the creation of
        Zipkin attributes. For example, if you want to deterministically
        calculate trace ID from some service-specific attributes.
    zipkin.transport_handler: how py-zipkin will log the spans it generates.
    zipkin.stream_name: an additional parameter to be used as the first arg
        to the transport_handler function. A good example is a Kafka topic.
    zipkin.add_logging_annotation: if true, the outermost span in this service
        will have an annotation set when py-zipkin begins its logging.
    zipkin.report_root_timestamp: if true, the outermost span in this service
        will set its timestamp and duration attributes. Use this only if this
        service is not going to have a corresponding client span. See
        https://github.com/Yelp/pyramid_zipkin/issues/68
    zipkin.firehose_handler: [EXPERIMENTAL] this enables "firehose tracing",
        which will log 100% of the spans to this handler, regardless of
        sampling decision. This is experimental and may change or be removed
        at any time without warning.
    zipkin.use_pattern_as_span_name: if true, we'll use the pyramid route pattern
        as span name. If false (default) we'll keep using the raw url path.
    """
    settings = request.registry.settings

    # Creates zipkin_attrs and attaches a zipkin_trace_id attr to the request
    if 'zipkin.create_zipkin_attr' in settings:
        zipkin_attrs = settings['zipkin.create_zipkin_attr'](request)
    else:
        zipkin_attrs = create_zipkin_attr(request)

    if 'zipkin.transport_handler' in settings:
        transport_handler = settings['zipkin.transport_handler']
        if not isinstance(transport_handler, BaseTransportHandler):
            warnings.warn(
                'Using a function as transport_handler is deprecated. '
                'Please extend py_zipkin.transport.BaseTransportHandler',
                DeprecationWarning,
            )
            stream_name = settings.get('zipkin.stream_name', 'zipkin')
            transport_handler = functools.partial(transport_handler, stream_name)
    else:
        raise ZipkinError(
            "`zipkin.transport_handler` is a required config property, which"
            " is missing. Have a look at py_zipkin's docs for how to implement"
            " it: https://github.com/Yelp/py_zipkin#transport"
        )

    context_stack = _getattr_path(request, settings.get('zipkin.request_context'))

    service_name = settings.get('service_name', 'unknown')
    span_name = '{0} {1}'.format(request.method, request.path)
    add_logging_annotation = settings.get(
        'zipkin.add_logging_annotation',
        False,
    )

    # If the incoming request doesn't have Zipkin headers, this request is
    # assumed to be the root span of a trace. There's also a configuration
    # override to allow services to write their own logic for reporting
    # timestamp/duration.
    if 'zipkin.report_root_timestamp' in settings:
        report_root_timestamp = settings['zipkin.report_root_timestamp']
    else:
        report_root_timestamp = 'X-B3-TraceId' not in request.headers
    zipkin_host = settings.get('zipkin.host')
    zipkin_port = settings.get('zipkin.port', request.server_port)
    firehose_handler = settings.get('zipkin.firehose_handler')
    post_handler_hook = settings.get('zipkin.post_handler_hook')
    max_span_batch_size = settings.get('zipkin.max_span_batch_size')
    use_pattern_as_span_name = bool(
        settings.get('zipkin.use_pattern_as_span_name', False),
    )
    encoding = settings.get('zipkin.encoding', Encoding.V1_THRIFT)
    return _ZipkinSettings(
        zipkin_attrs,
        transport_handler,
        service_name,
        span_name,
        add_logging_annotation,
        report_root_timestamp,
        zipkin_host,
        zipkin_port,
        context_stack,
        firehose_handler,
        post_handler_hook,
        max_span_batch_size,
        use_pattern_as_span_name,
        encoding=encoding,
    )

def zipkin_tween(handler, registry):
    """
    Factory for pyramid tween to handle zipkin server logging. Note that even
    if the request isn't sampled, Zipkin attributes are generated and pushed
    into threadlocal storage, so `create_http_headers_for_new_span` and
    `zipkin_span` will have access to the proper Zipkin state.

    Consumes custom create_zipkin_attr function if one is set in the pyramid
    registry.

    :param handler: pyramid request handler
    :param registry: pyramid app registry

    :returns: pyramid tween
    """
    def tween(request):
        zipkin_settings = _get_settings_from_request(request)
        tracer = get_default_tracer()

        tween_kwargs = dict(
            service_name=zipkin_settings.service_name,
            span_name=zipkin_settings.span_name,
            zipkin_attrs=zipkin_settings.zipkin_attrs,
            transport_handler=zipkin_settings.transport_handler,
            host=zipkin_settings.host,
            port=zipkin_settings.port,
            add_logging_annotation=zipkin_settings.add_logging_annotation,
            report_root_timestamp=zipkin_settings.report_root_timestamp,
            context_stack=zipkin_settings.context_stack,
            max_span_batch_size=zipkin_settings.max_span_batch_size,
            encoding=zipkin_settings.encoding,
            kind=Kind.SERVER,
        )

        # Only set the firehose_handler if it's defined and only if the current
        # request is not blacklisted. This prevents py_zipkin from emitting
        # firehose spans for blacklisted paths like /status
        if zipkin_settings.firehose_handler is not None and \
                not should_not_sample_path(request) and \
                not should_not_sample_route(request):
            tween_kwargs['firehose_handler'] = zipkin_settings.firehose_handler

        with tracer.zipkin_span(**tween_kwargs) as zipkin_context:
            response = handler(request)
            if zipkin_settings.use_pattern_as_span_name and request.matched_route:
                zipkin_context.override_span_name('{} {}'.format(
                    request.method,
                    request.matched_route.pattern,
                ))
            zipkin_context.update_binary_annotations(
                get_binary_annotations(request, response),
            )

            if zipkin_settings.post_handler_hook:
                zipkin_settings.post_handler_hook(
                    request,
                    response,
                    zipkin_context
                )

            return response

    return tween
<|endoftext|>


<commit_before>#   This Python module is part of the PyRate software package.
#
#   Copyright 2020 Geoscience Australia
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
from configparser import ConfigParser
from pathlib import Path, PurePath
import re
from pyrate.constants import NO_OF_PARALLEL_PROCESSES
from pyrate.default_parameters import PYRATE_DEFAULT_CONFIGRATION
from pyrate.core.user_experience import break_number_into_factors

def set_parameter_value(data_type, input_value, default_value, required, input_name):
    if len(input_value) < 1:
        input_value = None
        if required:
            raise ValueError("A required parameter is missing value in input configuration file: " + str(input_name))

    if input_value is not None:
        if str(data_type) in "path":
            return Path(input_value)
        return data_type(input_value)
    return default_value

def validate_parameter_value(input_name, input_value, min_value=None, max_value=None, possible_values=None):
    if isinstance(input_value, PurePath):
        if input_name in "outdir":
            input_value = input_value.parents[0]
        if not Path.exists(input_value):
            raise ValueError("Given path: " + str(input_value) + " dose not exist.")
    if input_value is not None:
        if min_value is not None:
            if input_value < min_value:
                raise ValueError(
                    "Invalid value for " + str(input_name) + " supplied: " + str(input_value) + ". Please provided a valid value greater than " + str(min_value) + ".")
    if input_value is not None:
        if max_value is not None:
            if input_value > max_value:
                raise ValueError(
                    "Invalid value for " + str(input_name) + " supplied: " + str(input_value) + ". Please provided a valid value less than " + str(max_value) + ".")

    if possible_values is not None:
        if input_value not in possible_values:
            raise ValueError(
                "Invalid value for " + str(input_name) + " supplied: " + str(input_value) + ". Please provided a valid value from with in: " + str(possible_values) + ".")
    return True

def validate_file_list_values(dir, fileList, no_of_epochs):
    if dir is None:
        raise ValueError("No value supplied for input directory: " + str(dir))
    if fileList is None:
        raise ValueError("No value supplied for input file list: " + str(fileList))

    for path_str in fileList.read_text().split('\n'):
        # ignore empty lines in file
        if len(path_str) > 1:
            if not Path(dir).joinpath(path_str).exists():
                raise ValueError("Give file name: " + str(path_str) + " dose not exist at: " + str(dir))
            else:
                matches = re.findall(r"(\d{8})", str(dir / path_str))
                if len(matches) < no_of_epochs:
                    raise ValueError("For the given file name: " + str(path_str) + " in: " + str(
                        dir) + " the number of epochs in file names are less the required number:" + str(
                        no_of_epochs))

class MultiplePaths:
    def __init__(self, baseDir, baseName, ifglksx=0, ifgcropopt=0):
        if ".tif" in baseName:
            self.unwrapped_path = None
            self.converted_path = Path(baseDir).joinpath(baseName).as_posix()
            self.sampled_path = self.converted_path.split(".tif")[0] + "_" + str(ifglksx) + "rlks_" + str(ifgcropopt) + "cr.tif"
        else:
            self.unwrapped_path = Path(baseDir).joinpath(baseName).as_posix()
            baseName = baseName.split(".")[0] + "_" + baseName.split(".")[1] + ".tif"

            self.converted_path = Path(baseDir).joinpath(baseName).as_posix()
            self.sampled_path = self.converted_path.split(".tif")[0] + "_" + str(ifglksx) + "rlks_" + str(ifgcropopt) + "cr.tif"

    def __str__(self):
        return"""
            unwrapped_path = """ + self.unwrapped_path+"""
            converted_path = """ + self.converted_path+"""
            sampled_path = """ + self.sampled_path+"""
            """

class Configuration:

    def __init__(self, config_file_path):

        parser = ConfigParser()
        parser.optionxform = str
        # mimic header to fulfil the requirement for configparser
        with open(config_file_path) as stream:
            parser.read_string("[root]\n" + stream.read())

        for key, value in parser._sections["root"].items():
            self.__dict__[key] = value

        # Validate required parameters exist.
        if not set(PYRATE_DEFAULT_CONFIGRATION).issubset(self.__dict__):
            raise ValueError("Required configuration parameters: " + str(
                set(PYRATE_DEFAULT_CONFIGRATION).difference(self.__dict__)) + " are missing from input config file.")

        # handle control parameters
        for parameter_name in PYRATE_DEFAULT_CONFIGRATION:
            self.__dict__[parameter_name] = set_parameter_value(PYRATE_DEFAULT_CONFIGRATION[parameter_name]["DataType"],
                                                                self.__dict__[parameter_name],
                                                                PYRATE_DEFAULT_CONFIGRATION[parameter_name]["DefaultValue"],
                                                                PYRATE_DEFAULT_CONFIGRATION[parameter_name]["Required"],
                                                                parameter_name)
            validate_parameter_value(parameter_name, self.__dict__[parameter_name],
                                     PYRATE_DEFAULT_CONFIGRATION[parameter_name]["MinValue"],
                                     PYRATE_DEFAULT_CONFIGRATION[parameter_name]["MaxValue"],
                                     PYRATE_DEFAULT_CONFIGRATION[parameter_name]["PossibleValues"])

        # bespoke parameter validation
        if self.refchipsize % 2 != 1:
            raise ValueError("Configuration parameters refchipsize must be odd: " + str(self.refchipsize))

        self.rows, self.cols = [int(no) for no in break_number_into_factors(NO_OF_PARALLEL_PROCESSES)]

        # create a temporary directory
        self.tmpdir = self.outdir / "tmpdir"
        self.tmpdir.mkdir(parents=True, exist_ok=True)
        self.tmpdir = str(self.tmpdir)

        # var no longer used
        self.APS_ELEVATION_EXT = None
        self.APS_INCIDENCE_EXT = None
        self.apscorrect = 0
        self.apsmethod = 0
        self.elevationmap = None
        self.incidencemap = None

        # define parallel processes that will run
        self.NUMEXPR_MAX_THREADS = str(NO_OF_PARALLEL_PROCESSES)

        # Validate file names supplied in list exist and contain correct epochs in file names

        if self.processor != 0:   # not roipac
            validate_file_list_values(self.obsdir, self.ifgfilelist, 2)
            validate_file_list_values(self.slcFileDir, self.slcfilelist, 1)

        self.coherence_file_paths = None
        if self.cohfiledir is not None and self.cohfilelist is not None:
            if self.processor != 0:  # not roipac
                validate_file_list_values(self.cohfiledir, self.cohfilelist, 1)
            self.coherence_file_paths = []
            for path_str in self.cohfilelist.read_text().split('\n'):
                # ignore empty lines in file
                if len(path_str) > 1:
                    self.coherence_file_paths.append(MultiplePaths(self.cohfiledir, path_str, self.ifglksx, self.ifgcropopt))

        self.header_file_paths = []
        for path_str in self.slcfilelist.read_text().split('\n'):
            # ignore empty lines in file
            if len(path_str) > 1:
                self.header_file_paths.append(MultiplePaths(self.slcFileDir, path_str, self.ifglksx, self.ifgcropopt))

        self.__get_interferrograms_files()

        self.dem_file = MultiplePaths(self.demfile.parents[0], self.demfile.name, self.ifglksx, self.ifgcropopt)

        # backward compatibility for string paths
        for key in self.__dict__:
            if isinstance(self.__dict__[key], PurePath):
                self.__dict__[key] = str(self.__dict__[key])

    def __get_interferrograms_files(self):
        self.interferogram_files = []
        for path_str in self.ifgfilelist.read_text().split('\n'):
            # ignore empty lines in file
            if len(path_str) > 1:
                self.interferogram_files.append(MultiplePaths(self.outdir, path_str, self.ifglksx, self.ifgcropopt))
<commit_msg>add pragmas<commit_after>#   This Python module is part of the PyRate software package.
#
#   Copyright 2020 Geoscience Australia
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
from configparser import ConfigParser
from pathlib import Path, PurePath
import re
from pyrate.constants import NO_OF_PARALLEL_PROCESSES
from pyrate.default_parameters import PYRATE_DEFAULT_CONFIGRATION
from pyrate.core.user_experience import break_number_into_factors

def set_parameter_value(data_type, input_value, default_value, required, input_name):
    if len(input_value) < 1:
        input_value = None
        if required:  # pragma: no cover
            raise ValueError("A required parameter is missing value in input configuration file: " + str(input_name))

    if input_value is not None:
        if str(data_type) in "path":
            return Path(input_value)
        return data_type(input_value)
    return default_value

def validate_parameter_value(input_name, input_value, min_value=None, max_value=None, possible_values=None):
    if isinstance(input_value, PurePath):
        if input_name in "outdir":
            input_value = input_value.parents[0]
        if not Path.exists(input_value):  # pragma: no cover
            raise ValueError("Given path: " + str(input_value) + " dose not exist.")
    if input_value is not None:
        if min_value is not None:
            if input_value < min_value:  # pragma: no cover
                raise ValueError(
                    "Invalid value for " + str(input_name) + " supplied: " + str(input_value) + ". Please provided a valid value greater than " + str(min_value) + ".")
    if input_value is not None:
        if max_value is not None:
            if input_value > max_value:  # pragma: no cover
                raise ValueError(
                    "Invalid value for " + str(input_name) + " supplied: " + str(input_value) + ". Please provided a valid value less than " + str(max_value) + ".")

    if possible_values is not None:
        if input_value not in possible_values:  # pragma: no cover
            raise ValueError(
                "Invalid value for " + str(input_name) + " supplied: " + str(input_value) + ". Please provided a valid value from with in: " + str(possible_values) + ".")
    return True

def validate_file_list_values(dir, fileList, no_of_epochs):
    if dir is None:  # pragma: no cover
        raise ValueError("No value supplied for input directory: " + str(dir))
    if fileList is None:  # pragma: no cover
        raise ValueError("No value supplied for input file list: " + str(fileList))

    for path_str in fileList.read_text().split('\n'):
        # ignore empty lines in file
        if len(path_str) > 1:
            if not Path(dir).joinpath(path_str).exists():  # pragma: no cover
                raise ValueError("Give file name: " + str(path_str) + " dose not exist at: " + str(dir))
            else:
                matches = re.findall(r"(\d{8})", str(dir / path_str))
                if len(matches) < no_of_epochs:  # pragma: no cover
                    raise ValueError("For the given file name: " + str(path_str) + " in: " + str(
                        dir) + " the number of epochs in file names are less the required number:" + str(
                        no_of_epochs))

class MultiplePaths:
    def __init__(self, baseDir, baseName, ifglksx=0, ifgcropopt=0):
        if ".tif" in baseName:
            self.unwrapped_path = None
            self.converted_path = Path(baseDir).joinpath(baseName).as_posix()
            self.sampled_path = self.converted_path.split(".tif")[0] + "_" + str(ifglksx) + "rlks_" + str(ifgcropopt) + "cr.tif"
        else:
            self.unwrapped_path = Path(baseDir).joinpath(baseName).as_posix()
            baseName = baseName.split(".")[0] + "_" + baseName.split(".")[1] + ".tif"

            self.converted_path = Path(baseDir).joinpath(baseName).as_posix()
            self.sampled_path = self.converted_path.split(".tif")[0] + "_" + str(ifglksx) + "rlks_" + str(ifgcropopt) + "cr.tif"

    def __str__(self):  # pragma: no cover
        return"""
            unwrapped_path = """ + self.unwrapped_path+"""
            converted_path = """ + self.converted_path+"""
            sampled_path = """ + self.sampled_path+"""
            """

class Configuration:

    def __init__(self, config_file_path):

        parser = ConfigParser()
        parser.optionxform = str
        # mimic header to fulfil the requirement for configparser
        with open(config_file_path) as stream:
            parser.read_string("[root]\n" + stream.read())

        for key, value in parser._sections["root"].items():
            self.__dict__[key] = value

        # Validate required parameters exist.
        if not set(PYRATE_DEFAULT_CONFIGRATION).issubset(self.__dict__):  # pragma: no cover
            raise ValueError("Required configuration parameters: " + str(
                set(PYRATE_DEFAULT_CONFIGRATION).difference(self.__dict__)) + " are missing from input config file.")

        # handle control parameters
        for parameter_name in PYRATE_DEFAULT_CONFIGRATION:
            self.__dict__[parameter_name] = set_parameter_value(PYRATE_DEFAULT_CONFIGRATION[parameter_name]["DataType"],
                                                                self.__dict__[parameter_name],
                                                                PYRATE_DEFAULT_CONFIGRATION[parameter_name]["DefaultValue"],
                                                                PYRATE_DEFAULT_CONFIGRATION[parameter_name]["Required"],
                                                                parameter_name)
            validate_parameter_value(parameter_name, self.__dict__[parameter_name],
                                     PYRATE_DEFAULT_CONFIGRATION[parameter_name]["MinValue"],
                                     PYRATE_DEFAULT_CONFIGRATION[parameter_name]["MaxValue"],
                                     PYRATE_DEFAULT_CONFIGRATION[parameter_name]["PossibleValues"])

        # bespoke parameter validation

        if self.refchipsize % 2 != 1:  # pragma: no cover
            raise ValueError("Configuration parameters refchipsize must be odd: " + str(self.refchipsize))

        self.rows, self.cols = [int(no) for no in break_number_into_factors(NO_OF_PARALLEL_PROCESSES)]

        # create a temporary directory
        self.tmpdir = self.outdir / "tmpdir"
        self.tmpdir.mkdir(parents=True, exist_ok=True)
        self.tmpdir = str(self.tmpdir)

        # var no longer used
        self.APS_ELEVATION_EXT = None
        self.APS_INCIDENCE_EXT = None
        self.apscorrect = 0
        self.apsmethod = 0
        self.elevationmap = None
        self.incidencemap = None

        # define parallel processes that will run
        self.NUMEXPR_MAX_THREADS = str(NO_OF_PARALLEL_PROCESSES)

        # Validate file names supplied in list exist and contain correct epochs in file names

        if self.processor != 0:   # not roipac
            validate_file_list_values(self.obsdir, self.ifgfilelist, 2)
            validate_file_list_values(self.slcFileDir, self.slcfilelist, 1)

        self.coherence_file_paths = None
        if self.cohfiledir is not None and self.cohfilelist is not None:
            # FIXME: tests don't reach here due to configs not having cohfilelist
            if self.processor != 0:  # not roipac
                validate_file_list_values(self.cohfiledir, self.cohfilelist, 1)
            self.coherence_file_paths = []
            for path_str in self.cohfilelist.read_text().split('\n'):
                # ignore empty lines in file
                if len(path_str) > 1:
                    self.coherence_file_paths.append(MultiplePaths(self.cohfiledir, path_str, self.ifglksx, self.ifgcropopt))

        self.header_file_paths = []
        for path_str in self.slcfilelist.read_text().split('\n'):
            # ignore empty lines in file
            if len(path_str) > 1:
                self.header_file_paths.append(MultiplePaths(self.slcFileDir, path_str, self.ifglksx, self.ifgcropopt))

        self.__get_interferrograms_files()

        self.dem_file = MultiplePaths(self.demfile.parents[0], self.demfile.name, self.ifglksx, self.ifgcropopt)

        # backward compatibility for string paths
        for key in self.__dict__:
            if isinstance(self.__dict__[key], PurePath):
                self.__dict__[key] = str(self.__dict__[key])

    def __get_interferrograms_files(self):
        self.interferogram_files = []
        for path_str in self.ifgfilelist.read_text().split('\n'):
            # ignore empty lines in file
            if len(path_str) > 1:
                self.interferogram_files.append(MultiplePaths(self.outdir, path_str, self.ifglksx, self.ifgcropopt))
<|endoftext|>


<commit_before>#   This Python module is part of the PyRate software package.
#
#   Copyright 2020 Geoscience Australia
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
from configparser import ConfigParser
from pathlib import Path, PurePath
import re
from pyrate.constants import NO_OF_PARALLEL_PROCESSES
from pyrate.default_parameters import PYRATE_DEFAULT_CONFIGRATION
from pyrate.core.user_experience import break_number_into_factors

def set_parameter_value(data_type, input_value, default_value, required, input_name):
    if len(input_value) < 1:
        input_value = None
        if required:
            raise ValueError("A required parameter is missing value in input configuration file: " + str(input_name))

    if input_value is not None:
        if str(data_type) in "path":
            return Path(input_value)
        return data_type(input_value)
    return default_value

def validate_parameter_value(input_name, input_value, min_value=None, max_value=None, possible_values=None):
    if isinstance(input_value, PurePath):
        if input_name in "outdir":
            input_value = input_value.parents[0]
        if not Path.exists(input_value):
            raise ValueError("Given path: " + str(input_value) + " dose not exist.")
    if input_value is not None:
        if min_value is not None:
            if input_value < min_value:
                raise ValueError(
                    "Invalid value for " + str(input_name) + " supplied: " + str(input_value) + ". Please provided a valid value greater than " + str(min_value) + ".")
    if input_value is not None:
        if max_value is not None:
            if input_value > max_value:
                raise ValueError(
                    "Invalid value for " + str(input_name) + " supplied: " + str(input_value) + ". Please provided a valid value less than " + str(max_value) + ".")

    if possible_values is not None:
        if input_value not in possible_values:
            raise ValueError(
                "Invalid value for " + str(input_name) + " supplied: " + str(input_value) + ". Please provided a valid value from with in: " + str(possible_values) + ".")
    return True

def validate_file_list_values(dir, fileList, no_of_epochs):
    if dir is None:
        raise ValueError("No value supplied for input directory: " + str(dir))
    if fileList is None:
        raise ValueError("No value supplied for input file list: " + str(fileList))

    for path_str in fileList.read_text().split('\n'):
        # ignore empty lines in file
        if len(path_str) > 1:
            if not Path(dir).joinpath(path_str).exists():
                raise ValueError("Give file name: " + str(path_str) + " dose not exist at: " + str(dir))
            else:
                matches = re.findall("(\d{8})", str(dir / path_str))
                if len(matches) < no_of_epochs:
                    raise ValueError("For the given file name: " + str(path_str) + " in: " + str(
                        dir) + " the number of epochs in file names are less the required number:" + str(
                        no_of_epochs))

class MultiplePaths:
    def __init__(self, baseDir, baseName, ifglksx=0, ifgcropopt=0):
        if ".tif" in baseName:
            self.unwrapped_path = None
            self.converted_path = Path(baseDir).joinpath(baseName).as_posix()
            self.sampled_path = self.converted_path.split(".tif")[0] + "_" + str(ifglksx) + "rlks_" + str(ifgcropopt) + "cr.tif"
        else:
            self.unwrapped_path = Path(baseDir).joinpath(baseName).as_posix()
            baseName = baseName.split(".")[0] + "_" + baseName.split(".")[1] + ".tif"

            self.converted_path = Path(baseDir).joinpath(baseName).as_posix()
            self.sampled_path = self.converted_path.split(".tif")[0] + "_" + str(ifglksx) + "rlks_" + str(ifgcropopt) + "cr.tif"

    def __str__(self):
        return"""
            unwrapped_path = """ + self.unwrapped_path+"""
            converted_path = """ + self.converted_path+"""
            sampled_path = """ + self.sampled_path+"""
            """

class Configuration:

    def __init__(self, config_file_path):

        parser = ConfigParser()
        parser.optionxform = str
        # mimic header to fulfil the requirement for configparser
        with open(config_file_path) as stream:
            parser.read_string("[root]\n" + stream.read())

        for key, value in parser._sections["root"].items():
            self.__dict__[key] = value

        # Validate required parameters exist.
        if not set(PYRATE_DEFAULT_CONFIGRATION).issubset(self.__dict__):
            raise ValueError("Required configuration parameters: " + str(
                set(PYRATE_DEFAULT_CONFIGRATION).difference(self.__dict__)) + " are missing from input config file.")

        # handle control parameters
        for parameter_name in PYRATE_DEFAULT_CONFIGRATION:
            self.__dict__[parameter_name] = set_parameter_value(PYRATE_DEFAULT_CONFIGRATION[parameter_name]["DataType"],
                                                                self.__dict__[parameter_name],
                                                                PYRATE_DEFAULT_CONFIGRATION[parameter_name]["DefaultValue"],
                                                                PYRATE_DEFAULT_CONFIGRATION[parameter_name]["Required"],
                                                                parameter_name)
            validate_parameter_value(parameter_name, self.__dict__[parameter_name],
                                     PYRATE_DEFAULT_CONFIGRATION[parameter_name]["MinValue"],
                                     PYRATE_DEFAULT_CONFIGRATION[parameter_name]["MaxValue"],
                                     PYRATE_DEFAULT_CONFIGRATION[parameter_name]["PossibleValues"])

        # bespoke parameter validation
        if self.refchipsize % 2 != 1:
            raise ValueError("Configuration parameters refchipsize must be odd: " + str(self.refchipsize))

        self.rows, self.cols = [int(no) for no in break_number_into_factors(NO_OF_PARALLEL_PROCESSES)]

        # create a temporary directory
        self.tmpdir = self.outdir / "tmpdir"
        self.tmpdir.mkdir(parents=True, exist_ok=True)
        self.tmpdir = str(self.tmpdir)

        # var no longer used
        self.APS_ELEVATION_EXT = None
        self.APS_INCIDENCE_EXT = None
        self.apscorrect = 0
        self.apsmethod = 0
        self.elevationmap = None
        self.incidencemap = None

        # define parallel processes that will run
        self.NUMEXPR_MAX_THREADS = str(NO_OF_PARALLEL_PROCESSES)

        # Validate file names supplied in list exist and contain correct epochs in file names

        if self.processor != 0:   # not roipac
            validate_file_list_values(self.obsdir, self.ifgfilelist, 2)
            validate_file_list_values(self.slcFileDir, self.slcfilelist, 1)

        self.coherence_file_paths = None
        if self.cohfiledir is not None and self.cohfilelist is not None:
            if self.processor != 0:  # not roipac
                validate_file_list_values(self.cohfiledir, self.cohfilelist, 1)
            self.coherence_file_paths = []
            for path_str in self.cohfilelist.read_text().split('\n'):
                # ignore empty lines in file
                if len(path_str) > 1:
                    self.coherence_file_paths.append(MultiplePaths(self.cohfiledir, path_str, self.ifglksx, self.ifgcropopt))

        self.header_file_paths = []
        for path_str in self.slcfilelist.read_text().split('\n'):
            # ignore empty lines in file
            if len(path_str) > 1:
                self.header_file_paths.append(MultiplePaths(self.slcFileDir, path_str, self.ifglksx, self.ifgcropopt))

        self.__get_interferrograms_files()

        self.dem_file = MultiplePaths(self.demfile.parents[0], self.demfile.name, self.ifglksx, self.ifgcropopt)

        # backward compatibility for string paths
        for key in self.__dict__:
            if isinstance(self.__dict__[key], PurePath):
                self.__dict__[key] = str(self.__dict__[key])

    def __get_interferrograms_files(self):
        self.interferogram_files = []
        for path_str in self.ifgfilelist.read_text().split('\n'):
            # ignore empty lines in file
            if len(path_str) > 1:
                self.interferogram_files.append(MultiplePaths(self.outdir, path_str, self.ifglksx, self.ifgcropopt))

<commit_msg>use raw string in python3<commit_after>#   This Python module is part of the PyRate software package.
#
#   Copyright 2020 Geoscience Australia
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
from configparser import ConfigParser
from pathlib import Path, PurePath
import re
from pyrate.constants import NO_OF_PARALLEL_PROCESSES
from pyrate.default_parameters import PYRATE_DEFAULT_CONFIGRATION
from pyrate.core.user_experience import break_number_into_factors

def set_parameter_value(data_type, input_value, default_value, required, input_name):
    if len(input_value) < 1:
        input_value = None
        if required:
            raise ValueError("A required parameter is missing value in input configuration file: " + str(input_name))

    if input_value is not None:
        if str(data_type) in "path":
            return Path(input_value)
        return data_type(input_value)
    return default_value

def validate_parameter_value(input_name, input_value, min_value=None, max_value=None, possible_values=None):
    if isinstance(input_value, PurePath):
        if input_name in "outdir":
            input_value = input_value.parents[0]
        if not Path.exists(input_value):
            raise ValueError("Given path: " + str(input_value) + " dose not exist.")
    if input_value is not None:
        if min_value is not None:
            if input_value < min_value:
                raise ValueError(
                    "Invalid value for " + str(input_name) + " supplied: " + str(input_value) + ". Please provided a valid value greater than " + str(min_value) + ".")
    if input_value is not None:
        if max_value is not None:
            if input_value > max_value:
                raise ValueError(
                    "Invalid value for " + str(input_name) + " supplied: " + str(input_value) + ". Please provided a valid value less than " + str(max_value) + ".")

    if possible_values is not None:
        if input_value not in possible_values:
            raise ValueError(
                "Invalid value for " + str(input_name) + " supplied: " + str(input_value) + ". Please provided a valid value from with in: " + str(possible_values) + ".")
    return True

def validate_file_list_values(dir, fileList, no_of_epochs):
    if dir is None:
        raise ValueError("No value supplied for input directory: " + str(dir))
    if fileList is None:
        raise ValueError("No value supplied for input file list: " + str(fileList))

    for path_str in fileList.read_text().split('\n'):
        # ignore empty lines in file
        if len(path_str) > 1:
            if not Path(dir).joinpath(path_str).exists():
                raise ValueError("Give file name: " + str(path_str) + " dose not exist at: " + str(dir))
            else:
                matches = re.findall(r"(\d{8})", str(dir / path_str))
                if len(matches) < no_of_epochs:
                    raise ValueError("For the given file name: " + str(path_str) + " in: " + str(
                        dir) + " the number of epochs in file names are less the required number:" + str(
                        no_of_epochs))

class MultiplePaths:
    def __init__(self, baseDir, baseName, ifglksx=0, ifgcropopt=0):
        if ".tif" in baseName:
            self.unwrapped_path = None
            self.converted_path = Path(baseDir).joinpath(baseName).as_posix()
            self.sampled_path = self.converted_path.split(".tif")[0] + "_" + str(ifglksx) + "rlks_" + str(ifgcropopt) + "cr.tif"
        else:
            self.unwrapped_path = Path(baseDir).joinpath(baseName).as_posix()
            baseName = baseName.split(".")[0] + "_" + baseName.split(".")[1] + ".tif"

            self.converted_path = Path(baseDir).joinpath(baseName).as_posix()
            self.sampled_path = self.converted_path.split(".tif")[0] + "_" + str(ifglksx) + "rlks_" + str(ifgcropopt) + "cr.tif"

    def __str__(self):
        return"""
            unwrapped_path = """ + self.unwrapped_path+"""
            converted_path = """ + self.converted_path+"""
            sampled_path = """ + self.sampled_path+"""
            """

class Configuration:

    def __init__(self, config_file_path):

        parser = ConfigParser()
        parser.optionxform = str
        # mimic header to fulfil the requirement for configparser
        with open(config_file_path) as stream:
            parser.read_string("[root]\n" + stream.read())

        for key, value in parser._sections["root"].items():
            self.__dict__[key] = value

        # Validate required parameters exist.
        if not set(PYRATE_DEFAULT_CONFIGRATION).issubset(self.__dict__):
            raise ValueError("Required configuration parameters: " + str(
                set(PYRATE_DEFAULT_CONFIGRATION).difference(self.__dict__)) + " are missing from input config file.")

        # handle control parameters
        for parameter_name in PYRATE_DEFAULT_CONFIGRATION:
            self.__dict__[parameter_name] = set_parameter_value(PYRATE_DEFAULT_CONFIGRATION[parameter_name]["DataType"],
                                                                self.__dict__[parameter_name],
                                                                PYRATE_DEFAULT_CONFIGRATION[parameter_name]["DefaultValue"],
                                                                PYRATE_DEFAULT_CONFIGRATION[parameter_name]["Required"],
                                                                parameter_name)
            validate_parameter_value(parameter_name, self.__dict__[parameter_name],
                                     PYRATE_DEFAULT_CONFIGRATION[parameter_name]["MinValue"],
                                     PYRATE_DEFAULT_CONFIGRATION[parameter_name]["MaxValue"],
                                     PYRATE_DEFAULT_CONFIGRATION[parameter_name]["PossibleValues"])

        # bespoke parameter validation
        if self.refchipsize % 2 != 1:
            raise ValueError("Configuration parameters refchipsize must be odd: " + str(self.refchipsize))

        self.rows, self.cols = [int(no) for no in break_number_into_factors(NO_OF_PARALLEL_PROCESSES)]

        # create a temporary directory
        self.tmpdir = self.outdir / "tmpdir"
        self.tmpdir.mkdir(parents=True, exist_ok=True)
        self.tmpdir = str(self.tmpdir)

        # var no longer used
        self.APS_ELEVATION_EXT = None
        self.APS_INCIDENCE_EXT = None
        self.apscorrect = 0
        self.apsmethod = 0
        self.elevationmap = None
        self.incidencemap = None

        # define parallel processes that will run
        self.NUMEXPR_MAX_THREADS = str(NO_OF_PARALLEL_PROCESSES)

        # Validate file names supplied in list exist and contain correct epochs in file names

        if self.processor != 0:   # not roipac
            validate_file_list_values(self.obsdir, self.ifgfilelist, 2)
            validate_file_list_values(self.slcFileDir, self.slcfilelist, 1)

        self.coherence_file_paths = None
        if self.cohfiledir is not None and self.cohfilelist is not None:
            if self.processor != 0:  # not roipac
                validate_file_list_values(self.cohfiledir, self.cohfilelist, 1)
            self.coherence_file_paths = []
            for path_str in self.cohfilelist.read_text().split('\n'):
                # ignore empty lines in file
                if len(path_str) > 1:
                    self.coherence_file_paths.append(MultiplePaths(self.cohfiledir, path_str, self.ifglksx, self.ifgcropopt))

        self.header_file_paths = []
        for path_str in self.slcfilelist.read_text().split('\n'):
            # ignore empty lines in file
            if len(path_str) > 1:
                self.header_file_paths.append(MultiplePaths(self.slcFileDir, path_str, self.ifglksx, self.ifgcropopt))

        self.__get_interferrograms_files()

        self.dem_file = MultiplePaths(self.demfile.parents[0], self.demfile.name, self.ifglksx, self.ifgcropopt)

        # backward compatibility for string paths
        for key in self.__dict__:
            if isinstance(self.__dict__[key], PurePath):
                self.__dict__[key] = str(self.__dict__[key])

    def __get_interferrograms_files(self):
        self.interferogram_files = []
        for path_str in self.ifgfilelist.read_text().split('\n'):
            # ignore empty lines in file
            if len(path_str) > 1:
                self.interferogram_files.append(MultiplePaths(self.outdir, path_str, self.ifglksx, self.ifgcropopt))

<|endoftext|>


<commit_before>#!/usr/bin/env python
# Full license can be found in License.md
# Full author list can be found in .zenodo.json file
# DOI:10.5281/zenodo.1199703
# ----------------------------------------------------------------------------
""" Created as part of a Spring 2018 UTDesign project.
"""

import importlib
import numpy as np

class Constellation(object):
    """Manage and analyze data from multiple pysat Instruments.

    Parameters
    ----------
    const_module : string
        Name of a pysat constellation module
    instruments : list-like
        A list of pysat Instruments to include in the Constellation

    Attributes
    ----------
    instruments : list
        A list of pysat Instruments that make up the Constellation
    bounds : (datetime/filename/None, datetime/filename/None)
        bounds for loading data, supply array_like for a season with gaps.
        Users may provide as a tuple or tuple of lists, but the attribute is
        stored as a tuple of lists for consistency

    """

    # -----------------------------------------------------------------------
    # Define the magic methods

    def __init__(self, const_module=None, instruments=None):
        """
        Constructs a Constellation given a list of instruments or the name of
        a file with a pre-defined constellation.

        Parameters
        ----------
        const_module : string
            Name of a pysat constellation module
        instruments : list-like
            A list of pysat Instruments to include in the Constellation

        Raises
        ------
        ValueError
            When `instruments` is not list-like

        Note
        ----
        Omit `instruments` and `name` to create an empty constellation.

        """

        # Include Instruments from the constellation module, if it exists
        if const_module is not None:
            const = importlib.import_module(const_module)
            self.instruments = const.instruments
        else:
            self.instruments = []

        # Add any Instruments provided in the list
        if instruments is not None:
            test_instruments = np.asarray(instruments)
            if test_instruments.shape == ():
                raise ValueError('instruments argument must be list-like')

            self.instruments.extend(list(instruments))

        return

    def __getitem__(self, *args, **kwargs):
        """
        Look up a member Instrument by index.

        """

        return self.instruments.__getitem__(*args, **kwargs)

    def __repr__(self):
        """ Print the basic Constellation properties

        """

        out_str = "Constellation(instruments={:}) -> {:d} Instruments".format(
            self.instruments, len(self.instruments))
        return out_str

    def __str__(self):
        """ Print names of instruments within constellation

        """

        output_str = 'pysat Constellation object:\n'
        output_str += '---------------------------\n'

        ninst = len(self.instruments)

        if ninst > 0:
            output_str += "\nIndex Platform Name Tag Inst_ID\n"
            output_str += "-------------------------------\n"
            for i, inst in enumerate(self.instruments):
                output_str += "{:d} '{:s}' '{:s}' '{:s}' '{:s}'\n".format(
                    i, inst.platform, inst.name, inst.tag, inst.inst_id)
        else:
            output_str += "No loaded Instruments\n"

        return output_str

    # -----------------------------------------------------------------------
    # Define the public methods and properties

    @property
    def bounds(self):
        return self.instruments[0].bounds

    @bounds.setter
    def bounds(self, value=None):
        """ Sets boundaries for all Instruments in Constellation

        Parameters
        ----------
        value : tuple or NoneType
            Tuple containing starting time and ending time for Instrument
            bounds attribute or None (default=None)

        """

        for instrument in self.instruments:
            instrument.bounds = value

        return

    def custom_attach(self, *args, **kwargs):
        """Register a function to modify data of member Instruments.

        Parameters
        ----------
        *args : list reference
            References a list of input arguments
        **kwargs : dict reference
            References a dict of input keyword arguments

        Note
        ----
        Wraps Instrument.custom_attach; see Instrument.custom_attach for more
        information.

        """

        for instrument in self.instruments:
            instrument.custom_attach(*args, **kwargs)

        return

    def load(self, *args, **kwargs):
        """ Load instrument data into Instrument object.data

        Parameters
        ----------
        *args : list reference
            References a list of input arguments
        **kwargs : dict reference
            References a dict of input keyword arguments

        Note
        ----
        Wraps pysat.Instrument.load; see Instrument.load for more information.

        """

        for instrument in self.instruments:
            instrument.load(*args, **kwargs)

        return
<commit_msg>DOC: updated docstring section<commit_after>#!/usr/bin/env python
# Full license can be found in License.md
# Full author list can be found in .zenodo.json file
# DOI:10.5281/zenodo.1199703
# ----------------------------------------------------------------------------
""" Created as part of a Spring 2018 UTDesign project.
"""

import importlib
import numpy as np

class Constellation(object):
    """Manage and analyze data from multiple pysat Instruments.

    Parameters
    ----------
    const_module : string
        Name of a pysat constellation module
    instruments : list-like
        A list of pysat Instruments to include in the Constellation

    Attributes
    ----------
    instruments : list
        A list of pysat Instruments that make up the Constellation
    bounds : (datetime/filename/None, datetime/filename/None)
        bounds for loading data, supply array_like for a season with gaps.
        Users may provide as a tuple or tuple of lists, but the attribute is
        stored as a tuple of lists for consistency

    """

    # -----------------------------------------------------------------------
    # Define the magic methods

    def __init__(self, const_module=None, instruments=None):
        """
        Constructs a Constellation given a list of instruments or the name of
        a file with a pre-defined constellation.

        Parameters
        ----------
        const_module : string
            Name of a pysat constellation module
        instruments : list-like
            A list of pysat Instruments to include in the Constellation

        Raises
        ------
        ValueError
            When `instruments` is not list-like

        Note
        ----
        Omit `instruments` and `name` to create an empty constellation.

        """

        # Include Instruments from the constellation module, if it exists
        if const_module is not None:
            const = importlib.import_module(const_module)
            self.instruments = const.instruments
        else:
            self.instruments = []

        # Add any Instruments provided in the list
        if instruments is not None:
            test_instruments = np.asarray(instruments)
            if test_instruments.shape == ():
                raise ValueError('instruments argument must be list-like')

            self.instruments.extend(list(instruments))

        return

    def __getitem__(self, *args, **kwargs):
        """
        Look up a member Instrument by index.

        """

        return self.instruments.__getitem__(*args, **kwargs)

    def __repr__(self):
        """ Print the basic Constellation properties

        """

        out_str = "Constellation(instruments={:}) -> {:d} Instruments".format(
            self.instruments, len(self.instruments))
        return out_str

    def __str__(self):
        """ Print names of instruments within constellation

        """

        output_str = 'pysat Constellation object:\n'
        output_str += '---------------------------\n'

        ninst = len(self.instruments)

        if ninst > 0:
            output_str += "\nIndex Platform Name Tag Inst_ID\n"
            output_str += "-------------------------------\n"
            for i, inst in enumerate(self.instruments):
                output_str += "{:d} '{:s}' '{:s}' '{:s}' '{:s}'\n".format(
                    i, inst.platform, inst.name, inst.tag, inst.inst_id)
        else:
            output_str += "No loaded Instruments\n"

        return output_str

    # -----------------------------------------------------------------------
    # Define the public methods and properties

    @property
    def bounds(self):
        return self.instruments[0].bounds

    @bounds.setter
    def bounds(self, value=None):
        """ Sets boundaries for all Instruments in Constellation

        Parameters
        ----------
        value : tuple or NoneType
            Tuple containing starting time and ending time for Instrument
            bounds attribute or None (default=None)

        """

        for instrument in self.instruments:
            instrument.bounds = value

        return

    def custom_attach(self, *args, **kwargs):
        """Register a function to modify data of member Instruments.

        Parameters
        ----------
        *args : list reference
            References a list of input arguments
        **kwargs : dict reference
            References a dict of input keyword arguments

        See Also
        ---------
        Instrument.custom_attach : base method for attaching custom functions

        """

        for instrument in self.instruments:
            instrument.custom_attach(*args, **kwargs)

        return

    def load(self, *args, **kwargs):
        """ Load instrument data into Instrument object.data

        Parameters
        ----------
        *args : list reference
            References a list of input arguments
        **kwargs : dict reference
            References a dict of input keyword arguments

        See Also
        ---------
        Instrument.load : base method for loading Instrument data

        """

        for instrument in self.instruments:
            instrument.load(*args, **kwargs)

        return
<|endoftext|>